@shaxpir/duiduidui-models 1.15.0 → 1.17.0

package/dist/models/Phrase.d.ts

@@ -17,7 +17,6 @@ export interface Phrase {
     sense_rank: number;
     difficulty: number;
     pinyin: string;
-    pinyin_tokenized: string;
     transliteration: string;
     translation: string;
     notes: string;

package/dist/models/Term.d.ts

@@ -42,7 +42,6 @@ export interface TermPayload extends BayesianScore {
     implied_review_count: number;
     hanzi_count?: number;
     pinyin?: string;
-    pinyin_tokenized?: string;
     transliteration?: string;
     translation?: string;
     notes?: string;

package/dist/models/Term.js

@@ -38,7 +38,6 @@ class Term extends Content_1.Content {
                 difficulty: difficulty,
                 hanzi_count: textOrPhrase.length,
                 pinyin: '',
-                pinyin_tokenized: '',
                 transliteration: '',
                 translation: '',
                 notes: '',
@@ -71,7 +70,6 @@ class Term extends Content_1.Content {
                 hanzi_count: phrase.hanzi_count,
                 difficulty: phrase.difficulty,
                 pinyin: phrase.pinyin,
-                pinyin_tokenized: phrase.pinyin_tokenized,
                 transliteration: phrase.transliteration,
                 translation: phrase.translation,
                 notes: phrase.notes,

package/dist/util/PinyinParser.d.ts

@@ -16,32 +16,78 @@ export declare const PinyinParser: {
      */
     parseAll(text: string): PinyinParseResult[];
     /**
-     * Parse pinyin string that contains apostrophes
+     * Parse and return only the best parsing
+     * Returns null if the text cannot be parsed as pinyin
      */
-    parseWithApostrophes(text: string): PinyinParseResult[];
+    parse(text: string): string[] | null;
     /**
-     * Parse ambiguous pinyin string (no apostrophes) into all possible valid combinations
+     * Parse with erhua (儿化) awareness.
+     * Recognizes patterns like 'nǎr' where the trailing 'r' is an erhua suffix
+     * (the base syllable is valid but base+r is not).
+     * Erhua 'r' is expanded to 'er' in the output.
+     *
+     * Also handles 'r after apostrophe: nǎ'r → ['nǎ', 'er']
      */
-    parseAmbiguous(text: string): PinyinParseResult[];
+    parseWithErhua(text: string): string[] | null;
     /**
-     * Get the best parsing from multiple options
+     * Parse with erhua awareness, returning all possible parsings.
      */
-    getBestParsing(results: PinyinParseResult[]): PinyinParseResult | null;
+    parseAllWithErhua(text: string): PinyinParseResult[];
     /**
-     * Parse and return only the best parsing
-     * Returns null if the text cannot be parsed as pinyin
+     * Take a raw pinyin string (possibly with punctuation, spaces, apostrophes,
+     * and erhua) and return it with proper syllable spacing.
+     *
+     * The output is lowercased with punctuation stripped, producing clean
+     * space-separated syllables suitable for search indexing and tokenization.
+     * Tone marks are preserved.
+     *
+     * Throws if any letter sequence cannot be parsed as pinyin.
+     *
+     * Examples:
+     *   'nǐhǎo'        → 'nǐ hǎo'
+     *   'nǎr'           → 'nǎ er'
+     *   "xī'ān"         → 'xī ān'
+     *   "gē'rmen"       → 'gē er men'
+     *   'Nǐ hǎo!'       → 'nǐ hǎo'
+     *   'tā shēng bìng le, jīng cháng' → 'tā shēng bìng le jīng cháng'
      */
-    parse(text: string): string[] | null;
+    ensurePinyinSpacing(text: string): string;
     /**
      * Check if a string could be compound pinyin
      */
     couldBeCompoundPinyin(text: string): boolean;
     /**
      * Handle special cases like 'zhèr', 'nǎr', 'zhèlǐ', etc.
+     * @deprecated Use parseWithErhua() instead for erhua handling
      */
     parseWithSpecialCases(text: string): PinyinParseResult[];
+    /**
+     * Get the best parsing from multiple options.
+     *
+     * Primary criterion: fewer syllables (more natural word-level grouping).
+     * Tiebreaker: when two parses have the same syllable count, prefer the
+     * one whose first syllable is shorter. This avoids greedy long matches
+     * (e.g. "dàng" in "dàngāo") that leave uncommon standalone-vowel
+     * remainders (e.g. "āo") when a more balanced split exists ("dàn gāo").
+     */
+    getBestParsing(results: PinyinParseResult[]): PinyinParseResult | null;
     /**
      * Validate that all syllables in a parsing are legitimate
      */
     validateParsing(syllables: string[]): boolean;
+    /**
+     * Parse pinyin string that contains apostrophes.
+     * Splits on apostrophes and parses each segment independently.
+     */
+    _parseApostropheSplit(text: string, erhua: boolean): PinyinParseResult[];
+    /**
+     * Core DP parser. Finds all valid syllable decompositions of a pinyin string.
+     * When erhua=true, also recognizes trailing 'r' as erhua suffix (expanded to 'er')
+     * when the base syllable is valid but base+r is not.
+     *
+     * Works on original text (preserves case and tone marks) because
+     * PinyinValidator.isValidPinyin() handles normalization internally.
+     */
+    _parseDP(text: string, erhua: boolean): PinyinParseResult[];
+    parseAmbiguous(text: string): PinyinParseResult[];
 };

package/dist/util/PinyinParser.js

@@ -2,6 +2,8 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.PinyinParser = void 0;
 const PinyinValidator_1 = require("./PinyinValidator");
+// Check if a character is a Unicode letter (including diacritics)
+const isLetter = (char) => /\p{L}/u.test(char);
 exports.PinyinParser = {
     /**
      * Normalize apostrophes to straight apostrophe for pinyin parsing.
@@ -23,90 +25,112 @@ exports.PinyinParser = {
         const normalized = this.normalizeApostrophes(text.toLowerCase().trim());
         // Handle explicit apostrophes first
         if (normalized.includes("'")) {
-            return this.parseWithApostrophes(normalized);
+            return this._parseApostropheSplit(normalized, false);
         }
         // For strings without apostrophes, try all possible parsings
-        return this.parseAmbiguous(normalized);
+        return this._parseDP(normalized, false);
     },
     /**
-     * Parse pinyin string that contains apostrophes
+     * Parse and return only the best parsing
+     * Returns null if the text cannot be parsed as pinyin
      */
-    parseWithApostrophes(text) {
-        const parts = text.split("'");
-        const results = [];
-        // The first part doesn't have a preceding apostrophe
-        let firstPartParsings = this.parseAmbiguous(parts[0]);
-        // For subsequent parts, we know they start with a vowel (that's why there's an apostrophe)
-        for (let i = 1; i < parts.length; i++) {
-            const part = parts[i];
-            const partParsings = this.parseAmbiguous(part);
-            // Combine all previous results with all current part results
-            const newResults = [];
-            if (firstPartParsings.length === 0) {
-                firstPartParsings = [{ syllables: [] }];
-            }
-            for (const prevResult of firstPartParsings) {
-                for (const partResult of partParsings) {
-                    newResults.push({
-                        syllables: [...prevResult.syllables, ...partResult.syllables]
-                    });
-                }
-            }
-            firstPartParsings = newResults;
-        }
-        return firstPartParsings.filter((result) => result.syllables.length > 0);
+    parse(text) {
+        const results = this.parseAll(text);
+        const best = this.getBestParsing(results);
+        return best ? best.syllables : null;
     },
     /**
-     * Parse ambiguous pinyin string (no apostrophes) into all possible valid combinations
+     * Parse with erhua (儿化) awareness.
+     * Recognizes patterns like 'nǎr' where the trailing 'r' is an erhua suffix
+     * (the base syllable is valid but base+r is not).
+     * Erhua 'r' is expanded to 'er' in the output.
+     *
+     * Also handles 'r after apostrophe: nǎ'r → ['nǎ', 'er']
      */
-    parseAmbiguous(text) {
-        if (!text)
-            return [];
-        // Use dynamic programming to find all valid parsings
-        const memo = new Map();
-        const parseRecursive = (remaining) => {
-            if (remaining.length === 0) {
-                return [{ syllables: [] }];
-            }
-            if (memo.has(remaining)) {
-                return memo.get(remaining);
-            }
-            const results = [];
-            // Try all possible syllable lengths from longest to shortest
-            for (let len = Math.min(6, remaining.length); len >= 1; len--) {
-                const candidate = remaining.substring(0, len);
-                if (PinyinValidator_1.PinyinValidator.isValidPinyin(candidate)) {
-                    const restResults = parseRecursive(remaining.substring(len));
-                    for (const restResult of restResults) {
-                        results.push({
-                            syllables: [candidate, ...restResult.syllables]
-                        });
-                    }
-                }
-            }
-            memo.set(remaining, results);
-            return results;
-        };
-        return parseRecursive(text);
+    parseWithErhua(text) {
+        const results = this.parseAllWithErhua(text);
+        const best = this.getBestParsing(results);
+        return best ? best.syllables : null;
     },
     /**
-     * Get the best parsing from multiple options
+     * Parse with erhua awareness, returning all possible parsings.
      */
-    getBestParsing(results) {
-        if (results.length === 0)
-            return null;
-        // Prefer fewer syllables (more natural parsing)
-        results.sort((a, b) => a.syllables.length - b.syllables.length);
-        return results[0];
+    parseAllWithErhua(text) {
+        if (!text || text.length === 0)
+            return [];
+        let normalized = this.normalizeApostrophes(text.toLowerCase().trim());
+        // Expand 'r (erhua via apostrophe) to 'er before splitting
+        normalized = normalized.replace(/'r/g, "'er");
+        if (normalized.includes("'")) {
+            return this._parseApostropheSplit(normalized, true);
+        }
+        return this._parseDP(normalized, true);
     },
     /**
-     * Parse and return only the best parsing
-     * Returns null if the text cannot be parsed as pinyin
+     * Take a raw pinyin string (possibly with punctuation, spaces, apostrophes,
+     * and erhua) and return it with proper syllable spacing.
+     *
+     * The output is lowercased with punctuation stripped, producing clean
+     * space-separated syllables suitable for search indexing and tokenization.
+     * Tone marks are preserved.
+     *
+     * Throws if any letter sequence cannot be parsed as pinyin.
+     *
+     * Examples:
+     *   'nǐhǎo'        → 'nǐ hǎo'
+     *   'nǎr'           → 'nǎ er'
+     *   "xī'ān"         → 'xī ān'
+     *   "gē'rmen"       → 'gē er men'
+     *   'Nǐ hǎo!'       → 'nǐ hǎo'
+     *   'tā shēng bìng le, jīng cháng' → 'tā shēng bìng le jīng cháng'
      */
-    parse(text) {
-        const results = this.parseAll(text);
-        const best = this.getBestParsing(results);
-        return best ? best.syllables : null;
+    ensurePinyinSpacing(text) {
+        // Normalize apostrophes and lowercase
+        text = this.normalizeApostrophes(text).toLowerCase();
+        // Expand 'r (erhua via apostrophe) to 'er, then replace apostrophes with spaces
+        text = text.replace(/'r/g, "'er");
+        text = text.replace(/'/g, ' ');
+        text = text.replace(/ +/g, ' ');
+        const parts = [];
+        let i = 0;
+        while (i < text.length) {
+            if (isLetter(text[i])) {
+                let j = i;
+                while (j < text.length && isLetter(text[j]))
+                    j++;
+                const letterRun = text.substring(i, j);
+                const results = this._parseDP(letterRun, true);
+                const best = this.getBestParsing(results);
+                if (best) {
+                    parts.push(best.syllables.join(' '));
+                }
+                else {
+                    throw new Error(`Unable to parse as pinyin: "${letterRun}" in "${text}"`);
+                }
+                i = j;
+            }
+            else {
+                // Non-letter run: preserve ellipsis (…) as a token (used in patterns
+                // like 太…了), but strip all other punctuation and collapse to a space.
+                let j = i;
+                while (j < text.length && !isLetter(text[j]))
+                    j++;
+                const nonLetterRun = text.substring(i, j);
+                if (nonLetterRun.includes('…')) {
+                    // Preserve ellipsis with surrounding spaces
+                    if (parts.length > 0)
+                        parts.push(' ');
+                    parts.push('…');
+                    if (j < text.length)
+                        parts.push(' ');
+                }
+                else if (parts.length > 0 && j < text.length) {
+                    parts.push(' ');
+                }
+                i = j;
+            }
+        }
+        return parts.join('').trim();
     },
     /**
      * Check if a string could be compound pinyin
@@ -119,6 +143,7 @@ exports.PinyinParser = {
     },
     /**
      * Handle special cases like 'zhèr', 'nǎr', 'zhèlǐ', etc.
+     * @deprecated Use parseWithErhua() instead for erhua handling
      */
     parseWithSpecialCases(text) {
         const normalized = text.toLowerCase();
@@ -129,7 +154,7 @@ exports.PinyinParser = {
                     if (PinyinValidator_1.PinyinValidator.isValidPinyin(match)) {
                         return [{ syllables: [match] }];
                     }
-                    const baseResults = this.parseAmbiguous(base);
+                    const baseResults = this._parseDP(base, false);
                     return baseResults.map((result) => ({
                         syllables: [...result.syllables, 'r']
                     }));
@@ -142,12 +167,117 @@ exports.PinyinParser = {
             }
         }
         // If no special cases match, fall back to regular parsing
-        return this.parseAmbiguous(normalized);
+        return this._parseDP(normalized, false);
+    },
+    /**
+     * Get the best parsing from multiple options.
+     *
+     * Primary criterion: fewer syllables (more natural word-level grouping).
+     * Tiebreaker: when two parses have the same syllable count, prefer the
+     * one whose first syllable is shorter. This avoids greedy long matches
+     * (e.g. "dàng" in "dàngāo") that leave uncommon standalone-vowel
+     * remainders (e.g. "āo") when a more balanced split exists ("dàn gāo").
+     */
+    getBestParsing(results) {
+        if (results.length === 0)
+            return null;
+        results.sort((a, b) => {
+            // Primary: fewer syllables
+            if (a.syllables.length !== b.syllables.length) {
+                return a.syllables.length - b.syllables.length;
+            }
+            // Tiebreaker: shorter first syllable
+            for (let i = 0; i < a.syllables.length; i++) {
+                if (a.syllables[i].length !== b.syllables[i].length) {
+                    return a.syllables[i].length - b.syllables[i].length;
+                }
+            }
+            return 0;
+        });
+        return results[0];
     },
     /**
      * Validate that all syllables in a parsing are legitimate
      */
     validateParsing(syllables) {
         return syllables.every(syllable => PinyinValidator_1.PinyinValidator.isValidPinyin(syllable));
-    }
+    },
+    // ---------------------------------------------------------------------------
+    // Internal methods
+    // ---------------------------------------------------------------------------
+    /**
+     * Parse pinyin string that contains apostrophes.
+     * Splits on apostrophes and parses each segment independently.
+     */
+    _parseApostropheSplit(text, erhua) {
+        const parts = text.split("'");
+        let combined = [{ syllables: [] }];
+        for (const part of parts) {
+            if (!part)
+                continue; // skip empty segments from consecutive apostrophes
+            const partResults = this._parseDP(part, erhua);
+            if (partResults.length === 0)
+                return [];
+            const newCombined = [];
+            for (const prev of combined) {
+                for (const curr of partResults) {
+                    newCombined.push({
+                        syllables: [...prev.syllables, ...curr.syllables]
+                    });
+                }
+            }
+            combined = newCombined;
+        }
+        return combined.filter((result) => result.syllables.length > 0);
+    },
+    /**
+     * Core DP parser. Finds all valid syllable decompositions of a pinyin string.
+     * When erhua=true, also recognizes trailing 'r' as erhua suffix (expanded to 'er')
+     * when the base syllable is valid but base+r is not.
+     *
+     * Works on original text (preserves case and tone marks) because
+     * PinyinValidator.isValidPinyin() handles normalization internally.
+     */
+    _parseDP(text, erhua) {
+        if (!text)
+            return [];
+        const memo = new Map();
+        const parseFromPos = (pos) => {
+            if (pos >= text.length)
+                return [{ syllables: [] }];
+            if (memo.has(pos))
+                return memo.get(pos);
+            const results = [];
+            const maxLen = Math.min(6, text.length - pos);
+            for (let len = maxLen; len >= 1; len--) {
+                const candidate = text.substring(pos, pos + len);
+                if (PinyinValidator_1.PinyinValidator.isValidPinyin(candidate)) {
+                    // Standard match
+                    for (const rest of parseFromPos(pos + len)) {
+                        results.push({ syllables: [candidate, ...rest.syllables] });
+                    }
+                    // Erhua: if next char is 'r' and candidate+'r' is NOT a valid syllable,
+                    // treat the 'r' as an erhua suffix → expand to 'er'
+                    if (erhua) {
+                        const nextPos = pos + len;
+                        if (nextPos < text.length && text[nextPos].toLowerCase() === 'r') {
+                            const withR = text.substring(pos, nextPos + 1);
+                            if (!PinyinValidator_1.PinyinValidator.isValidPinyin(withR)) {
+                                for (const rest of parseFromPos(nextPos + 1)) {
+                                    results.push({ syllables: [candidate, 'er', ...rest.syllables] });
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+            memo.set(pos, results);
+            return results;
+        };
+        return parseFromPos(0);
+    },
+    // Legacy aliases for backward compatibility
+    parseAmbiguous(text) {
+        return this._parseDP(text, false);
+    },
 };

package/dist/util/PinyinValidator.d.ts

@@ -11,6 +11,18 @@
  * convention the database might use.
  */
 export declare const PinyinValidator: {
+    /**
+     * Strip tone marks from pinyin, lowercasing the result.
+     * Maps ǖǘǚǜ → ü, toned vowels → base vowels, v with combining marks → v.
+     * Used for search/matching where tone distinctions are irrelevant.
+     */
+    stripToneMarks(text: string): string;
+    /**
+     * Remove all accent/tone marks from pinyin without lowercasing.
+     * Maps ü and ǖǘǚǜ → v (pipeline convention for normalized pinyin).
+     * Used by the data pipeline to create pinyin_normalized fields.
+     */
+    removeAccentMarks(text: string): string;
     /**
      * Check if a string is a valid pinyin syllable (with or without tone marks)
      */

package/dist/util/PinyinValidator.js

@@ -197,6 +197,24 @@ const TONE_MAP = {
     'ū': 'u', 'ú': 'u', 'ǔ': 'u', 'ù': 'u',
     'ǖ': 'ü', 'ǘ': 'ü', 'ǚ': 'ü', 'ǜ': 'ü'
 };
+// Maps toned vowels to base letters with ü→v (pipeline convention)
+// Includes both lowercase and uppercase variants.
+const ACCENT_MAP = {
+    'ā': 'a', 'á': 'a', 'ǎ': 'a', 'à': 'a',
+    'Ā': 'A', 'Á': 'A', 'Ǎ': 'A', 'À': 'A',
+    'ē': 'e', 'é': 'e', 'ě': 'e', 'è': 'e',
+    'Ē': 'E', 'É': 'E', 'Ě': 'E', 'È': 'E',
+    'ī': 'i', 'í': 'i', 'ǐ': 'i', 'ì': 'i',
+    'Ī': 'I', 'Í': 'I', 'Ǐ': 'I', 'Ì': 'I',
+    'ō': 'o', 'ó': 'o', 'ǒ': 'o', 'ò': 'o',
+    'Ō': 'O', 'Ó': 'O', 'Ǒ': 'O', 'Ò': 'O',
+    'ū': 'u', 'ú': 'u', 'ǔ': 'u', 'ù': 'u',
+    'Ū': 'U', 'Ú': 'U', 'Ǔ': 'U', 'Ù': 'U',
+    'ǖ': 'v', 'ǘ': 'v', 'ǚ': 'v', 'ǜ': 'v',
+    'Ǖ': 'V', 'Ǘ': 'V', 'Ǚ': 'V', 'Ǜ': 'V',
+    'ü': 'v', 'Ü': 'V', 'ǹ': 'n', 'Ǹ': 'N',
+};
+const ACCENT_MARKS = /[āáǎàĀÁǍÀēéěèĒÉĚÈīíǐìĪÍǏÌōóǒòŌÓǑÒūúǔùŪÚǓÙǖǘǚǜǕǗǙǛüÜǹǸ]|v[\u0301\u030C\u0300]?/g;
 function stripToneMarks(text) {
     return text.toLowerCase().replace(TONE_MARKS, (match) => {
         if (match.startsWith('v'))
@@ -205,6 +223,26 @@ function stripToneMarks(text) {
     });
 }
 exports.PinyinValidator = {
+    /**
+     * Strip tone marks from pinyin, lowercasing the result.
+     * Maps ǖǘǚǜ → ü, toned vowels → base vowels, v with combining marks → v.
+     * Used for search/matching where tone distinctions are irrelevant.
+     */
+    stripToneMarks(text) {
+        return stripToneMarks(text);
+    },
+    /**
+     * Remove all accent/tone marks from pinyin without lowercasing.
+     * Maps ü and ǖǘǚǜ → v (pipeline convention for normalized pinyin).
+     * Used by the data pipeline to create pinyin_normalized fields.
+     */
+    removeAccentMarks(text) {
+        return text.replace(ACCENT_MARKS, (match) => {
+            if (match.startsWith('v'))
+                return 'v';
+            return ACCENT_MAP[match] || match;
+        });
+    },
     /**
      * Check if a string is a valid pinyin syllable (with or without tone marks)
      */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@shaxpir/duiduidui-models",
-  "version": "1.15.0",
+  "version": "1.17.0",
   "repository": {
     "type": "git",
     "url": "https://github.com/shaxpir/duiduidui-models"