@shaxpir/duiduidui-models 1.9.26 → 1.9.28

package/dist/models/Condition.js

@@ -160,6 +160,14 @@ exports.Condition = {
                 .join(', ');
             errors.push(`Conditions [${conflictingTypes}] in 'any' section require a term record, but has_term is false`);
         }
+        // Check for contradictory difficulty conditions (min >= max)
+        const difficultyCondition = conditions.all?.find(c => c.type === 'difficulty');
+        if (difficultyCondition) {
+            const { min, max } = difficultyCondition;
+            if (min !== undefined && max !== undefined && min >= max) {
+                errors.push(`Min difficulty (${min}) must be less than max difficulty (${max})`);
+            }
+        }
         return errors;
     },
     // Backward compatibility aliases (deprecated - use requiresStarred/allowsStarred/excludesStarred instead)

package/dist/util/PinyinParser.d.ts

@@ -0,0 +1,47 @@
+/**
+ * Parser for handling compound pinyin strings with proper syllable boundaries
+ */
+export interface PinyinParseResult {
+    syllables: string[];
+}
+export declare const PinyinParser: {
+    /**
+     * Normalize apostrophes to straight apostrophe for pinyin parsing.
+     * Pinyin uses straight apostrophe (') as the standard syllable separator.
+     */
+    normalizeApostrophes(text: string): string;
+    /**
+     * Parse a pinyin string into all possible valid syllable combinations
+     * Returns multiple parsing options for ambiguous cases
+     */
+    parseAll(text: string): PinyinParseResult[];
+    /**
+     * Parse pinyin string that contains apostrophes
+     */
+    parseWithApostrophes(text: string): PinyinParseResult[];
+    /**
+     * Parse ambiguous pinyin string (no apostrophes) into all possible valid combinations
+     */
+    parseAmbiguous(text: string): PinyinParseResult[];
+    /**
+     * Get the best parsing from multiple options
+     */
+    getBestParsing(results: PinyinParseResult[]): PinyinParseResult | null;
+    /**
+     * Parse and return only the best parsing
+     * Returns null if the text cannot be parsed as pinyin
+     */
+    parse(text: string): string[] | null;
+    /**
+     * Check if a string could be compound pinyin
+     */
+    couldBeCompoundPinyin(text: string): boolean;
+    /**
+     * Handle special cases like 'zhèr', 'nǎr', 'zhèlǐ', etc.
+     */
+    parseWithSpecialCases(text: string): PinyinParseResult[];
+    /**
+     * Validate that all syllables in a parsing are legitimate
+     */
+    validateParsing(syllables: string[]): boolean;
+};

package/dist/util/PinyinParser.js

@@ -0,0 +1,153 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PinyinParser = void 0;
+const PinyinValidator_1 = require("./PinyinValidator");
+exports.PinyinParser = {
+    /**
+     * Normalize apostrophes to straight apostrophe for pinyin parsing.
+     * Pinyin uses straight apostrophe (') as the standard syllable separator.
+     */
+    normalizeApostrophes(text) {
+        // U+2018 = ' (left single quotation mark)
+        // U+2019 = ' (right single quotation mark / curly apostrophe)
+        return text.replace(/[\u2018\u2019]/g, "'");
+    },
+    /**
+     * Parse a pinyin string into all possible valid syllable combinations
+     * Returns multiple parsing options for ambiguous cases
+     */
+    parseAll(text) {
+        if (!text || text.length === 0)
+            return [];
+        // Normalize the input (including curly apostrophes to straight)
+        const normalized = this.normalizeApostrophes(text.toLowerCase().trim());
+        // Handle explicit apostrophes first
+        if (normalized.includes("'")) {
+            return this.parseWithApostrophes(normalized);
+        }
+        // For strings without apostrophes, try all possible parsings
+        return this.parseAmbiguous(normalized);
+    },
+    /**
+     * Parse pinyin string that contains apostrophes
+     */
+    parseWithApostrophes(text) {
+        const parts = text.split("'");
+        const results = [];
+        // The first part doesn't have a preceding apostrophe
+        let firstPartParsings = this.parseAmbiguous(parts[0]);
+        // For subsequent parts, we know they start with a vowel (that's why there's an apostrophe)
+        for (let i = 1; i < parts.length; i++) {
+            const part = parts[i];
+            const partParsings = this.parseAmbiguous(part);
+            // Combine all previous results with all current part results
+            const newResults = [];
+            if (firstPartParsings.length === 0) {
+                firstPartParsings = [{ syllables: [] }];
+            }
+            for (const prevResult of firstPartParsings) {
+                for (const partResult of partParsings) {
+                    newResults.push({
+                        syllables: [...prevResult.syllables, ...partResult.syllables]
+                    });
+                }
+            }
+            firstPartParsings = newResults;
+        }
+        return firstPartParsings.filter((result) => result.syllables.length > 0);
+    },
+    /**
+     * Parse ambiguous pinyin string (no apostrophes) into all possible valid combinations
+     */
+    parseAmbiguous(text) {
+        if (!text)
+            return [];
+        // Use dynamic programming to find all valid parsings
+        const memo = new Map();
+        const parseRecursive = (remaining) => {
+            if (remaining.length === 0) {
+                return [{ syllables: [] }];
+            }
+            if (memo.has(remaining)) {
+                return memo.get(remaining);
+            }
+            const results = [];
+            // Try all possible syllable lengths from longest to shortest
+            for (let len = Math.min(6, remaining.length); len >= 1; len--) {
+                const candidate = remaining.substring(0, len);
+                if (PinyinValidator_1.PinyinValidator.isValidPinyin(candidate)) {
+                    const restResults = parseRecursive(remaining.substring(len));
+                    for (const restResult of restResults) {
+                        results.push({
+                            syllables: [candidate, ...restResult.syllables]
+                        });
+                    }
+                }
+            }
+            memo.set(remaining, results);
+            return results;
+        };
+        return parseRecursive(text);
+    },
+    /**
+     * Get the best parsing from multiple options
+     */
+    getBestParsing(results) {
+        if (results.length === 0)
+            return null;
+        // Prefer fewer syllables (more natural parsing)
+        results.sort((a, b) => a.syllables.length - b.syllables.length);
+        return results[0];
+    },
+    /**
+     * Parse and return only the best parsing
+     * Returns null if the text cannot be parsed as pinyin
+     */
+    parse(text) {
+        const results = this.parseAll(text);
+        const best = this.getBestParsing(results);
+        return best ? best.syllables : null;
+    },
+    /**
+     * Check if a string could be compound pinyin
+     */
+    couldBeCompoundPinyin(text) {
+        const results = this.parseAll(text);
+        // Only consider it compound if the BEST parsing has multiple syllables
+        const best = this.getBestParsing(results);
+        return best !== null && best.syllables.length > 1;
+    },
+    /**
+     * Handle special cases like 'zhèr', 'nǎr', 'zhèlǐ', etc.
+     */
+    parseWithSpecialCases(text) {
+        const normalized = text.toLowerCase();
+        // Handle common r-colored syllables (儿化音)
+        const rColoredPatterns = [
+            { pattern: /^(.+)r$/, replacement: (match, base) => {
+                    // If base + 'r' is valid, keep it; otherwise try to parse base separately
+                    if (PinyinValidator_1.PinyinValidator.isValidPinyin(match)) {
+                        return [{ syllables: [match] }];
+                    }
+                    const baseResults = this.parseAmbiguous(base);
+                    return baseResults.map((result) => ({
+                        syllables: [...result.syllables, 'r']
+                    }));
+                } }
+        ];
+        for (const { pattern, replacement } of rColoredPatterns) {
+            const match = normalized.match(pattern);
+            if (match) {
+                return replacement(match[0], match[1]);
+            }
+        }
+        // If no special cases match, fall back to regular parsing
+        return this.parseAmbiguous(normalized);
+    },
+    /**
+     * Validate that all syllables in a parsing are legitimate
+     */
+    validateParsing(syllables) {
+        return syllables.every(syllable => PinyinValidator_1.PinyinValidator.isValidPinyin(syllable));
+    }
+};

package/dist/util/PinyinValidator.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Utility for validating pinyin tokens
+ */
+export declare const PinyinValidator: {
+    /**
+     * Check if a string is a valid pinyin syllable (with or without tone marks)
+     */
+    isValidPinyin(text: string): boolean;
+    /**
+     * Check if a string could be a pinyin token (more lenient, for prefix matching)
+     */
+    couldBePinyinPrefix(text: string): boolean;
+    /**
+     * Split a string into potential pinyin tokens (for compound pinyin like "nihao")
+     * Returns empty array if not valid pinyin
+     */
+    splitPinyinTokens(text: string): string[];
+};

package/dist/util/PinyinValidator.js

@@ -0,0 +1,162 @@
+"use strict";
+/**
+ * Utility for validating pinyin tokens
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PinyinValidator = void 0;
+// Valid pinyin initials (including empty string for syllables like 'a', 'e')
+const INITIALS = new Set([
+    '', // empty initial
+    'b', 'p', 'm', 'f',
+    'd', 't', 'n', 'l',
+    'g', 'k', 'h',
+    'j', 'q', 'x',
+    'zh', 'ch', 'sh', 'r',
+    'z', 'c', 's',
+    'y', 'w'
+]);
+// Valid pinyin finals
+const FINALS = new Set([
+    'a', 'o', 'e', 'i', 'u', 'ü', 'v', // 'v' is often used instead of 'ü'
+    'ai', 'ei', 'ui', 'ao', 'ou', 'iu',
+    'ie', 'üe', 've', 'ue', 'er',
+    'an', 'en', 'in', 'un', 'ün', 'vn',
+    'ang', 'eng', 'ing', 'ong',
+    'ia', 'iao', 'ian', 'iang', 'iong',
+    'ua', 'uo', 'uai', 'uan', 'uang',
+    'üan', 'van', 'yuan'
+]);
+// Common standalone syllables
+const STANDALONE_SYLLABLES = new Set([
+    'a', 'ai', 'an', 'ang', 'ao',
+    'e', 'ei', 'en', 'eng', 'er',
+    'o', 'ou',
+    'yi', 'ya', 'yao', 'ye', 'you', 'yan', 'yang', 'yin', 'ying', 'yong',
+    'wu', 'wa', 'wo', 'wai', 'wei', 'wan', 'wang', 'wen', 'weng',
+    'yu', 'yue', 'yuan', 'yun'
+]);
+// Tone marks that might appear in pinyin (including v with combining tone marks)
+const TONE_MARKS = /[āáǎàēéěèīíǐìōóǒòūúǔùǖǘǚǜ]|v[\u0301\u030C\u0300]?/g;
+exports.PinyinValidator = {
+    /**
+     * Check if a string is a valid pinyin syllable (with or without tone marks)
+     */
+    isValidPinyin(text) {
+        if (!text || text.length === 0)
+            return false;
+        // Convert to lowercase and remove tone marks for validation
+        const normalized = text.toLowerCase().replace(TONE_MARKS, (match) => {
+            // Convert tone marks back to base vowels
+            const toneMap = {
+                'ā': 'a', 'á': 'a', 'ǎ': 'a', 'à': 'a',
+                'ē': 'e', 'é': 'e', 'ě': 'e', 'è': 'e',
+                'ī': 'i', 'í': 'i', 'ǐ': 'i', 'ì': 'i',
+                'ō': 'o', 'ó': 'o', 'ǒ': 'o', 'ò': 'o',
+                'ū': 'u', 'ú': 'u', 'ǔ': 'u', 'ù': 'u',
+                'ǖ': 'ü', 'ǘ': 'ü', 'ǚ': 'ü', 'ǜ': 'ü'
+            };
+            // Handle v with combining tone marks
+            if (match.startsWith('v')) {
+                return 'v';
+            }
+            return toneMap[match] || match;
+        });
+        // Check if it's a standalone syllable
+        if (STANDALONE_SYLLABLES.has(normalized)) {
+            return true;
+        }
+        // Try to parse as initial + final
+        // Check longest possible initial first (2 chars)
+        if (normalized.length >= 2) {
+            const possibleInitial2 = normalized.substring(0, 2);
+            if (INITIALS.has(possibleInitial2)) {
+                const remaining = normalized.substring(2);
+                if (FINALS.has(remaining)) {
+                    return true;
+                }
+            }
+        }
+        // Check single character initial
+        if (normalized.length >= 1) {
+            const possibleInitial1 = normalized.substring(0, 1);
+            if (INITIALS.has(possibleInitial1)) {
+                const remaining = normalized.substring(1);
+                if (FINALS.has(remaining)) {
+                    return true;
+                }
+            }
+        }
+        // Check if the whole string is a valid final (for syllables without initials)
+        if (INITIALS.has('') && FINALS.has(normalized)) {
+            return true;
+        }
+        return false;
+    },
+    /**
+     * Check if a string could be a pinyin token (more lenient, for prefix matching)
+     */
+    couldBePinyinPrefix(text) {
+        if (!text || text.length === 0)
+            return false;
+        const normalized = text.toLowerCase().replace(TONE_MARKS, (match) => {
+            const toneMap = {
+                'ā': 'a', 'á': 'a', 'ǎ': 'a', 'à': 'a',
+                'ē': 'e', 'é': 'e', 'ě': 'e', 'è': 'e',
+                'ī': 'i', 'í': 'i', 'ǐ': 'i', 'ì': 'i',
+                'ō': 'o', 'ó': 'o', 'ǒ': 'o', 'ò': 'o',
+                'ū': 'u', 'ú': 'u', 'ǔ': 'u', 'ù': 'u',
+                'ǖ': 'ü', 'ǘ': 'ü', 'ǚ': 'ü', 'ǜ': 'ü'
+            };
+            // Handle v with combining tone marks
+            if (match.startsWith('v')) {
+                return 'v';
+            }
+            return toneMap[match] || match;
+        });
+        // Check if it's already valid pinyin
+        if (this.isValidPinyin(text))
+            return true;
+        // Check if any standalone syllable starts with this prefix
+        for (const syllable of STANDALONE_SYLLABLES) {
+            if (syllable.startsWith(normalized))
+                return true;
+        }
+        // Check if it could be the start of initial + final combination
+        // Check if it matches any initial exactly or partially
+        for (const initial of INITIALS) {
+            if (initial.startsWith(normalized) || normalized.startsWith(initial)) {
+                return true;
+            }
+        }
+        return false;
+    },
+    /**
+     * Split a string into potential pinyin tokens (for compound pinyin like "nihao")
+     * Returns empty array if not valid pinyin
+     */
+    splitPinyinTokens(text) {
+        // This is a simplified version - full implementation would need
+        // more sophisticated parsing to handle ambiguous cases
+        const tokens = [];
+        const normalized = text.toLowerCase();
+        let remaining = normalized;
+        while (remaining.length > 0) {
+            let found = false;
+            // Try to match the longest possible valid pinyin syllable
+            for (let len = Math.min(6, remaining.length); len >= 1; len--) {
+                const candidate = remaining.substring(0, len);
+                if (this.isValidPinyin(candidate)) {
+                    tokens.push(candidate);
+                    remaining = remaining.substring(len);
+                    found = true;
+                    break;
+                }
+            }
+            // If no valid syllable found, this isn't valid pinyin
+            if (!found) {
+                return [];
+            }
+        }
+        return tokens;
+    }
+};

package/dist/util/SearchPreprocessor.d.ts

@@ -0,0 +1,21 @@
+/**
+ * Utility functions for preprocessing search queries
+ */
+/**
+ * Cleans and normalizes a search string for database queries
+ * - Removes trailing punctuation
+ * - Trims whitespace
+ * - Preserves internal punctuation (e.g., apostrophes in "don't")
+ *
+ * @param searchString The raw search string from user input
+ * @returns The cleaned search string
+ */
+export declare function preprocessSearchString(searchString: string | null | undefined): string;
+/**
+ * Tests if a string contains Chinese characters
+ */
+export declare function containsChineseCharacters(text: string): boolean;
+/**
+ * Tests if a string contains only ASCII punctuation
+ */
+export declare function isOnlyPunctuation(text: string): boolean;

package/dist/util/SearchPreprocessor.js

@@ -0,0 +1,42 @@
+"use strict";
+/**
+ * Utility functions for preprocessing search queries
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.preprocessSearchString = preprocessSearchString;
+exports.containsChineseCharacters = containsChineseCharacters;
+exports.isOnlyPunctuation = isOnlyPunctuation;
+/**
+ * Cleans and normalizes a search string for database queries
+ * - Removes trailing punctuation
+ * - Trims whitespace
+ * - Preserves internal punctuation (e.g., apostrophes in "don't")
+ *
+ * @param searchString The raw search string from user input
+ * @returns The cleaned search string
+ */
+function preprocessSearchString(searchString) {
+    if (!searchString) {
+        return '';
+    }
+    // Trim whitespace
+    let cleaned = searchString.trim();
+    // Remove common trailing punctuation that users might accidentally include
+    // but preserve internal punctuation like apostrophes
+    cleaned = cleaned.replace(/[!?.,:;]+$/, '');
+    // Remove leading punctuation as well
+    cleaned = cleaned.replace(/^[!?.,:;]+/, '');
+    return cleaned;
+}
+/**
+ * Tests if a string contains Chinese characters
+ */
+function containsChineseCharacters(text) {
+    return /[\u4e00-\u9fff]/.test(text);
+}
+/**
+ * Tests if a string contains only ASCII punctuation
+ */
+function isOnlyPunctuation(text) {
+    return /^[!?.,:;]+$/.test(text);
+}

package/dist/util/SearchTokenizer.d.ts

@@ -0,0 +1,72 @@
+/**
+ * Token types for search query tokenization
+ */
+export declare enum TokenType {
+    HANZI = "hanzi",
+    PINYIN = "pinyin",
+    ENGLISH = "english",
+    AMBIGUOUS = "ambiguous",// Could be either pinyin or English
+    PUNCTUATION = "punctuation",
+    NUMBER = "number"
+}
+/**
+ * Represents a single token from the search query
+ */
+export interface SearchToken {
+    text: string;
+    type: TokenType;
+    normalized: string;
+    isPossiblePinyin?: boolean;
+    pinyinVariants?: string[];
+}
+/**
+ * Result of tokenizing a search string
+ */
+export interface TokenizedSearch {
+    tokens: SearchToken[];
+    hanziTokens: SearchToken[];
+    pinyinTokens: SearchToken[];
+    englishTokens: SearchToken[];
+    ambiguousTokens: SearchToken[];
+    hasHanzi: boolean;
+    hasPinyin: boolean;
+    hasEnglish: boolean;
+}
+/**
+ * Tokenizes and classifies search strings for multi-modal search
+ */
+export declare class SearchTokenizer {
+    private static readonly BOUNDARY_PUNCTUATION;
+    private static readonly EMBEDDED_PUNCTUATION;
+    private static readonly APOSTROPHES;
+    private static readonly CHINESE_PUNCTUATION;
+    private static readonly ALL_PUNCTUATION;
+    private static readonly HANZI_REGEX;
+    private static readonly PINYIN_TONE_MARKS;
+    /**
+     * Main tokenization method
+     */
+    static tokenize(searchString: string): TokenizedSearch;
+    /**
+     * Splits input string into tokens, handling punctuation intelligently
+     */
+    private static splitIntoTokens;
+    /**
+     * Normalize apostrophes to curly for English database matching.
+     * The database stores English contractions with curly apostrophes (U+2019).
+     */
+    private static normalizeApostrophesToCurly;
+    /**
+     * Check if a token with embedded apostrophes could be pinyin with syllable separators.
+     * Returns the parsed syllables if valid, null otherwise.
+     */
+    private static tryParsePinyinWithApostrophe;
+    /**
+     * Classifies a single token
+     */
+    private static classifyToken;
+    /**
+     * Checks if a string contains only hanzi characters
+     */
+    private static isAllHanzi;
+}

package/dist/util/SearchTokenizer.js

@@ -0,0 +1,300 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SearchTokenizer = exports.TokenType = void 0;
+const PinyinParser_1 = require("./PinyinParser");
+/**
+ * Token types for search query tokenization
+ */
+var TokenType;
+(function (TokenType) {
+    TokenType["HANZI"] = "hanzi";
+    TokenType["PINYIN"] = "pinyin";
+    TokenType["ENGLISH"] = "english";
+    TokenType["AMBIGUOUS"] = "ambiguous";
+    TokenType["PUNCTUATION"] = "punctuation";
+    TokenType["NUMBER"] = "number";
+})(TokenType || (exports.TokenType = TokenType = {}));
+/**
+ * Tokenizes and classifies search strings for multi-modal search
+ */
+class SearchTokenizer {
+    /**
+     * Main tokenization method
+     */
+    static tokenize(searchString) {
+        if (!searchString || !searchString.trim()) {
+            return {
+                tokens: [],
+                hanziTokens: [],
+                pinyinTokens: [],
+                englishTokens: [],
+                ambiguousTokens: [],
+                hasHanzi: false,
+                hasPinyin: false,
+                hasEnglish: false
+            };
+        }
+        // First, split the string into raw tokens
+        const rawTokens = this.splitIntoTokens(searchString);
+        // Classify each token
+        const classifiedTokens = rawTokens.map(token => this.classifyToken(token));
+        // Group tokens by type
+        const hanziTokens = classifiedTokens.filter(t => t.type === TokenType.HANZI);
+        const pinyinTokens = classifiedTokens.filter(t => t.type === TokenType.PINYIN);
+        const englishTokens = classifiedTokens.filter(t => t.type === TokenType.ENGLISH);
+        const ambiguousTokens = classifiedTokens.filter(t => t.type === TokenType.AMBIGUOUS);
+        return {
+            tokens: classifiedTokens,
+            hanziTokens,
+            pinyinTokens,
+            englishTokens,
+            ambiguousTokens,
+            hasHanzi: hanziTokens.length > 0,
+            hasPinyin: pinyinTokens.length > 0 || ambiguousTokens.some(t => t.isPossiblePinyin),
+            hasEnglish: englishTokens.length > 0 || ambiguousTokens.length > 0
+        };
+    }
+    /**
+     * Splits input string into tokens, handling punctuation intelligently
+     */
+    static splitIntoTokens(input) {
+        const tokens = [];
+        let current = '';
+        for (let i = 0; i < input.length; i++) {
+            const char = input[i];
+            const nextChar = i < input.length - 1 ? input[i + 1] : '';
+            const prevChar = i > 0 ? input[i - 1] : '';
+            // Handle whitespace - always splits tokens
+            if (/\s/.test(char)) {
+                if (current) {
+                    tokens.push(current);
+                    current = '';
+                }
+                continue;
+            }
+            // Handle Chinese characters - each is its own token
+            if (this.HANZI_REGEX.test(char)) {
+                if (current && !this.isAllHanzi(current)) {
+                    tokens.push(current);
+                    current = '';
+                }
+                current += char;
+                // Check if next char is also hanzi, if not, push current
+                if (!this.HANZI_REGEX.test(nextChar)) {
+                    tokens.push(current);
+                    current = '';
+                }
+                continue;
+            }
+            // Handle numbers - they should be separate tokens
+            if (/\d/.test(char)) {
+                // If we have accumulated non-digit text, push it
+                if (current && !/^\d+$/.test(current)) {
+                    tokens.push(current);
+                    current = '';
+                }
+                current += char;
+                // If next char is not a digit, push the number
+                if (!/\d/.test(nextChar)) {
+                    tokens.push(current);
+                    current = '';
+                }
+                continue;
+            }
+            // Handle punctuation
+            if (this.ALL_PUNCTUATION.test(char)) {
+                // Check if it's embedded punctuation (like apostrophe in "don't" or "xi'an")
+                if (this.EMBEDDED_PUNCTUATION.test(char)) {
+                    // Only keep embedded if surrounded by letters (including accented pinyin vowels)
+                    // This regex matches ASCII letters plus common pinyin tone-marked vowels
+                    const letterPattern = /[a-zA-Z\u0101\u00E1\u01CE\u00E0\u0113\u00E9\u011B\u00E8\u012B\u00ED\u01D0\u00EC\u014D\u00F3\u01D2\u00F2\u016B\u00FA\u01D4\u00F9\u01D6\u01D8\u01DA\u01DC]/;
+                    const isEmbedded = prevChar && nextChar &&
+                        letterPattern.test(prevChar) &&
+                        letterPattern.test(nextChar);
+                    if (isEmbedded) {
+                        current += char;
+                    }
+                    else {
+                        // Treat as boundary
+                        if (current) {
+                            tokens.push(current);
+                            current = '';
+                        }
+                        tokens.push(char);
+                    }
+                }
+                else {
+                    // Boundary punctuation - always splits
+                    if (current) {
+                        tokens.push(current);
+                        current = '';
+                    }
+                    tokens.push(char);
+                }
+                continue;
+            }
+            // Regular character - add to current token
+            // But split if transitioning from digits
+            if (current && /^\d+$/.test(current)) {
+                tokens.push(current);
+                current = '';
+            }
+            current += char;
+        }
+        // Don't forget the last token
+        if (current) {
+            tokens.push(current);
+        }
+        return tokens.filter(t => t.length > 0);
+    }
+    /**
+     * Normalize apostrophes to curly for English database matching.
+     * The database stores English contractions with curly apostrophes (U+2019).
+     */
+    static normalizeApostrophesToCurly(text) {
+        // U+0027 = ' (straight apostrophe)
+        // U+2018 = ' (left single quotation mark)
+        // Replace both with U+2019 (right single quotation mark / curly apostrophe)
+        return text.replace(/[\u0027\u2018]/g, '\u2019');
+    }
+    /**
+     * Check if a token with embedded apostrophes could be pinyin with syllable separators.
+     * Returns the parsed syllables if valid, null otherwise.
+     */
+    static tryParsePinyinWithApostrophe(token) {
+        // Only consider tokens that contain apostrophes
+        if (!this.APOSTROPHES.test(token)) {
+            return null;
+        }
+        // Use PinyinParser which now handles both straight and curly apostrophes
+        const pinyinParsing = PinyinParser_1.PinyinParser.parse(token);
+        // Only consider it pinyin if the parsing produced multiple syllables
+        // (the apostrophe actually acted as a separator)
+        if (pinyinParsing !== null && pinyinParsing.length > 1) {
+            return pinyinParsing;
+        }
+        return null;
+    }
+    /**
+     * Classifies a single token
+     */
+    static classifyToken(token) {
+        const normalized = token.toLowerCase();
+        // Check for single-character punctuation
+        if (token.length === 1 && this.ALL_PUNCTUATION.test(token)) {
+            return {
+                text: token,
+                type: TokenType.PUNCTUATION,
+                normalized
+            };
+        }
+        // Check for numbers
+        if (/^\d+$/.test(token)) {
+            return {
+                text: token,
+                type: TokenType.NUMBER,
+                normalized
+            };
+        }
+        // Check for Chinese characters
+        if (this.HANZI_REGEX.test(token)) {
+            return {
+                text: token,
+                type: TokenType.HANZI,
+                normalized: token // Hanzi doesn't need lowercasing
+            };
+        }
+        // Check for tokens with embedded apostrophes - could be pinyin syllable separators
+        if (this.APOSTROPHES.test(token)) {
+            const pinyinWithApostrophe = this.tryParsePinyinWithApostrophe(token);
+            if (pinyinWithApostrophe !== null) {
+                // Token contains apostrophe that acts as pinyin syllable separator
+                // Check if it has tone marks to determine if it's definitely pinyin or ambiguous
+                if (this.PINYIN_TONE_MARKS.test(token)) {
+                    return {
+                        text: token,
+                        type: TokenType.PINYIN,
+                        normalized,
+                        isPossiblePinyin: true,
+                        pinyinVariants: pinyinWithApostrophe
+                    };
+                }
+                else {
+                    // Could be pinyin (xi'an) or could be English with apostrophe
+                    // Treat as ambiguous
+                    return {
+                        text: token,
+                        type: TokenType.AMBIGUOUS,
+                        normalized,
+                        isPossiblePinyin: true,
+                        pinyinVariants: pinyinWithApostrophe
+                    };
+                }
+            }
+            // Not valid pinyin with apostrophe - treat as English
+            // Normalize apostrophes to curly for database matching
+            return {
+                text: token,
+                type: TokenType.ENGLISH,
+                normalized: this.normalizeApostrophesToCurly(normalized)
+            };
+        }
+        // Check for pinyin with tone marks
+        if (this.PINYIN_TONE_MARKS.test(token)) {
+            const pinyinParsing = PinyinParser_1.PinyinParser.parse(token);
+            if (pinyinParsing !== null) {
+                return {
+                    text: token,
+                    type: TokenType.PINYIN,
+                    normalized,
+                    isPossiblePinyin: true,
+                    pinyinVariants: pinyinParsing
+                };
+            }
+        }
+        // Try to parse as pinyin (without tone marks)
+        const pinyinParsing = PinyinParser_1.PinyinParser.parse(token);
+        if (pinyinParsing !== null) {
+            // Ambiguous - could be pinyin or English
+            return {
+                text: token,
+                type: TokenType.AMBIGUOUS,
+                normalized,
+                isPossiblePinyin: true,
+                pinyinVariants: pinyinParsing
+            };
+        }
+        // Default to English
+        return {
+            text: token,
+            type: TokenType.ENGLISH,
+            normalized
+        };
+    }
+    /**
+     * Checks if a string contains only hanzi characters
+     */
+    static isAllHanzi(str) {
+        return str.split('').every(char => this.HANZI_REGEX.test(char));
+    }
+}
+exports.SearchTokenizer = SearchTokenizer;
+// Common English punctuation that should be treated as word boundaries
+SearchTokenizer.BOUNDARY_PUNCTUATION = /[.!?;:,]/;
+// Punctuation that can be embedded in words (apostrophes, hyphens)
+// U+0027 = ' (straight apostrophe)
+// U+2018 = ' (left single quotation mark)
+// U+2019 = ' (right single quotation mark / curly apostrophe)
+SearchTokenizer.EMBEDDED_PUNCTUATION = /[\u0027\u2018\u2019\-]/;
+// Apostrophe characters (both straight and curly)
+SearchTokenizer.APOSTROPHES = /[\u0027\u2018\u2019]/;
+// Chinese punctuation marks
+// U+2018/U+2019 = '' (curly single quotes)
+// U+201C/U+201D = "" (curly double quotes)
+SearchTokenizer.CHINESE_PUNCTUATION = /[。！？；：，、\u201C\u201D\u2018\u2019（）《》【】]/;
+// All punctuation for detection
+SearchTokenizer.ALL_PUNCTUATION = /[.!?;:,\u0027\u2018\u2019\-()。！？；：，、\u201C\u201D（）《》【】]/;
+// Chinese character ranges
+SearchTokenizer.HANZI_REGEX = /[\u4e00-\u9fff]/;
+// Pinyin tone marks
+SearchTokenizer.PINYIN_TONE_MARKS = /[āáǎàēéěèīíǐìōóǒòūúǔùǖǘǚǜ]/i;

package/dist/util/index.d.ts

@@ -4,4 +4,8 @@ export * from './Database';
 export * from './DifficultyRange';
 export * from './Encryption';
 export * from './Logging';
+export * from './PinyinParser';
+export * from './PinyinValidator';
+export * from './SearchPreprocessor';
+export * from './SearchTokenizer';
 export * from './SenseRankEncoder';

package/dist/util/index.js

@@ -21,4 +21,8 @@ __exportStar(require("./Database"), exports);
 __exportStar(require("./DifficultyRange"), exports);
 __exportStar(require("./Encryption"), exports);
 __exportStar(require("./Logging"), exports);
+__exportStar(require("./PinyinParser"), exports);
+__exportStar(require("./PinyinValidator"), exports);
+__exportStar(require("./SearchPreprocessor"), exports);
+__exportStar(require("./SearchTokenizer"), exports);
 __exportStar(require("./SenseRankEncoder"), exports);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@shaxpir/duiduidui-models",
-  "version": "1.9.26",
+  "version": "1.9.28",
   "repository": {
     "type": "git",
     "url": "https://github.com/shaxpir/duiduidui-models"