@aster-cloud/aster-lang-ts 0.0.1

package/dist/src/frontend/canonicalizer.js

@@ -0,0 +1,348 @@
+/**
+ * @module canonicalizer
+ *
+ * Canonicalizer（规范化器）：将 CNL 源代码规范化为标准格式。
+ *
+ * **功能**：
+ * - 规范化关键字（根据 Lexicon 配置）
+ * - 强制语句以句号或冒号结尾
+ * - 规范化空白符和缩进（2 空格为标准）
+ * - 保留标识符的大小写
+ * - 移除注释（`//` 和 `#`）
+ * - 去除冠词（根据 Lexicon 配置）
+ *
+ * **多语言支持**：
+ * - 通过 Lexicon 参数支持不同自然语言
+ * - 默认使用英语（en-US）词法表
+ *
+ * **注意**：
+ * - Aster 使用 2 空格缩进，缩进具有语法意义
+ * - 制表符会被自动转换为 2 个空格
+ */
+import { getMultiWordKeywords } from '../config/lexicons/types.js';
+import { LexiconRegistry, initializeDefaultLexicons } from '../config/lexicons/index.js';
+// 默认正则表达式（英语）
+const LINE_COMMENT_RE = /^\s*(?:\/\/|#)/;
+const SPACE_RUN_RE = /[ \t]+/g;
+const PUNCT_NORMAL_RE = /\s+([.,:])/g;
+const PUNCT_FINAL_RE = /\s+([.,:!;?])/g;
+const TRAILING_SPACE_RE = /\s+$/g;
+// 中文标点模式
+const ZH_PUNCT_NORMAL_RE = /\s+([。，：、])/g;
+const ZH_PUNCT_FINAL_RE = /\s+([。，：、！；？])/g;
+/**
+ * 获取标点符号正则表达式。
+ *
+ * @param lexicon - 可选的词法表
+ * @param isFinal - 是否为最终标点（包含更多标点符号）
+ * @returns 标点符号正则表达式
+ */
+function getPunctuationRegex(lexicon, isFinal) {
+    const effectiveLexicon = getEffectiveLexicon(lexicon);
+    // 根据 whitespaceMode 选择标点模式
+    if (effectiveLexicon.canonicalization.whitespaceMode === 'chinese') {
+        return isFinal ? ZH_PUNCT_FINAL_RE : ZH_PUNCT_NORMAL_RE;
+    }
+    return isFinal ? PUNCT_FINAL_RE : PUNCT_NORMAL_RE;
+}
+/**
+ * 获取冠词移除正则表达式。
+ *
+ * @param lexicon - 可选的词法表
+ * @returns 冠词正则表达式，如果语言不支持冠词则返回 null
+ */
+function getArticleRegex(lexicon) {
+    const effectiveLexicon = getEffectiveLexicon(lexicon);
+    if (!effectiveLexicon.canonicalization.removeArticles) {
+        return null;
+    }
+    const articles = effectiveLexicon.canonicalization.articles;
+    if (!articles || articles.length === 0) {
+        return null;
+    }
+    const pattern = `\\b(${articles.join('|')})\\b(?=\\s)`;
+    return new RegExp(pattern, 'gi');
+}
+/**
+ * 获取有效的词法表（提供的或注册表默认）。
+ *
+ * @param lexicon - 可选的词法表
+ * @returns 有效的词法表
+ */
+function getEffectiveLexicon(lexicon) {
+    if (lexicon) {
+        return lexicon;
+    }
+    // 确保注册表已初始化
+    initializeDefaultLexicons();
+    return LexiconRegistry.getDefault();
+}
+/**
+ * 获取多词关键字列表（按长度降序排列，用于贪婪匹配）。
+ *
+ * @param lexicon - 可选的词法表
+ * @returns 多词关键字数组
+ */
+function getMultiWordKeywordList(lexicon) {
+    const effectiveLexicon = getEffectiveLexicon(lexicon);
+    return getMultiWordKeywords(effectiveLexicon);
+}
+// 判断指定位置的引号是否被转义
+function isEscaped(str, index) {
+    let slashCount = 0;
+    for (let i = index - 1; i >= 0 && str[i] === '\\'; i--) {
+        slashCount++;
+    }
+    return slashCount % 2 === 1;
+}
+/**
+ * 规范化 CNL 源代码为标准格式。
+ *
+ * 这是 Aster 编译管道的第一步，将原始 CNL 文本转换为规范化的格式，
+ * 以便后续的词法分析和语法分析阶段处理。
+ *
+ * **转换步骤**：
+ * 1. 规范化换行符为 `\n`
+ * 2. 将制表符转换为 2 个空格
+ * 3. 移除行注释（`//` 和 `#`）
+ * 4. 规范化引号（智能引号 → 标准引号）
+ * 5. 强制语句以句号或冒号结尾
+ * 6. 去除冠词（根据语言配置）
+ * 7. 规范化多词关键字大小写
+ *
+ * @param input - 原始 CNL 源代码字符串
+ * @param lexicon - 可选的词法表，默认使用英语（en-US）配置
+ * @returns 规范化后的 CNL 源代码
+ *
+ * @example
+ * ```typescript
+ * import { canonicalize } from '@wontlost-ltd/aster-lang';
+ *
+ * // 英语（默认）
+ * const canonical = canonicalize(raw);
+ *
+ * // 中文
+ * import { ZH_CN } from './config/lexicons/zh-CN.js';
+ * const zhCanonical = canonicalize(raw, ZH_CN);
+ * ```
+ */
+export function canonicalize(input, lexicon) {
+    // 缓存有效的词法表，确保所有配置访问都使用同一来源
+    const effectiveLexicon = getEffectiveLexicon(lexicon);
+    const quotes = effectiveLexicon.punctuation.stringQuotes;
+    const articleRe = getArticleRegex(lexicon);
+    const multiWordKeywords = getMultiWordKeywordList(lexicon);
+    const punctNormalRe = getPunctuationRegex(lexicon, false);
+    const punctFinalRe = getPunctuationRegex(lexicon, true);
+    // Normalize newlines to \n
+    let s = input.replace(/\r\n?/g, '\n');
+    // Normalize tabs to two spaces (indentation is 2-space significant)
+    // Convert all tabs, including leading indentation, to ensure the lexer
+    // measures indentation consistently.
+    s = s.replace(/\t/g, '  ');
+    // Drop line comments (// and #) while 保留换行占位，formatter/LSP 另行处理注释内容
+    s = s
+        .split('\n')
+        .map(line => (LINE_COMMENT_RE.test(line) ? '' : line))
+        .join('\n');
+    // Normalize smart quotes to target quotes
+    // 英语：智能引号 → 直引号
+    // 中文：各种引号 → 直角引号（根据 lexicon 配置）
+    if (quotes.open === '"' && quotes.close === '"') {
+        // 英语：标准化为直引号
+        s = s.replace(/[\u201C\u201D]/g, '"').replace(/[\u2018\u2019]/g, "'");
+    }
+    else {
+        // 中文或其他语言：标准化为配置的引号
+        // 智能引号明确映射
+        s = s.replace(/\u201C/g, quotes.open); // 左双引号 " → 开引号
+        s = s.replace(/\u201D/g, quotes.close); // 右双引号 " → 闭引号
+        // 直引号需要成对处理：奇数位置为开，偶数位置为闭
+        s = normalizeAlternatingQuotes(s, '"', quotes.open, quotes.close);
+    }
+    // 应用 lexicon 的自定义规范化规则
+    if (effectiveLexicon.canonicalization.customRules) {
+        for (const rule of effectiveLexicon.canonicalization.customRules) {
+            const re = new RegExp(rule.pattern, 'g');
+            s = s.replace(re, rule.replacement);
+        }
+    }
+    // 全角转半角（如果配置启用）
+    if (effectiveLexicon.canonicalization.fullWidthToHalf) {
+        s = fullWidthToHalfWidth(s);
+    }
+    // Ensure lines end with either period or colon before newline if they look like statements
+    s = s
+        .split('\n')
+        .map(line => {
+        const trimmed = line.trim();
+        if (trimmed === '')
+            return line; // keep empty
+        // If ends with ':' or '.' already, keep
+        if (/[:.]$/.test(trimmed))
+            return line;
+        // 中文标点检查
+        if (effectiveLexicon.canonicalization.whitespaceMode === 'chinese') {
+            if (/[。：]$/.test(trimmed))
+                return line;
+        }
+        // Heuristic: if line appears to open a block (keywords like match/within/to ... produce ...:)
+        // We won't add punctuation here; parser will require proper punctuation and offer fix-it.
+        return line; // do nothing; errors will prompt fixes
+    })
+        .join('\n');
+    // Fold multiple spaces (but not newlines); keep indentation (2-space rule) for leading spaces only
+    s = s
+        .split('\n')
+        .map(line => normalizeLine(line, punctNormalRe, false, quotes))
+        .join('\n');
+    // Keep original casing to preserve TypeIdents. We only normalize multi-word keywords by hinting
+    // but we leave actual case handling to the parser (case-insensitive compare).
+    let marked = s;
+    for (const phrase of multiWordKeywords) {
+        const re = new RegExp(phrase.replace(/[-/\\^$*+?.()|[\]{}]/g, '\\$&'), 'ig');
+        marked = marked.replace(re, m => m.toLowerCase());
+    }
+    // Remove articles in allowed contexts (lightweight; parser will enforce correctness)
+    if (articleRe) {
+        marked = segmentString(marked, quotes)
+            .map(segment => (segment.inString ? segment.text : segment.text.replace(articleRe, '')))
+            .join('');
+    }
+    // Do not collapse newlines globally.
+    marked = marked.replace(/^\s+$/gm, '');
+    // Final whitespace normalization to ensure idempotency after article/macro passes
+    marked = marked
+        .split('\n')
+        .map(line => normalizeLine(line, punctFinalRe, true, quotes))
+        .join('\n');
+    return marked;
+}
+/**
+ * 全角字符转半角（数字和运算符）。
+ *
+ * @param str - 输入字符串
+ * @returns 转换后的字符串
+ */
+function fullWidthToHalfWidth(str) {
+    return str.replace(/[\uFF10-\uFF19\uFF21-\uFF3A\uFF41-\uFF5A]/g, ch => {
+        // 全角数字 0-9：0xFF10-0xFF19 → 0x30-0x39
+        // 全角大写字母 A-Z：0xFF21-0xFF3A → 0x41-0x5A
+        // 全角小写字母 a-z：0xFF41-0xFF5A → 0x61-0x7A
+        return String.fromCharCode(ch.charCodeAt(0) - 0xFEE0);
+    }).replace(/[\uFF0B\uFF0D\uFF0A\uFF0F\uFF1D\uFF1C\uFF1E\uFF08\uFF09\uFF3B\uFF3D]/g, ch => {
+        // 全角运算符和括号
+        const map = {
+            '\uFF0B': '+', // ＋
+            '\uFF0D': '-', // －
+            '\uFF0A': '*', // ＊
+            '\uFF0F': '/', // ／
+            '\uFF1D': '=', // ＝
+            '\uFF1C': '<', // ＜
+            '\uFF1E': '>', // ＞
+            '\uFF08': '(', // （全角左圆括号
+            '\uFF09': ')', // ）全角右圆括号
+            '\uFF3B': '[', // ［全角左方括号
+            '\uFF3D': ']', // ］全角右方括号
+        };
+        return map[ch] ?? ch;
+    });
+}
+/**
+ * 交替规范化直引号。
+ *
+ * 对于无法区分开闭的直引号，按出现顺序交替替换：
+ * 奇数位置（1st, 3rd, ...）→ 开引号
+ * 偶数位置（2nd, 4th, ...）→ 闭引号
+ *
+ * @param str - 输入字符串
+ * @param sourceQuote - 要替换的源引号字符
+ * @param openQuote - 目标开引号
+ * @param closeQuote - 目标闭引号
+ * @returns 替换后的字符串
+ */
+function normalizeAlternatingQuotes(str, sourceQuote, openQuote, closeQuote) {
+    let isOpen = true; // 下一个应该是开引号
+    let result = '';
+    for (let i = 0; i < str.length; i++) {
+        const ch = str[i];
+        if (ch === sourceQuote && !isEscaped(str, i)) {
+            result += isOpen ? openQuote : closeQuote;
+            isOpen = !isOpen;
+        }
+        else {
+            result += ch;
+        }
+    }
+    return result;
+}
+function segmentString(text, quotes) {
+    const segments = [];
+    let inString = false;
+    let current = '';
+    const openQuote = quotes.open;
+    const closeQuote = quotes.close;
+    for (let i = 0; i < text.length; i++) {
+        const ch = text[i];
+        current += ch;
+        if (inString) {
+            // 在字符串内，检查结束引号
+            if (ch === closeQuote && !isEscaped(text, i)) {
+                segments.push({ text: current, inString: true });
+                current = '';
+                inString = false;
+            }
+        }
+        else {
+            // 在字符串外，检查开始引号
+            if (ch === openQuote && !isEscaped(text, i)) {
+                const before = current.slice(0, -1);
+                if (before) {
+                    segments.push({ text: before, inString: false });
+                }
+                current = openQuote;
+                inString = true;
+            }
+        }
+    }
+    if (current) {
+        segments.push({ text: current, inString });
+    }
+    return segments;
+}
+function normalizeLine(line, punctuationPattern, trimTrailing, quotes) {
+    if (line === '') {
+        return line;
+    }
+    const match = line.match(/^(\s*)(.*)$/);
+    if (!match) {
+        return line;
+    }
+    const indent = match[1] ?? '';
+    const rest = match[2] ?? '';
+    if (rest === '') {
+        return indent;
+    }
+    const normalizedRest = normalizeRest(rest, punctuationPattern, trimTrailing, quotes);
+    return indent + normalizedRest;
+}
+function normalizeRest(rest, punctuationPattern, trimTrailing, quotes) {
+    const segments = segmentString(rest, quotes);
+    if (segments.length === 0) {
+        return rest;
+    }
+    return segments
+        .map((segment, index) => {
+        if (segment.inString) {
+            return segment.text;
+        }
+        let normalized = segment.text.replace(SPACE_RUN_RE, ' ');
+        normalized = normalized.replace(punctuationPattern, '$1');
+        if (trimTrailing && index === segments.length - 1) {
+            normalized = normalized.replace(TRAILING_SPACE_RE, '');
+        }
+        return normalized;
+    })
+        .join('');
+}
+//# sourceMappingURL=canonicalizer.js.map

package/dist/src/frontend/canonicalizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"canonicalizer.js","sourceRoot":"","sources":["../../../src/frontend/canonicalizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAGH,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AACnE,OAAO,EAAE,eAAe,EAAE,yBAAyB,EAAE,MAAM,6BAA6B,CAAC;AAEzF,cAAc;AACd,MAAM,eAAe,GAAG,gBAAgB,CAAC;AACzC,MAAM,YAAY,GAAG,SAAS,CAAC;AAC/B,MAAM,eAAe,GAAG,aAAa,CAAC;AACtC,MAAM,cAAc,GAAG,gBAAgB,CAAC;AACxC,MAAM,iBAAiB,GAAG,OAAO,CAAC;AAElC,SAAS;AACT,MAAM,kBAAkB,GAAG,cAAc,CAAC;AAC1C,MAAM,iBAAiB,GAAG,iBAAiB,CAAC;AAE5C;;;;;;GAMG;AACH,SAAS,mBAAmB,CAAC,OAAiB,EAAE,OAAiB;IAC/D,MAAM,gBAAgB,GAAG,mBAAmB,CAAC,OAAO,CAAC,CAAC;IAEtD,2BAA2B;IAC3B,IAAI,gBAAgB,CAAC,gBAAgB,CAAC,cAAc,KAAK,SAAS,EAAE,CAAC;QACnE,OAAO,OAAO,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,kBAAkB,CAAC;IAC1D,CAAC;IAED,OAAO,OAAO,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,eAAe,CAAC;AACpD,CAAC;AAED;;;;;GAKG;AACH,SAAS,eAAe,CAAC,OAAiB;IACxC,MAAM,gBAAgB,GAAG,mBAAmB,CAAC,OAAO,CAAC,CAAC;IAEtD,IAAI,CAAC,gBAAgB,CAAC,gBAAgB,CAAC,cAAc,EAAE,CAAC;QACtD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,QAAQ,GAAG,gBAAgB,CAAC,gBAAgB,CAAC,QAAQ,CAAC;IAC5D,IAAI,CAAC,QAAQ,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,OAAO,GAAG,OAAO,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,aAAa,CAAC;IACvD,OAAO,IAAI,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;AACnC,CAAC;AAED;;;;;GAKG;AACH,SAAS,mBAAmB,CAAC,OAAiB;IAC5C,IAAI,OAAO,EAAE,CAAC;QACZ,OAAO,OAAO,CAAC;IACjB,CAAC;IACD,YAAY;IACZ,yBAAyB,EAAE,CAAC;IAC5B,OAAO,eAAe,CAAC,UAAU,EAAE,CAAC;AACtC,CAAC;AAED;;;;;GAKG;AACH,SAAS,uBAAuB,CAAC,OAAiB;IAChD,MAAM,gBAAgB,GAAG,mBAAmB,CAAC,OAAO,CAAC,CAAC;IACtD,OAAO,oBAAoB,CAAC,gBAAgB,CAAC,CAAC;AAChD,CAAC;AAED,iBAAiB;AACjB,SAAS,SAAS,CAAC,GAAW,EAAE,KAAa;IAC3C,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,KAAK,IAAI,CAAC,GAAG,KAAK,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;QACvD,UAAU,EAAE,CAAC;IACf,CAAC;IACD,OAAO,UAAU,GAAG,CAAC,KAAK,CAAC,CAAC;AAC9B,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,MAAM,UAAU,YAAY,CAAC,KAAa,EAAE,OAAiB;IAC3D,2BAA2B;IAC3B,MAAM,gBAAgB,GAAG,mBAAmB,CAAC,OAAO,CAAC,CAAC;IACtD,MAAM,MAAM,GAAG,gBAAgB,CAAC,WAAW,CAAC,YAAY,CAAC;IACzD,MAAM,SAAS,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;IAC3C,MAAM,iBAAiB,GAAG,uBAAuB,CAAC,OAAO,CAAC,CAAC;IAC3D,MAAM,aAAa,GAAG,mBAAmB,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;IAC1D,MAAM,YAAY,GAAG,mBAAmB,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;IAExD,2BAA2B;IAC3B,IAAI,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;IAEtC,oEAAoE;IACpE,uEAAuE;IACvE,qCAAqC;IACrC,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IAE3B,oEAAoE;IACpE,CAAC,GAAG,CAAC;SACF,KAAK,CAAC,IAAI,CAAC;SACX,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;SACrD,IAAI,CAAC,IAAI,CAAC,CAAC;IAEd,0CAA0C;IAC1C,gBAAgB;IAChB,gCAAgC;IAChC,IAAI,MAAM,CAAC,IAAI,KAAK,GAAG,IAAI,MAAM,CAAC,KAAK,KAAK,GAAG,EAAE,CAAC;QAChD,aAAa;QACb,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,iBAAiB,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,iBAAiB,EAAE,GAAG,CAAC,CAAC;IACxE,CAAC;SAAM,CAAC;QACN,oBAAoB;QACpB,WAAW;QACX,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAG,eAAe;QACxD,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAE,eAAe;QACxD,0BAA0B;QAC1B,CAAC,GAAG,0BAA0B,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;IACpE,CAAC;IAED,uBAAuB;IACvB,IAAI,gBAAgB,CAAC,gBAAgB,CAAC,WAAW,EAAE,CAAC;QAClD,KAAK,MAAM,IAAI,IAAI,gBAAgB,CAAC,gBAAgB,CAAC,WAAW,EAAE,CAAC;YACjE,MAAM,EAAE,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;YACzC,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,EAAE,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QACtC,CAAC;IACH,CAAC;IAED,gBAAgB;IAChB,IAAI,gBAAgB,CAAC,gBAAgB,CAAC,eAAe,EAAE,CAAC;QACtD,CAAC,GAAG,oBAAoB,CAAC,CAAC,CAAC,CAAC;IAC9B,CAAC;IAED,2FAA2F;IAC3F,CAAC,GAAG,CAAC;SACF,KAAK,CAAC,IAAI,CAAC;SACX,GAAG,CAAC,IAAI,CAAC,EAAE;QACV,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAC5B,IAAI,OAAO,KAAK,EAAE;YAAE,OAAO,IAAI,CAAC,CAAC,aAAa;QAC9C,wCAAwC;QACxC,IAAI,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC;YAAE,OAAO,IAAI,CAAC;QACvC,SAAS;QACT,IAAI,gBAAgB,CAAC,gBAAgB,CAAC,cAAc,KAAK,SAAS,EAAE,CAAC;YACnE,IAAI,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC;gBAAE,OAAO,IAAI,CAAC;QACzC,CAAC;QACD,8FAA8F;QAC9F,0FAA0F;QAC1F,OAAO,IAAI,CAAC,CAAC,uCAAuC;IACtD,CAAC,CAAC;SACD,IAAI,CAAC,IAAI,CAAC,CAAC;IAEd,mGAAmG;IACnG,CAAC,GAAG,CAAC;SACF,KAAK,CAAC,IAAI,CAAC;SACX,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,aAAa,CAAC,IAAI,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;SAC9D,IAAI,CAAC,IAAI,CAAC,CAAC;IAEd,gGAAgG;IAChG,8EAA8E;IAC9E,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,MAAM,MAAM,IAAI,iBAAiB,EAAE,CAAC;QACvC,MAAM,EAAE,GAAG,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,uBAAuB,EAAE,MAAM,CAAC,EAAE,IAAI,CAAC,CAAC;QAC7E,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;IACpD,CAAC;IAED,qFAAqF;IACrF,IAAI,SAAS,EAAE,CAAC;QACd,MAAM,GAAG,aAAa,CAAC,MAAM,EAAE,MAAM,CAAC;aACnC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,CAAC;aACvF,IAAI,CAAC,EAAE,CAAC,CAAC;IACd,CAAC;IACD,qCAAqC;IACrC,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAEvC,kFAAkF;IAClF,MAAM,GAAG,MAAM;SACZ,KAAK,CAAC,IAAI,CAAC;SACX,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,aAAa,CAAC,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;SAC5D,IAAI,CAAC,IAAI,CAAC,CAAC;IAEd,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;GAKG;AACH,SAAS,oBAAoB,CAAC,GAAW;IACvC,OAAO,GAAG,CAAC,OAAO,CAAC,4CAA4C,EAAE,EAAE,CAAC,EAAE;QACpE,qCAAqC;QACrC,uCAAuC;QACvC,uCAAuC;QACvC,OAAO,MAAM,CAAC,YAAY,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC;IACxD,CAAC,CAAC,CAAC,OAAO,CAAC,uEAAuE,EAAE,EAAE,CAAC,EAAE;QACvF,WAAW;QACX,MAAM,GAAG,GAA2B;YAClC,QAAQ,EAAE,GAAG,EAAE,IAAI;YACnB,QAAQ,EAAE,GAAG,EAAE,IAAI;YACnB,QAAQ,EAAE,GAAG,EAAE,IAAI;YACnB,QAAQ,EAAE,GAAG,EAAE,IAAI;YACnB,QAAQ,EAAE,GAAG,EAAE,IAAI;YACnB,QAAQ,EAAE,GAAG,EAAE,IAAI;YACnB,QAAQ,EAAE,GAAG,EAAE,IAAI;YACnB,QAAQ,EAAE,GAAG,EAAE,UAAU;YACzB,QAAQ,EAAE,GAAG,EAAE,UAAU;YACzB,QAAQ,EAAE,GAAG,EAAE,UAAU;YACzB,QAAQ,EAAE,GAAG,EAAE,UAAU;SAC1B,CAAC;QACF,OAAO,GAAG,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC;IACvB,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,SAAS,0BAA0B,CACjC,GAAW,EACX,WAAmB,EACnB,SAAiB,EACjB,UAAkB;IAElB,IAAI,MAAM,GAAG,IAAI,CAAC,CAAC,YAAY;IAC/B,IAAI,MAAM,GAAG,EAAE,CAAC;IAEhB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACpC,MAAM,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;QAClB,IAAI,EAAE,KAAK,WAAW,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC;YAC7C,MAAM,IAAI,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC;YAC1C,MAAM,GAAG,CAAC,MAAM,CAAC;QACnB,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,EAAE,CAAC;QACf,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAID,SAAS,aAAa,CAAC,IAAY,EAAE,MAAuC;IAC1E,MAAM,QAAQ,GAAc,EAAE,CAAC;IAC/B,IAAI,QAAQ,GAAG,KAAK,CAAC;IACrB,IAAI,OAAO,GAAG,EAAE,CAAC;IAEjB,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC;IAC9B,MAAM,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC;IAEhC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,MAAM,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QACnB,OAAO,IAAI,EAAE,CAAC;QAEd,IAAI,QAAQ,EAAE,CAAC;YACb,eAAe;YACf,IAAI,EAAE,KAAK,UAAU,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;gBAC7C,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;gBACjD,OAAO,GAAG,EAAE,CAAC;gBACb,QAAQ,GAAG,KAAK,CAAC;YACnB,CAAC;QACH,CAAC;aAAM,CAAC;YACN,eAAe;YACf,IAAI,EAAE,KAAK,SAAS,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;gBAC5C,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;gBACpC,IAAI,MAAM,EAAE,CAAC;oBACX,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;gBACnD,CAAC;gBACD,OAAO,GAAG,SAAS,CAAC;gBACpB,QAAQ,GAAG,IAAI,CAAC;YAClB,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,OAAO,EAAE,CAAC;QACZ,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;IAC7C,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAS,aAAa,CACpB,IAAY,EACZ,kBAA0B,EAC1B,YAAqB,EACrB,MAAuC;IAEvC,IAAI,IAAI,KAAK,EAAE,EAAE,CAAC;QAChB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;IACxC,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC9B,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC5B,IAAI,IAAI,KAAK,EAAE,EAAE,CAAC;QAChB,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,MAAM,cAAc,GAAG,aAAa,CAAC,IAAI,EAAE,kBAAkB,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;IACrF,OAAO,MAAM,GAAG,cAAc,CAAC;AACjC,CAAC;AAED,SAAS,aAAa,CACpB,IAAY,EACZ,kBAA0B,EAC1B,YAAqB,EACrB,MAAuC;IAEvC,MAAM,QAAQ,GAAG,aAAa,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAC7C,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO,QAAQ;SACZ,GAAG,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE;QACtB,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;YACrB,OAAO,OAAO,CAAC,IAAI,CAAC;QACtB,CAAC;QAED,IAAI,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,GAAG,CAAC,CAAC;QACzD,UAAU,GAAG,UAAU,CAAC,OAAO,CAAC,kBAAkB,EAAE,IAAI,CAAC,CAAC;QAE1D,IAAI,YAAY,IAAI,KAAK,KAAK,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClD,UAAU,GAAG,UAAU,CAAC,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAC;QACzD,CAAC;QAED,OAAO,UAAU,CAAC;IACpB,CAAC,CAAC;SACD,IAAI,CAAC,EAAE,CAAC,CAAC;AACd,CAAC"}

package/dist/src/frontend/index.d.ts

@@ -0,0 +1,17 @@
+/**
+ * @module frontend
+ *
+ * 编译器前端模块：词法分析和源码规范化。
+ *
+ * 包含：
+ * - 规范化器 (canonicalizer)
+ * - 词法分析器 (lexer)
+ * - Token 定义 (tokens)
+ * - 关键词翻译 (keyword-translator)
+ */
+export { canonicalize } from './canonicalizer.js';
+export { lex } from './lexer.js';
+export { TokenKind, KW } from './tokens.js';
+export { createKeywordTranslator, buildKeywordTranslationIndex, buildFullTranslationIndex, translateTokens, translateTokensWithMarkers, translateToken, needsKeywordTranslation, } from './keyword-translator.js';
+export type { KeywordTranslationIndex, MarkerKeywordIndex, TranslationIndexResult, } from './keyword-translator.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/src/frontend/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/frontend/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AACjC,OAAO,EAAE,SAAS,EAAE,EAAE,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EACL,uBAAuB,EACvB,4BAA4B,EAC5B,yBAAyB,EACzB,eAAe,EACf,0BAA0B,EAC1B,cAAc,EACd,uBAAuB,GACxB,MAAM,yBAAyB,CAAC;AACjC,YAAY,EACV,uBAAuB,EACvB,kBAAkB,EAClB,sBAAsB,GACvB,MAAM,yBAAyB,CAAC"}

package/dist/src/frontend/index.js

@@ -0,0 +1,16 @@
+/**
+ * @module frontend
+ *
+ * 编译器前端模块：词法分析和源码规范化。
+ *
+ * 包含：
+ * - 规范化器 (canonicalizer)
+ * - 词法分析器 (lexer)
+ * - Token 定义 (tokens)
+ * - 关键词翻译 (keyword-translator)
+ */
+export { canonicalize } from './canonicalizer.js';
+export { lex } from './lexer.js';
+export { TokenKind, KW } from './tokens.js';
+export { createKeywordTranslator, buildKeywordTranslationIndex, buildFullTranslationIndex, translateTokens, translateTokensWithMarkers, translateToken, needsKeywordTranslation, } from './keyword-translator.js';
+//# sourceMappingURL=index.js.map

package/dist/src/frontend/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/frontend/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AACjC,OAAO,EAAE,SAAS,EAAE,EAAE,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EACL,uBAAuB,EACvB,4BAA4B,EAC5B,yBAAyB,EACzB,eAAe,EACf,0BAA0B,EAC1B,cAAc,EACd,uBAAuB,GACxB,MAAM,yBAAyB,CAAC"}

package/dist/src/frontend/keyword-translator.d.ts

@@ -0,0 +1,134 @@
+/**
+ * @module keyword-translator
+ *
+ * 关键词翻译器：将本地化关键词转换为规范化（英文）关键词。
+ *
+ * **设计原则**：
+ * - 透明层：在 lexer 和 parser 之间插入，保持两者接口不变
+ * - 双向映射：支持本地化→规范化（编译）和规范化→本地化（显示）
+ * - 幂等性：对英文关键词调用不改变任何内容
+ *
+ * **工作流程**：
+ * 1. Lexer 产生本地化 token（如 `{ kind: IDENT, value: '若' }`）
+ * 2. KeywordTranslator 将其转换为规范化 token（如 `{ kind: IDENT, value: 'if' }`）
+ * 3. Parser 使用规范化的 token 进行解析
+ */
+import type { Token } from '../types.js';
+import type { Lexicon } from '../config/lexicons/types.js';
+/**
+ * 关键词映射索引类型。
+ *
+ * 从本地化关键词（小写）映射到规范化关键词。
+ */
+export type KeywordTranslationIndex = Map<string, string>;
+/**
+ * 标记关键词索引类型。
+ *
+ * 存储被方括号包裹的关键词（如【定义】）的内部值到翻译结果的映射。
+ * 例如：'定义' -> 'Define'
+ */
+export type MarkerKeywordIndex = Map<string, string>;
+/**
+ * 构建关键词翻译索引。
+ *
+ * 将源词法表的关键词映射到目标词法表（默认英文）的关键词。
+ *
+ * @param sourceLexicon - 源词法表（如 zh-CN）
+ * @param targetLexicon - 目标词法表（默认 en-US）
+ * @returns 翻译索引：源关键词 -> 目标关键词
+ */
+/**
+ * 翻译索引结果。
+ */
+export interface TranslationIndexResult {
+    /** 普通关键词索引：源关键词 -> 目标关键词 */
+    index: KeywordTranslationIndex;
+    /** 标记关键词索引：方括号内部值 -> 目标关键词 */
+    markerIndex: MarkerKeywordIndex;
+}
+/**
+ * 构建关键词翻译索引。
+ *
+ * 将源词法表的关键词映射到目标词法表（默认英文）的关键词。
+ * 同时识别被标记符号包裹的关键词（如【定义】）并建立单独索引。
+ *
+ * @param sourceLexicon - 源词法表（如 zh-CN）
+ * @param targetLexicon - 目标词法表（默认 en-US）
+ * @returns 翻译索引：源关键词 -> 目标关键词
+ */
+export declare function buildKeywordTranslationIndex(sourceLexicon: Lexicon, targetLexicon?: Lexicon): KeywordTranslationIndex;
+/**
+ * 构建完整的翻译索引（包括普通关键词和标记关键词）。
+ *
+ * @param sourceLexicon - 源词法表（如 zh-CN）
+ * @param targetLexicon - 目标词法表（默认 en-US）
+ * @returns 包含普通索引和标记索引的结果
+ */
+export declare function buildFullTranslationIndex(sourceLexicon: Lexicon, targetLexicon?: Lexicon): TranslationIndexResult;
+/**
+ * 翻译单个 token 的值。
+ *
+ * @param token - 要翻译的 token
+ * @param index - 翻译索引
+ * @returns 翻译后的 token（新对象，不修改原 token）
+ */
+export declare function translateToken(token: Token, index: KeywordTranslationIndex): Token;
+/**
+ * 翻译 token 流中的所有关键词。
+ *
+ * @param tokens - 原始 token 数组
+ * @param index - 翻译索引
+ * @returns 翻译后的 token 数组（新数组，不修改原数组）
+ */
+export declare function translateTokens(tokens: readonly Token[], index: KeywordTranslationIndex): Token[];
+/**
+ * 翻译 token 流中的所有关键词（包括标记关键词序列）。
+ *
+ * 此函数会识别 LBRACKET + IDENT + RBRACKET 序列，
+ * 如果内部 IDENT 是标记关键词，则将三个 token 合并为单个翻译后的 token。
+ *
+ * @param tokens - 原始 token 数组
+ * @param index - 普通关键词翻译索引
+ * @param markerIndex - 标记关键词翻译索引
+ * @returns 翻译后的 token 数组（新数组，不修改原数组）
+ */
+export declare function translateTokensWithMarkers(tokens: readonly Token[], index: KeywordTranslationIndex, markerIndex: MarkerKeywordIndex): Token[];
+/**
+ * 创建完整的关键词翻译器。
+ *
+ * 这是一个高级 API，封装了索引构建和 token 翻译。
+ * 支持普通关键词和标记关键词（如【定义】）的翻译。
+ *
+ * @param sourceLexicon - 源词法表
+ * @param targetLexicon - 目标词法表（默认 en-US）
+ * @returns 翻译器对象
+ */
+export declare function createKeywordTranslator(sourceLexicon: Lexicon, targetLexicon?: Lexicon): {
+    /** 普通关键词翻译索引 */
+    index: KeywordTranslationIndex;
+    /** 标记关键词翻译索引（【】包裹的关键词） */
+    markerIndex: MarkerKeywordIndex;
+    /** 翻译单个 token */
+    translateToken: (token: Token) => Token;
+    /** 翻译 token 数组（包括标记关键词序列处理） */
+    translateTokens: (tokens: readonly Token[]) => Token[];
+    /** 检查普通关键词是否有翻译 */
+    hasTranslation: (value: string) => boolean;
+    /** 获取普通关键词翻译结果 */
+    getTranslation: (value: string) => string | undefined;
+    /** 检查标记关键词是否有翻译 */
+    hasMarkerTranslation: (value: string) => boolean;
+    /** 获取标记关键词翻译结果 */
+    getMarkerTranslation: (value: string) => string | undefined;
+};
+/**
+ * 判断词法表是否需要关键词翻译。
+ *
+ * 如果源词法表与目标词法表相同（或都是英文），则不需要翻译。
+ *
+ * @param sourceLexicon - 源词法表
+ * @param targetLexicon - 目标词法表（默认 en-US）
+ * @returns 如果需要翻译返回 true
+ */
+export declare function needsKeywordTranslation(sourceLexicon: Lexicon, targetLexicon?: Lexicon): boolean;
+//# sourceMappingURL=keyword-translator.d.ts.map

package/dist/src/frontend/keyword-translator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"keyword-translator.d.ts","sourceRoot":"","sources":["../../../src/frontend/keyword-translator.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AAEzC,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,6BAA6B,CAAC;AAI3D;;;;GAIG;AACH,MAAM,MAAM,uBAAuB,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAE1D;;;;;GAKG;AACH,MAAM,MAAM,kBAAkB,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAErD;;;;;;;;GAQG;AACH;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACrC,4BAA4B;IAC5B,KAAK,EAAE,uBAAuB,CAAC;IAC/B,8BAA8B;IAC9B,WAAW,EAAE,kBAAkB,CAAC;CACjC;AAED;;;;;;;;;GASG;AACH,wBAAgB,4BAA4B,CAC1C,aAAa,EAAE,OAAO,EACtB,aAAa,GAAE,OAAe,GAC7B,uBAAuB,CAGzB;AAED;;;;;;GAMG;AACH,wBAAgB,yBAAyB,CACvC,aAAa,EAAE,OAAO,EACtB,aAAa,GAAE,OAAe,GAC7B,sBAAsB,CA4BxB;AAED;;;;;;GAMG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,uBAAuB,GAAG,KAAK,CAsBlF;AAED;;;;;;GAMG;AACH,wBAAgB,eAAe,CAC7B,MAAM,EAAE,SAAS,KAAK,EAAE,EACxB,KAAK,EAAE,uBAAuB,GAC7B,KAAK,EAAE,CAET;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,0BAA0B,CACxC,MAAM,EAAE,SAAS,KAAK,EAAE,EACxB,KAAK,EAAE,uBAAuB,EAC9B,WAAW,EAAE,kBAAkB,GAC9B,KAAK,EAAE,CAqCT;AAED;;;;;;;;;GASG;AACH,wBAAgB,uBAAuB,CACrC,aAAa,EAAE,OAAO,EACtB,aAAa,GAAE,OAAe;IAK5B,gBAAgB;;IAGhB,0BAA0B;;IAG1B,iBAAiB;4BACO,KAAK,KAAG,KAAK;IAErC,+BAA+B;8BACL,SAAS,KAAK,EAAE,KAAG,KAAK,EAAE;IAGpD,mBAAmB;4BACK,MAAM,KAAG,OAAO;IAExC,kBAAkB;4BACM,MAAM,KAAG,MAAM,GAAG,SAAS;IAEnD,mBAAmB;kCACW,MAAM,KAAG,OAAO;IAE9C,kBAAkB;kCACY,MAAM,KAAG,MAAM,GAAG,SAAS;EAE5D;AAED;;;;;;;;GAQG;AACH,wBAAgB,uBAAuB,CACrC,aAAa,EAAE,OAAO,EACtB,aAAa,GAAE,OAAe,GAC7B,OAAO,CAET"}