dsh-cot-anchor 0.4.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/index.js +22 -12
- package/package.json +1 -1
package/lib/index.js
CHANGED
|
@@ -1639,13 +1639,19 @@ function computeWordBigramFingerprint(words) {
|
|
|
1639
1639
|
function computeNGramFingerprint(normText, nGramSize) {
|
|
1640
1640
|
if (typeof normText !== "string" || normText.length < nGramSize * 4) return null;
|
|
1641
1641
|
|
|
1642
|
-
// 轨 1:字符级 n-gram
|
|
1642
|
+
// 轨 1:字符级 n-gram(原逻辑不变 + 英文词内部子串隔离)。
|
|
1643
|
+
// 隔离规则:gram 完全由英文字母组成时跳过(normalize 已小写化,/^[a-z]+$/ 即纯英文)。
|
|
1644
|
+
// 纯英文 gram 一定是某个英文词的内部子串或跨词片段,英文语义统一交给词轨判定;
|
|
1645
|
+
// 字符轨只保留含 CJK/数字/符号的 gram,避免中英混合文本里 "sessi"/"ancho" 这类
|
|
1646
|
+
// 英文词子串被独立计数而误报(本会话实测复现:count 16、coverage 8.8% 触发 cut)。
|
|
1643
1647
|
const charGramCount = new Map();
|
|
1644
1648
|
let charTotalGrams = 0;
|
|
1645
1649
|
for (let i = 0; i <= normText.length - nGramSize; i++) {
|
|
1646
1650
|
const gram = normText.slice(i, i + nGramSize);
|
|
1647
1651
|
// 跳过含空白的 gram(normalize 已把标点换成空格,避免跨标点短语虚高)
|
|
1648
1652
|
if (gram.includes(" ") || gram.includes("\n") || gram.includes("\t")) continue;
|
|
1653
|
+
// 跳过纯英文 gram:英文词内部子串隔离(方案 1)
|
|
1654
|
+
if (/^[a-z]+$/.test(gram)) continue;
|
|
1649
1655
|
let v = charGramCount.get(gram) ?? 0;
|
|
1650
1656
|
charGramCount.set(gram, ++v);
|
|
1651
1657
|
charTotalGrams++;
|
|
@@ -1656,8 +1662,7 @@ function computeNGramFingerprint(normText, nGramSize) {
|
|
|
1656
1662
|
const words = normText.match(/[a-z]+/g) || [];
|
|
1657
1663
|
const wordFp = words.length >= 2 ? computeWordBigramFingerprint(words) : null;
|
|
1658
1664
|
|
|
1659
|
-
//
|
|
1660
|
-
// 全部 token ≈ 英文词数 + CJK 单字数(其他符号已被 normalize 归并为空白)。
|
|
1665
|
+
// 英文 token 占比统计(供 detectNGramLoop 词轨覆盖率分母用)。
|
|
1661
1666
|
const englishTokenCount = words.length;
|
|
1662
1667
|
const cjkCount = (normText.match(/[\u4e00-\u9fff]/g) || []).length;
|
|
1663
1668
|
const totalTokenCount = englishTokenCount + cjkCount;
|
|
@@ -1666,7 +1671,8 @@ function computeNGramFingerprint(normText, nGramSize) {
|
|
|
1666
1671
|
wordFp,
|
|
1667
1672
|
englishTokenCount,
|
|
1668
1673
|
totalTokenCount,
|
|
1669
|
-
|
|
1674
|
+
// 方案 1:英文词轨并行——只要有英文词就具备走词轨的条件,不再要求英文主导。
|
|
1675
|
+
hasEnglishWords: englishTokenCount > 0
|
|
1670
1676
|
};
|
|
1671
1677
|
}
|
|
1672
1678
|
|
|
@@ -1728,17 +1734,21 @@ function detectNGramLoop(fullVisibleText) {
|
|
|
1728
1734
|
const fp = computeNGramFingerprint(norm, NGram_PHRASE_MIN);
|
|
1729
1735
|
if (!fp) return null;
|
|
1730
1736
|
|
|
1731
|
-
|
|
1732
|
-
|
|
1733
|
-
|
|
1737
|
+
// 轨 A(方案 1 并行):英文词级 2-gram——只要文本含英文词就跑词轨。
|
|
1738
|
+
// 英文单词是原子 token,词轨本身不会产生 "sessi"/"ancho" 这类词内子串误报;
|
|
1739
|
+
// 覆盖率分母用英文 token 总数,避免中文主体把分母撑大掩盖英文真循环。
|
|
1740
|
+
if (fp.hasEnglishWords && fp.wordFp) {
|
|
1734
1741
|
const chosen = pickHighestNonStopwordBigram(fp.wordFp.gramCount);
|
|
1735
|
-
if (
|
|
1736
|
-
|
|
1737
|
-
|
|
1738
|
-
|
|
1742
|
+
if (chosen && chosen.count >= NGram_MIN_COUNT) {
|
|
1743
|
+
const coverage = chosen.count * 2 / fp.englishTokenCount;
|
|
1744
|
+
if (coverage >= NGram_COVERAGE_RATIO) {
|
|
1745
|
+
return { maxGram: chosen.gram, count: chosen.count, coverage };
|
|
1746
|
+
}
|
|
1747
|
+
}
|
|
1739
1748
|
}
|
|
1740
1749
|
|
|
1741
|
-
//
|
|
1750
|
+
// 轨 B(方案 1 并行):字符级 n-gram——纯英文 gram 已在 computeNGramFingerprint
|
|
1751
|
+
// 中被隔离(英文词内部子串不再计入),字符轨只承担含 CJK/数字/符号的文本。
|
|
1742
1752
|
if (!fp.charFp) return null;
|
|
1743
1753
|
const chosen = pickHighestNonStopwordGram(fp.charFp.gramCount);
|
|
1744
1754
|
if (!chosen || chosen.count < NGram_MIN_COUNT) return null;
|