hanzi-chai 0.3.0

package/dist/utils.d.ts

@@ -0,0 +1,107 @@
+import type { Feature } from "./classifier.js";
+import type { DerivedComponent, Compound, Repertoire, Draw, Operator, Point, SVGStroke, BasicComponent, ReferenceStroke, PrimitiveRepertoire, Character, SplicedComponent, Glyph, Identity } from "./data.js";
+import { type Value, type Mapping, GeneralizedKey, CharacterSetSpecifier } from "./config.js";
+import type { IndexedElement } from "./assembly.js";
+interface Loss {
+    ideal: number;
+    lt_penalty: number;
+    gt_penalty: number;
+}
+export type Dictionary = [string, string][];
+export type Frequency = Record<string, number>;
+export type AdaptedFrequency = Map<string, number>;
+export type Distribution = Record<string, Loss>;
+export type Equivalence = Record<string, number>;
+export type CustomElementMap = Record<string, string[]>;
+export declare const printableAscii: string[];
+export declare const formatDate: (date: Date) => string;
+export interface UnicodeBlock {
+    name: string;
+    label: string;
+    begin: number;
+    end: number;
+    count: number;
+}
+export declare const unicodeBlocks: UnicodeBlock[];
+export declare const unicodeBlock: (code: number) => string;
+export declare const unicodeBlockRank: (code: number) => number;
+export declare const isValidCJKBasicChar: (char: string) => boolean;
+export declare const isValidCJKChar: (char: string) => boolean;
+export declare const isValidCJKSupplement: (char: string) => boolean;
+export declare const isPUA: (char: string) => boolean;
+export declare const chars: (s: string) => number;
+export declare const getDummyBasicComponent: () => BasicComponent;
+export declare const getDummyDerivedComponent: () => DerivedComponent;
+export declare const getDummySplicedComponent: () => SplicedComponent;
+export declare const getDummyReferenceStroke: () => ReferenceStroke;
+export declare const getDummySVGStroke: (feature: Feature, start?: Point, oldCurveList?: Draw[]) => SVGStroke;
+export declare const getDummyCompound: (operator: Operator) => Compound;
+export declare const isComponent: (glyph: Glyph) => glyph is BasicComponent | DerivedComponent | SplicedComponent | Identity;
+export declare const getSupplemental: (repertoire: Repertoire, list: string[]) => string[];
+export declare const listToObject: <T extends {
+    unicode: number;
+}>(list: T[]) => {
+    [k: string]: T;
+};
+export declare function getRecordFromTSV(text: string): Record<string, number>;
+export declare function getDistributionFromTSV(text: string): Distribution;
+export declare function getDictFromTSV(text: string): [string, string][];
+export declare const exportYAML: (config: object, filename: string, flowLevel?: number, sanitize?: boolean) => void;
+export declare const exportJSON: (data: object, filename: string) => void;
+export declare const exportTSV: (data: string[][], filename: string) => void;
+export declare const stringify: (element: IndexedElement | undefined) => string;
+export declare const parseKey: (key: string) => IndexedElement | undefined;
+export declare const joinKeys: (keys: GeneralizedKey[]) => string | GeneralizedKey[];
+export declare const makeCharacterFilter: (input: CharacterFilter, repertoire: Repertoire | PrimitiveRepertoire, sequenceMap: Map<string, string>) => (name: string) => boolean | undefined;
+export interface CharacterFilter {
+    name?: string;
+    sequence?: string;
+    unicode?: number;
+    tag?: string;
+    part?: string;
+    operator?: Operator;
+}
+export declare const makeFilter: (input: string, form: Repertoire, sequence: Map<string, string>) => (char: string) => boolean;
+export interface AnalyzerForm {
+    type: "single" | "multi" | "all";
+    position: number[];
+    top: number;
+}
+export declare const makeDefaultAnalyzer: (maxLength: number) => AnalyzerForm;
+interface LevelMetric {
+    length: number;
+    frequency: number;
+}
+interface TierMetric {
+    top?: number;
+    duplication?: number;
+    levels?: LevelMetric[];
+    fingering?: (number | undefined)[];
+}
+export interface PartialMetric {
+    tiers?: TierMetric[];
+    duplication?: number;
+    key_distribution?: number;
+    pair_equivalence?: number;
+    extended_pair_equivalence?: number;
+    fingering?: (number | undefined)[];
+    levels?: LevelMetric[];
+}
+export interface Metric {
+    characters_full?: PartialMetric;
+    characters_short?: PartialMetric;
+    words_full?: PartialMetric;
+    words_short?: PartialMetric;
+}
+export declare const adapt: (原始词频映射: Frequency, 词典: Set<string>) => Map<string, number>;
+export type Tuple<T, N extends number> = N extends N ? number extends N ? T[] : _TupleOf<T, N, []> : never;
+type _TupleOf<T, N extends number, R extends unknown[]> = R["length"] extends N ? R : _TupleOf<T, N, [T, ...R]>;
+export interface MappedInfo {
+    name: string;
+    code: Exclude<Value, null | {
+        element: string;
+    }>;
+}
+export declare const getReversedMapping: (mapping: Mapping, alphabet: string) => Map<string, MappedInfo[]>;
+export declare const characterSetFilters: Record<CharacterSetSpecifier, (k: string, v: Character, s?: Set<string>) => boolean>;
+export {};

package/dist/utils.js

@@ -0,0 +1,546 @@
+import { schema } from "./classifier.js";
+import { range } from "lodash-es";
+import { dump } from "js-yaml";
+export const printableAscii = range(33, 127).map((x) => String.fromCodePoint(x));
+export const formatDate = (date) => {
+    return `${date.getMonth() + 1}-${date.getDate()} ${date.getHours()}:${date.getMinutes()}:${date.getSeconds()}`;
+};
+export const unicodeBlocks = [
+    // 统一汉字基本集与扩展
+    {
+        name: "cjk",
+        label: "中日韩统一表意文字",
+        begin: 0x4e00,
+        end: 0x9fff,
+        count: 20992,
+    },
+    {
+        name: "cjk-a",
+        label: "中日韩统一表意文字扩展A",
+        begin: 0x3400,
+        end: 0x4dbf,
+        count: 6592,
+    },
+    {
+        name: "cjk-b",
+        label: "中日韩统一表意文字扩展B",
+        begin: 0x20000,
+        end: 0x2a6df,
+        count: 42720,
+    },
+    {
+        name: "cjk-c",
+        label: "中日韩统一表意文字扩展C",
+        begin: 0x2a700,
+        end: 0x2b73f,
+        count: 4160,
+    },
+    {
+        name: "cjk-d",
+        label: "中日韩统一表意文字扩展D",
+        begin: 0x2b740,
+        end: 0x2b81f,
+        count: 222,
+    },
+    {
+        name: "cjk-e",
+        label: "中日韩统一表意文字扩展E",
+        begin: 0x2b820,
+        end: 0x2ceaf,
+        count: 5774,
+    },
+    {
+        name: "cjk-f",
+        label: "中日韩统一表意文字扩展F",
+        begin: 0x2ceb0,
+        end: 0x2ebef,
+        count: 7473,
+    },
+    {
+        name: "cjk-g",
+        label: "中日韩统一表意文字扩展G",
+        begin: 0x30000,
+        end: 0x3134f,
+        count: 4939,
+    },
+    {
+        name: "cjk-h",
+        label: "中日韩统一表意文字扩展H",
+        begin: 0x31350,
+        end: 0x323af,
+        count: 4192,
+    },
+    {
+        name: "cjk-i",
+        label: "中日韩统一表意文字扩展I",
+        begin: 0x2ebf0,
+        end: 0x2ee5f,
+        count: 622,
+    },
+    {
+        name: "cjk-j",
+        label: "中日韩统一表意文字扩展J",
+        begin: 0x323b0,
+        end: 0x3347f,
+        count: 4298,
+    },
+    // 部件
+    {
+        name: "radicals-sup",
+        label: "中日韩部首补充",
+        begin: 0x2e80,
+        end: 0x2eff,
+        count: 115,
+    },
+    { name: "kangxi", label: "康熙部首", begin: 0x2f00, end: 0x2fdf, count: 214 },
+    {
+        name: "strokes",
+        label: "中日韩笔画",
+        begin: 0x31c0,
+        end: 0x31ef,
+        count: 38,
+    },
+    // 兼容汉字
+    {
+        name: "compat",
+        label: "中日韩兼容表意文字",
+        begin: 0xf900,
+        end: 0xfaff,
+        count: 472,
+    },
+    {
+        name: "compat-sup",
+        label: "中日韩兼容表意文字补充",
+        begin: 0x2f800,
+        end: 0x2fa1f,
+        count: 542,
+    },
+    // 古文字系统
+    {
+        name: "tangut",
+        label: "西夏文",
+        begin: 0x17000,
+        end: 0x187ff,
+        count: 6144,
+    },
+    {
+        name: "tangut-comp",
+        label: "西夏文字构件",
+        begin: 0x18800,
+        end: 0x18aff,
+        count: 768,
+    },
+    {
+        name: "tangut-sup",
+        label: "西夏文补充",
+        begin: 0x18d00,
+        end: 0x18d7f,
+        count: 31,
+    },
+    {
+        name: "tangut-comp-sup",
+        label: "西夏文字构件补充",
+        begin: 0x18d80,
+        end: 0x18dff,
+        count: 115,
+    },
+    {
+        name: "khitan-small",
+        label: "契丹小字",
+        begin: 0x18b00,
+        end: 0x18cff,
+        count: 470,
+    },
+    // 标点符号与排版字符
+    {
+        name: "punct",
+        label: "中日韩符号和标点",
+        begin: 0x3000,
+        end: 0x303f,
+        count: 64,
+    },
+    {
+        name: "enclosed",
+        label: "中日韩带圈字母和月份",
+        begin: 0x3200,
+        end: 0x32ff,
+        count: 255,
+    },
+    {
+        name: "enclosed-sup",
+        label: "中日韩带圈文字补充",
+        begin: 0x1f200,
+        end: 0x1f2ff,
+        count: 64,
+    },
+    {
+        name: "ideo-symbols",
+        label: "表意文字符号与标点",
+        begin: 0x16fe0,
+        end: 0x16fff,
+        count: 7,
+    },
+    // 私用区
+    { name: "pua", label: "私用区 (BMP)", begin: 0xe000, end: 0xf8ff, count: 0 },
+    {
+        name: "pua-plane15",
+        label: "私用区 (Plane 15)",
+        begin: 0xf0000,
+        end: 0xffffd,
+        count: 0,
+    },
+];
+export const unicodeBlock = (code) => {
+    for (const block of unicodeBlocks) {
+        if (code >= block.begin && code <= block.end) {
+            return block.name;
+        }
+    }
+    return "unknown";
+};
+export const unicodeBlockRank = (code) => {
+    for (const [rank, block] of unicodeBlocks.entries()) {
+        if (code >= block.begin && code <= block.end) {
+            return rank;
+        }
+    }
+    return -1;
+};
+export const isValidCJKBasicChar = (char) => {
+    const code = char.codePointAt(0);
+    const block = unicodeBlock(code);
+    return block === "cjk";
+};
+export const isValidCJKChar = (char) => {
+    const code = char.codePointAt(0);
+    const block = unicodeBlock(code);
+    return block.startsWith("cjk");
+};
+export const isValidCJKSupplement = (char) => {
+    const code = char.codePointAt(0);
+    const block = unicodeBlock(code);
+    return (block === "radicals-sup" ||
+        block === "kangxi" ||
+        block === "strokes" ||
+        block === "compat" ||
+        block === "compat-sup" ||
+        block === "punct");
+};
+export const isPUA = (char) => {
+    const code = char.codePointAt(0);
+    const block = unicodeBlock(code);
+    return block === "pua" || block === "pua-plane15";
+};
+export const chars = (s) => {
+    return Array.from(s).length;
+};
+export const getDummyBasicComponent = () => ({
+    type: "basic_component",
+    strokes: [getDummySVGStroke("横")],
+});
+export const getDummyDerivedComponent = () => ({
+    type: "derived_component",
+    source: "一",
+    strokes: [getDummyReferenceStroke()],
+});
+export const getDummySplicedComponent = () => ({
+    type: "spliced_component",
+    operator: "⿰",
+    operandList: ["一", "丨"],
+});
+export const getDummyReferenceStroke = () => ({
+    feature: "reference",
+    index: 0,
+});
+export const getDummySVGStroke = (feature, start = [0, 0], oldCurveList = []) => {
+    const typelist = schema[feature];
+    return {
+        feature,
+        start,
+        curveList: typelist.map((command, index) => {
+            if (oldCurveList[index]?.command === command) {
+                return oldCurveList[index];
+            }
+            switch (command) {
+                case "a":
+                    return { command, parameterList: [20] };
+                case "h":
+                case "v":
+                    return { command, parameterList: [20] };
+                default:
+                    return { command, parameterList: [10, 10, 20, 20, 30, 30] };
+            }
+        }),
+    };
+};
+export const getDummyCompound = (operator) => ({
+    type: "compound",
+    operator,
+    operandList: ["一", "一"],
+});
+export const isComponent = (glyph) => glyph.type === "basic_component" ||
+    glyph.type === "derived_component" ||
+    glyph.type === "spliced_component" ||
+    glyph.type === "identity";
+export const getSupplemental = (repertoire, list) => {
+    const set = new Set(list);
+    const reverseForm = Object.fromEntries(Object.entries(repertoire).map(([x]) => [x, []]));
+    for (const [char, { glyph }] of Object.entries(repertoire)) {
+        if (glyph?.type === "compound") {
+            glyph.operandList.forEach((x) => reverseForm[x]?.push(char));
+        }
+    }
+    const componentsNotChar = Object.entries(repertoire)
+        .filter(([, v]) => v.glyph?.type === "basic_component")
+        .map(([x]) => x)
+        .filter((x) => !set.has(x));
+    const suppList = [];
+    componentsNotChar.forEach((char) => {
+        let trial = char;
+        while (trial && !set.has(trial)) {
+            trial = reverseForm[trial]?.[0];
+        }
+        if (trial)
+            suppList.push(trial);
+    });
+    return Array.from(new Set(suppList));
+};
+export const listToObject = (list) => Object.fromEntries(list.map((x) => [String.fromCodePoint(x.unicode), x]));
+export function getRecordFromTSV(text) {
+    const tsv = text
+        .trim()
+        .split("\n")
+        .map((x) => x.trim().split("\t"));
+    const data = {};
+    tsv.forEach(([char, freq]) => {
+        if (char === undefined || freq === undefined)
+            return;
+        const maybeNumber = Number(freq);
+        if (Number.isNaN(maybeNumber))
+            return;
+        data[char] = maybeNumber;
+    });
+    return data;
+}
+export function getDistributionFromTSV(text) {
+    const tsv = text
+        .trim()
+        .split("\n")
+        .map((x) => x.trim().split("\t"));
+    const data = {};
+    tsv.forEach(([char, ideal_s, lt_penalty_s, gt_penalty_s]) => {
+        if (char === undefined ||
+            ideal_s === undefined ||
+            lt_penalty_s === undefined ||
+            gt_penalty_s === undefined)
+            return;
+        const [ideal, lt_penalty, gt_penalty] = [
+            ideal_s,
+            lt_penalty_s,
+            gt_penalty_s,
+        ].map(Number);
+        if (Number.isNaN(ideal) ||
+            Number.isNaN(lt_penalty) ||
+            Number.isNaN(gt_penalty))
+            return;
+        data[char] = { ideal, lt_penalty, gt_penalty };
+    });
+    return data;
+}
+export function getDictFromTSV(text) {
+    const result = [];
+    for (const line of text.trim().split("\n")) {
+        const [key, value] = line.trim().split("\t");
+        if (key === undefined || value === undefined)
+            continue;
+        result.push([key, value]);
+    }
+    return result;
+}
+const processExport = (content, filename) => {
+    const blob = new Blob([content], { type: "text/plain" });
+    const a = document.createElement("a");
+    a.download = filename;
+    const url = window.URL.createObjectURL(blob);
+    a.href = url;
+    a.click();
+    window.URL.revokeObjectURL(url); // 避免内存泄漏
+};
+export const exportYAML = (config, filename, flowLevel = 4, sanitize = true) => {
+    const unsafeContent = dump(config, { flowLevel });
+    if (!sanitize) {
+        processExport(unsafeContent, `${filename}.yaml`);
+    }
+    else {
+        const fileContent = unsafeContent.replace(/[\uE000-\uFFFF]/g, (c) => {
+            return `"\\u${c.codePointAt(0)?.toString(16)}"`;
+        });
+        processExport(fileContent, `${filename}.yaml`);
+    }
+};
+export const exportJSON = (data, filename) => {
+    const unsafeContent = JSON.stringify(data);
+    const fileContent = unsafeContent.replace(/[\uE000-\uFFFF]/g, (c) => {
+        return `\\u${c.codePointAt(0)?.toString(16)}`;
+    });
+    processExport(fileContent, filename);
+};
+export const exportTSV = (data, filename) => {
+    const fileContent = data.map((x) => x.join("\t")).join("\n");
+    processExport(fileContent, filename);
+};
+export const stringify = (element) => {
+    if (element === undefined) {
+        return "ε";
+    }
+    else if (typeof element === "string") {
+        return element;
+    }
+    else {
+        return `${element.element}.${element.index}`;
+    }
+};
+export const parseKey = (key) => {
+    if (key === "ε") {
+        return undefined;
+    }
+    else if (key.includes(".")) {
+        const [element, index_s] = key.split(".");
+        const index = Number(index_s);
+        if (Number.isNaN(index)) {
+            throw new Error(`无法解析的键：${key}`);
+        }
+        return { element: element, index };
+    }
+    else {
+        return key;
+    }
+};
+export const joinKeys = (keys) => {
+    return keys.every((x) => typeof x === "string") ? keys.join("") : keys;
+};
+const match = (character, input) => {
+    const { tag, operator, part } = input;
+    if ("glyphs" in character) {
+        const isTagMatched = tag === undefined || character.glyphs.some((x) => x.tags?.includes(tag));
+        const isOperatorMatched = operator === undefined ||
+            character.glyphs.some((x) => "operator" in x && x.operator.includes(operator));
+        const isPartMatched = part === undefined ||
+            character.glyphs.some((x) => "operandList" in x && x.operandList.includes(part));
+        return isTagMatched && isOperatorMatched && isPartMatched;
+    }
+    const isTagMatched = tag === undefined || character.glyph?.tags?.includes(tag);
+    const isOperatorMatched = operator === undefined ||
+        (character.glyph?.type === "compound" &&
+            character.glyph.operator.includes(operator));
+    const isPartMatched = part === undefined ||
+        (character.glyph?.type === "compound" &&
+            character.glyph.operandList.includes(part));
+    return isTagMatched && isOperatorMatched && isPartMatched;
+};
+export const makeCharacterFilter = (input, repertoire, sequenceMap) => {
+    let sequenceRegex;
+    try {
+        if (input.sequence) {
+            sequenceRegex = new RegExp(input.sequence);
+        }
+    }
+    catch { }
+    return (name) => {
+        const character = repertoire[name];
+        if (character === undefined)
+            return false;
+        const sequence = sequenceMap.get(name) ?? "";
+        const isNameMatched = ((character.name ?? "") + name).includes(input.name ?? "");
+        const isSequenceMatched = sequenceRegex?.test(sequence) ?? true;
+        const isUnicodeMatched = input.unicode === undefined || input.unicode === name.codePointAt(0);
+        const isMatched = match(character, input);
+        return isNameMatched && isSequenceMatched && isUnicodeMatched && isMatched;
+    };
+};
+export const makeFilter = (input, form, sequence) => (char) => {
+    if ((sequence.get(char)?.length ?? 0) <= 1)
+        return false;
+    const name = form[char]?.name ?? "";
+    const seq = sequence.get(char) ?? "";
+    return (name.includes(input) || char.includes(input) || seq.startsWith(input));
+};
+export const makeDefaultAnalyzer = (maxLength) => {
+    const form = {
+        type: "all",
+        position: range(0, maxLength),
+        top: 0,
+    };
+    return form;
+};
+function 最大逆向匹配分词(词典, 文本) {
+    const 词列表 = [];
+    const 字符列表 = [...文本];
+    let 结束位置 = 字符列表.length;
+    while (结束位置 > 0) {
+        const 末字 = 字符列表[结束位置 - 1];
+        if (!词典.has(末字)) {
+            词列表.unshift(末字);
+            结束位置 -= 1;
+            continue;
+        }
+        for (let 开始位置 = 0; 开始位置 < 结束位置; 开始位置++) {
+            const 词 = 字符列表.slice(开始位置, 结束位置).join("");
+            if (词典.has(词)) {
+                词列表.unshift(词);
+                结束位置 = 开始位置;
+                break;
+            }
+        }
+    }
+    return 词列表;
+}
+// 使用使用逆向最大匹配算法来分词
+export const adapt = (原始词频映射, 词典) => {
+    const 新词频映射 = new Map();
+    for (const [词, 词频] of Object.entries(原始词频映射)) {
+        if (词典.has(词)) {
+            新词频映射.set(词, (新词频映射.get(词) ?? 0) + 词频);
+        }
+        else {
+            const 子词列表 = 最大逆向匹配分词(词典, 词);
+            for (const 子词 of 子词列表) {
+                新词频映射.set(子词, (新词频映射.get(子词) ?? 0) + 词频);
+            }
+        }
+    }
+    return 新词频映射;
+};
+export const getReversedMapping = (mapping, alphabet) => {
+    const reversedMapping = new Map(Array.from(alphabet).map((key) => [key, []]));
+    const recursivelyRenderKey = (key) => {
+        if (typeof key === "string")
+            return key;
+        const mapped = mapping[key.element];
+        if (typeof mapped === "object" && "element" in mapped) {
+            return "";
+        }
+        return recursivelyRenderKey(mapped[key.index]);
+    };
+    for (const [name, code] of Object.entries(mapping)) {
+        if (typeof code === "string" || Array.isArray(code)) {
+            const first = recursivelyRenderKey(code[0]);
+            reversedMapping.get(first)?.push({ name, code });
+        }
+    }
+    return reversedMapping;
+};
+export const characterSetFilters = {
+    minimal: (_, v) => v.gb2312 > 0 && v.tygf > 0,
+    gb2312: (_, v) => v.gb2312 > 0,
+    general: (_, v) => v.tygf > 0,
+    basic: (k, v) => v.tygf > 0 || isValidCJKBasicChar(k),
+    extended: (k, v) => v.tygf > 0 || isValidCJKChar(k),
+    supplement: (k, v) => v.tygf > 0 || isValidCJKChar(k) || isValidCJKSupplement(k),
+    maximal: (k, v) => !isPUA(k),
+    custom: (k, v, s) => {
+        if (s !== undefined)
+            return s.has(k);
+        return v.gb2312 > 0 && v.tygf > 0;
+    },
+};

package/package.json

@@ -0,0 +1,34 @@
+{
+  "name": "hanzi-chai",
+  "version": "0.3.0",
+  "description": "提取自主仓库的汉字拆分/编码核心算法包",
+  "type": "module",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "build": "tsc -p tsconfig.build.json && cp -r src/data dist/data",
+    "test": "vitest",
+    "testui": "vitest --ui --coverage",
+    "prepublishOnly": "npm run build"
+  },
+  "dependencies": {
+    "@vitest/coverage-v8": "^4.0.16",
+    "@vitest/ui": "^4.0.16",
+    "d3-array": "^3.0.2",
+    "js-yaml": "^4.1.0",
+    "lodash-es": "^4.17.21"
+  },
+  "devDependencies": {
+    "@types/node": "^25.0.3",
+    "tsup": "^6.0.0",
+    "typescript": "^5.0.0",
+    "vitest": "^4.0.16"
+  }
+}