@laisuk/opencc-fmmseg-wasm 0.3.4 → 0.3.6

package/README.md

@@ -21,6 +21,7 @@ Features:
 * Traditional Chinese regional variants
 * Japanese Shinjitai conversion support
 * Chinese script detection (`zho_check`)
+* Optional CJK Compatibility Ideograph normalization
 * In-memory Office / EPUB document conversion
 * Zero-dependency Node.js CLI
 
@@ -162,6 +163,41 @@ cc.convert("汉字", false);
 
 ---
 
+### normalizeCompat
+
+Normalize Unicode CJK Compatibility Ideographs before conversion.
+
+```javascript
+cc.normalizeCompat(text)
+```
+
+Parameters:
+
+* `text`: input string
+
+Returns:
+
+* normalized string
+
+Example:
+
+```javascript
+const cc = new OpenccWasm("t2s");
+
+const input = "天龍八部書裡的喬峰是契丹人";
+const normalized = cc.normalizeCompat(input);
+
+console.log(normalized);
+// 天龍八部書裡的喬峰是契丹人
+
+console.log(cc.convert(normalized, false));
+// 天龙八部书里的乔峰是契丹人
+```
+
+This is an optional pre-conversion pass for text that contains compatibility ideographs from Unicode compatibility ranges. Unmapped characters are preserved unchanged. Normal OpenCC conversion does not automatically run this pass, so call it explicitly when compatibility normalization is desired.
+
+---
+
 ### detofu
 
 Replace tofu-risk rare CJK extension characters with display-compatible fallbacks.
@@ -387,8 +423,8 @@ const cc = OpenccWasm.newWithCustomDicts("s2t", [
 `Override` replaces the selected slot before inserting the provided pairs. It is powerful and should be used only when
 the caller intentionally wants to discard built-in entries for that slot.
 
-Custom dictionary specs identify the target dictionary slot by `DictSlot` name. Slot names are trimmed and normalized
-case-insensitively for the known slots, so `"stphrases"`, `" STPhrases "`, and `"STPhrases"` all select
+Custom dictionary specs identify the target dictionary slot by `DictSlot` name. Slot names are trimmed and normalized
+case-insensitively for the known slots, so `"stphrases"`, `" STPhrases "`, and `"STPhrases"` all select
 `STPhrases`. Canonical names are recommended in TypeScript code and docs:
 
 ```text
@@ -415,7 +451,7 @@ STPunctuations
 TSPunctuations
 ```
 
-Suffixes such as `.txt` are not accepted, even though case and surrounding whitespace are normalized. Use
+Suffixes such as `.txt` are not accepted, even though case and surrounding whitespace are normalized. Use
 `"STPhrases"` or `"stphrases"`, not `"STPhrases.txt"`.
 
 Merge contract:
@@ -584,6 +620,8 @@ The package includes a zero-dependency Node.js CLI:
 opencc-fmmseg convert -i input.txt -o output.txt -c s2t -p
 opencc-fmmseg convert -i input.txt -o output.txt -c t2s -p --detofu all
 echo "别随便录影侵犯个人隐私权" | opencc-fmmseg convert -c s2hkp
+echo "天龍八部書裡的喬峰是契丹人" | opencc-fmmseg convert -c t2s --norm-compat
+// 天龙八部书里的乔峰是契丹人
 echo "這個細路哥很靈活" | opencc-fmmseg convert -c hk2sp --custom-dict hkphrasesrev:append:my_hk_dict.txt  
 // 这个小男孩很灵活
 ```
@@ -603,21 +641,23 @@ opencc-fmmseg office -i input.docx -o output.docx -c s2t -p --keep-font
 ### Text Conversion Options
 
 ```text
--i, --input <file>          Input text file
--o, --output <file>         Output text file
--c, --config <conversion>   Conversion config
+-i, --input <file>          Input text file; stdin if omitted
+-o, --output <file>         Output text file; stdout if omitted
+-c, --config <conversion>   Conversion config (default: s2t)
 -p, --punct                 Enable punctuation conversion
 --detofu [level]            Replace tofu-risk rare CJK extension chars after conversion
                               level: all | ext-b | ext-c | ext-d | ext-e | ext-f | ext-g | ext-h | ext-i
                               default when omitted value: all
---custom-dict <slot:mode:file>
+--keep-ids                  Preserve complete IDS expressions during conversion (default: false)
+-n, --norm-compat           Normalize CJK Compatibility Ideographs before conversion (default: false)
+-D, --custom-dict <slot:mode:file>
                             Load a custom dictionary.
                             May be specified multiple times.
                             Examples:
                               --custom-dict hkphrasesrev:append:my_hk_dict.txt
                               --custom-dict stphrases:override:terms.txt
---in-enc <encoding>         Input encoding
---out-enc <encoding>        Output encoding
+--in-enc <encoding>         Input encoding (default: utf8)
+--out-enc <encoding>        Output encoding (default: utf8)
 ```
 
 Supported conversion configs:
@@ -632,12 +672,18 @@ tw2s, tw2sp, tw2t, tw2tp, hk2s, hk2sp, hk2t, jp2t, t2jp
 ```text
 -i, --input <file>          Input Office / EPUB file
 -o, --output <file>         Output file
--c, --config <conversion>   Conversion config
+-c, --config <conversion>   Conversion config (default: s2t)
 -p, --punct                 Enable punctuation conversion
---format <format>           docx | xlsx | pptx | odt | ods | odp | epub
---auto-ext                  Append extension to output if missing
---keep-font                 Preserve font-family information
+-f, --format <format>       docx | xlsx | pptx | odt | ods | odp | epub
+-F, --convert-filename      Convert generated output filename stem (default: false)
+--keep-font                 Preserve font-family information (default)
 --no-keep-font              Do not preserve font-family information
+--custom-dict <slot:mode:file>
+                            Load a custom dictionary.
+                            May be specified multiple times.
+                            Examples:
+                              --custom-dict hkphrasesrev:append:my_hk_dict.txt
+                              --custom-dict stphrases:override:terms.txt
 ```
 
 For `office`, the format is inferred from the input file extension when `--format` is omitted.

package/bin/opencc.js

@@ -57,7 +57,8 @@ Convert options:
                               level: all | ext-b | ext-c | ext-d | ext-e | ext-f | ext-g | ext-h | ext-i
                               default when omitted value: all
   --keep-ids                  Preserve complete IDS expressions during conversion (default: false)
-  --custom-dict <slot:mode:file>
+  -n, --norm-compat           Normalize CJK Compatibility Ideographs before conversion (default: false)
+  -D, --custom-dict <slot:mode:file>
                               Load a custom dictionary.
                               May be specified multiple times.
                               Examples:
@@ -75,8 +76,8 @@ Office options:
   -o, --output <file>         Output file
   -c, --config <conversion>   Conversion config (default: s2t)
   -p, --punct                 Enable punctuation conversion
-  --format <format>           docx | xlsx | pptx | odt | ods | odp | epub
-  --convert-filename          Convert generated output filename stem (default: false)
+  -f, --format <format>       docx | xlsx | pptx | odt | ods | odp | epub
+  -F, --convert-filename      Convert generated output filename stem (default: false)
   --keep-font                 Preserve font-family information (default)
   --no-keep-font              Do not preserve font-family information
   --custom-dict <slot:mode:file>
@@ -126,12 +127,18 @@ function getArg(args, shortName, longName, defaultValue = null) {
     return defaultValue;
 }
 
-function getArgs(args, longName) {
+function getArgs(args, shortName, longName) {
     const values = [];
+    const candidates = [];
+
+    if (shortName) candidates.push(shortName);
+    if (longName) candidates.push(longName);
+
+    if (candidates.length === 0) return values;
 
     for (let i = 0; i < args.length; i++) {
-        if (args[i] === longName && i + 1 < args.length) {
-            values.push(parseCustomDictSpec(args[i + 1]));
+        if (candidates.includes(args[i]) && i + 1 < args.length) {
+            values.push(args[i + 1]);
             i++;
         }
     }
@@ -344,7 +351,9 @@ async function runConvert(args) {
     const outEnc = getArg(args, null, "--out-enc", "utf8");
     const punct = hasFlag(args, "-p", "--punct");
     const keepIds = hasFlag(args, null, "--keep-ids");
-    const customDicts = getArgs(args, "--custom-dict");
+    const normCompat = hasFlag(args, "-n", "--norm-compat");
+    const customDicts = getArgs(args, "-D", "--custom-dict")
+        .map(parseCustomDictSpec);
 
     const detofuIndex = args.indexOf("--detofu");
     const detofuEnabled = detofuIndex !== -1;
@@ -376,7 +385,12 @@ async function runConvert(args) {
         console.error("Input text to convert, <Ctrl+Z>/<Ctrl+D> to submit:");
     }
 
-    const inputText = readInputText(input, inEnc);
+    let inputText = readInputText(input, inEnc);
+
+    if (normCompat) {
+        inputText = cc.normalizeCompat(inputText);
+    }
+
     let outputText = cc.convert(inputText, punct);
 
     if (detofuEnabled) {
@@ -394,6 +408,7 @@ async function runConvert(args) {
         }
 
         const suffixParts = [];
+        if (normCompat) suffixParts.push("normalized");
         if (detofuEnabled) suffixParts.push("detofu");
         if (keepIds) suffixParts.push("keep-ids");
 
@@ -406,11 +421,12 @@ async function runOffice(args) {
     const input = getArg(args, "-i", "--input");
     let output = getArg(args, "-o", "--output");
     const config = getArg(args, "-c", "--config", "s2t");
-    const explicitFormat = getArg(args, null, "--format");
+    const explicitFormat = getArg(args, "-f", "--format");
     const punct = hasFlag(args, "-p", "--punct");
-    const convertFilename = hasFlag(args, null, "--convert-filename");
+    const convertFilename = hasFlag(args, "-F", "--convert-filename");
     const keepFont = !hasFlag(args, null, "--no-keep-font");
-    const customDicts = getArgs(args, "--custom-dict");
+    const customDicts = getArgs(args, "-D", "--custom-dict")
+        .map(parseCustomDictSpec);
 
     if (!input) {
         throw new Error("Input file is missing.");

package/opencc_fmmseg_wasm.d.ts

@@ -49,6 +49,7 @@ export class OpenccWasm {
     constructor(config?: string | null);
     static newWithCustomDicts(config: string | null | undefined, specs: any): OpenccWasm;
     static newWithEnum(config?: OpenccConfigWasm | null): OpenccWasm;
+    normalizeCompat(text: string): string;
     setConfig(config: string): boolean;
     setConfigEnum(config: OpenccConfigWasm): void;
     setPreserveIds(value: boolean): void;
@@ -77,6 +78,7 @@ export interface InitOutput {
     readonly openccwasm_new: (a: number, b: number) => [number, number, number];
     readonly openccwasm_newWithCustomDicts: (a: number, b: number, c: any) => [number, number, number];
     readonly openccwasm_newWithEnum: (a: number) => [number, number, number];
+    readonly openccwasm_normalizeCompat: (a: number, b: number, c: number) => [number, number];
     readonly openccwasm_setConfig: (a: number, b: number, c: number) => number;
     readonly openccwasm_setConfigEnum: (a: number, b: number) => void;
     readonly openccwasm_setPreserveIds: (a: number, b: number) => void;

package/opencc_fmmseg_wasm.js

@@ -235,6 +235,24 @@ export class OpenccWasm {
         }
         return OpenccWasm.__wrap(ret[0]);
     }
+    /**
+     * @param {string} text
+     * @returns {string}
+     */
+    normalizeCompat(text) {
+        let deferred2_0;
+        let deferred2_1;
+        try {
+            const ptr0 = passStringToWasm0(text, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+            const len0 = WASM_VECTOR_LEN;
+            const ret = wasm.openccwasm_normalizeCompat(this.__wbg_ptr, ptr0, len0);
+            deferred2_0 = ret[0];
+            deferred2_1 = ret[1];
+            return getStringFromWasm0(ret[0], ret[1]);
+        } finally {
+            wasm.__wbindgen_free(deferred2_0, deferred2_1, 1);
+        }
+    }
     /**
      * @param {string} config
      * @returns {boolean}

package/package.json

@@ -2,7 +2,7 @@
   "name": "@laisuk/opencc-fmmseg-wasm",
   "type": "module",
   "description": "WebAssembly bindings for opencc-fmmseg, a high-performance OpenCC-compatible Simplified/Traditional Chinese converter.",
-  "version": "0.3.4",
+  "version": "0.3.6",
   "license": "MIT",
   "repository": {
     "type": "git",