@mailwoman/normalize 9.1.0 → 9.2.0

package/README.md

@@ -61,7 +61,7 @@ Stage 1 in the [Staged Pipeline Contract](https://github.com/sister-software/mai
 
 - [`@mailwoman/query-shape`](../query-shape) — Stage 1.5, structural priors that consume the normalized output
 - [Staged Pipeline Contract](https://github.com/sister-software/mailwoman/blob/main/docs/engineering/reference/STAGES.mdx)
-- [Tokenization concepts](https://mailwoman.sister.software/articles/concepts/tokenization/)
+- [Tokenization concepts](https://mailwoman.ai/articles/concepts/tokenization/)
 
 ## License
 

package/cjk.ts

@@ -16,6 +16,9 @@
  *       1, a full-width `－` always a hyphen — keyboards and copy-paste produce these constantly.
  *       Folding them to ASCII makes `１０４−００６１` and `104-0061` the same input.
  *   - **Fold the ideographic space (U+3000 → ' ').**
+ *   - **Fold half-width katakana (U+FF61–U+FF9F).** This matches the JP corpus builder. A voiced
+ *       pair such as `ﾃﾞ` contracts from two UTF-16 units to one `デ`; the output offset maps to the
+ *       first raw unit in the pair.
  *
  *   It deliberately does NOT convert **kanji numerals** (一二三…): place names carry numeral kanji as
  *   ordinary characters (三田 _Mita_, 四谷 _Yotsuya_), so a blind 三→3 would corrupt them.
@@ -59,6 +62,13 @@ const IDEOGRAPHIC_SPACE = 0x30_00
  * 〒.
  */
 const POSTAL_MARK = 0x30_12
+const HALFWIDTH_KATAKANA_START = 0xff_61
+const HALFWIDTH_KATAKANA_END = 0xff_9f
+const HALFWIDTH_VOICING_START = 0xff_9e
+
+function isHalfwidthKatakana(code: number): boolean {
+	return code >= HALFWIDTH_KATAKANA_START && code <= HALFWIDTH_KATAKANA_END
+}
 
 export function applyCjkNormalization(input: string): CjkResult {
 	let folded = 0
@@ -93,6 +103,30 @@ export function applyCjkNormalization(input: string): CjkResult {
 			continue
 		}
 
+		if (isHalfwidthKatakana(code)) {
+			const next = input.charCodeAt(i + 1)
+
+			const consumesVoicingMark =
+				code < HALFWIDTH_VOICING_START && next >= HALFWIDTH_VOICING_START && next <= HALFWIDTH_KATAKANA_END
+
+			const width = consumesVoicingMark ? 2 : 1
+
+			const normalized = input
+				.slice(i, i + width)
+				.normalize("NFKC")
+				.normalize("NFC")
+
+			for (const unit of normalized) {
+				out.push(unit)
+				map.push(i)
+			}
+
+			folded += width
+			i += width - 1
+
+			continue
+		}
+
 		out.push(input[i]!)
 		map.push(i)
 	}

package/out/cjk.d.ts

@@ -16,6 +16,9 @@
  *       1, a full-width `－` always a hyphen — keyboards and copy-paste produce these constantly.
  *       Folding them to ASCII makes `１０４−００６１` and `104-0061` the same input.
  *   - **Fold the ideographic space (U+3000 → ' ').**
+ *   - **Fold half-width katakana (U+FF61–U+FF9F).** This matches the JP corpus builder. A voiced
+ *       pair such as `ﾃﾞ` contracts from two UTF-16 units to one `デ`; the output offset maps to the
+ *       first raw unit in the pair.
  *
  *   It deliberately does NOT convert **kanji numerals** (一二三…): place names carry numeral kanji as
  *   ordinary characters (三田 _Mita_, 四谷 _Yotsuya_), so a blind 三→3 would corrupt them.

package/out/cjk.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"cjk.d.ts","sourceRoot":"","sources":["../cjk.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAIH,MAAM,WAAW,SAAS;IACzB,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,MAAM,EAAE,CAAA;IACb;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;IACd;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAA;CAChB;AAoBD,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,CA0C9D"}
+{"version":3,"file":"cjk.d.ts","sourceRoot":"","sources":["../cjk.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAIH,MAAM,WAAW,SAAS;IACzB,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,MAAM,EAAE,CAAA;IACb;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;IACd;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAA;CAChB;AA2BD,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,CAkE9D"}

package/out/cjk.js

@@ -16,6 +16,9 @@
  *       1, a full-width `－` always a hyphen — keyboards and copy-paste produce these constantly.
  *       Folding them to ASCII makes `１０４−００６１` and `104-0061` the same input.
  *   - **Fold the ideographic space (U+3000 → ' ').**
+ *   - **Fold half-width katakana (U+FF61–U+FF9F).** This matches the JP corpus builder. A voiced
+ *       pair such as `ﾃﾞ` contracts from two UTF-16 units to one `デ`; the output offset maps to the
+ *       first raw unit in the pair.
  *
  *   It deliberately does NOT convert **kanji numerals** (一二三…): place names carry numeral kanji as
  *   ordinary characters (三田 _Mita_, 四谷 _Yotsuya_), so a blind 三→3 would corrupt them.
@@ -44,6 +47,12 @@ const IDEOGRAPHIC_SPACE = 0x30_00;
  * 〒.
  */
 const POSTAL_MARK = 0x30_12;
+const HALFWIDTH_KATAKANA_START = 0xff_61;
+const HALFWIDTH_KATAKANA_END = 0xff_9f;
+const HALFWIDTH_VOICING_START = 0xff_9e;
+function isHalfwidthKatakana(code) {
+    return code >= HALFWIDTH_KATAKANA_START && code <= HALFWIDTH_KATAKANA_END;
+}
 export function applyCjkNormalization(input) {
     let folded = 0;
     let stripped = 0;
@@ -69,6 +78,22 @@ export function applyCjkNormalization(input) {
             folded += 1;
             continue;
         }
+        if (isHalfwidthKatakana(code)) {
+            const next = input.charCodeAt(i + 1);
+            const consumesVoicingMark = code < HALFWIDTH_VOICING_START && next >= HALFWIDTH_VOICING_START && next <= HALFWIDTH_KATAKANA_END;
+            const width = consumesVoicingMark ? 2 : 1;
+            const normalized = input
+                .slice(i, i + width)
+                .normalize("NFKC")
+                .normalize("NFC");
+            for (const unit of normalized) {
+                out.push(unit);
+                map.push(i);
+            }
+            folded += width;
+            i += width - 1;
+            continue;
+        }
         out.push(input[i]);
         map.push(i);
     }

package/out/cjk.js.map

@@ -1 +1 @@
-{"version":3,"file":"cjk.js","sourceRoot":"","sources":["../cjk.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAA;AAe7C;;GAEG;AACH,MAAM,eAAe,GAAG,OAAO,CAAA;AAC/B;;GAEG;AACH,MAAM,aAAa,GAAG,OAAO,CAAA;AAC7B;;GAEG;AACH,MAAM,kBAAkB,GAAG,OAAO,CAAA;AAClC,MAAM,iBAAiB,GAAG,OAAO,CAAA;AACjC;;GAEG;AACH,MAAM,WAAW,GAAG,OAAO,CAAA;AAE3B,MAAM,UAAU,qBAAqB,CAAC,KAAa;IAClD,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,IAAI,QAAQ,GAAG,CAAC,CAAA;IAChB,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,MAAM,GAAG,GAAa,EAAE,CAAA;IAExB,gGAAgG;IAChG,oFAAoF;IACpF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,IAAI,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAA;QAEhC,IAAI,IAAI,KAAK,WAAW,EAAE,CAAC;YAC1B,QAAQ,IAAI,CAAC,CAAA;YAEb,SAAQ,CAAC,yEAAyE;QACnF,CAAC;QAED,IAAI,IAAI,IAAI,eAAe,IAAI,IAAI,IAAI,aAAa,EAAE,CAAC;YACtD,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,GAAG,kBAAkB,CAAC,CAAC,CAAA;YACxD,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACX,MAAM,IAAI,CAAC,CAAA;YAEX,SAAQ;QACT,CAAC;QAED,IAAI,IAAI,KAAK,iBAAiB,EAAE,CAAC;YAChC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YACb,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACX,MAAM,IAAI,CAAC,CAAA;YAEX,SAAQ;QACT,CAAC;QAED,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,CAAA;QACnB,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACZ,CAAC;IAED,IAAI,MAAM,KAAK,CAAC,IAAI,QAAQ,KAAK,CAAC,EAAE,CAAC;QACpC,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAA;IAC/E,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAA;AACrD,CAAC"}
+{"version":3,"file":"cjk.js","sourceRoot":"","sources":["../cjk.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAA;AAe7C;;GAEG;AACH,MAAM,eAAe,GAAG,OAAO,CAAA;AAC/B;;GAEG;AACH,MAAM,aAAa,GAAG,OAAO,CAAA;AAC7B;;GAEG;AACH,MAAM,kBAAkB,GAAG,OAAO,CAAA;AAClC,MAAM,iBAAiB,GAAG,OAAO,CAAA;AACjC;;GAEG;AACH,MAAM,WAAW,GAAG,OAAO,CAAA;AAC3B,MAAM,wBAAwB,GAAG,OAAO,CAAA;AACxC,MAAM,sBAAsB,GAAG,OAAO,CAAA;AACtC,MAAM,uBAAuB,GAAG,OAAO,CAAA;AAEvC,SAAS,mBAAmB,CAAC,IAAY;IACxC,OAAO,IAAI,IAAI,wBAAwB,IAAI,IAAI,IAAI,sBAAsB,CAAA;AAC1E,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,KAAa;IAClD,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,IAAI,QAAQ,GAAG,CAAC,CAAA;IAChB,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,MAAM,GAAG,GAAa,EAAE,CAAA;IAExB,gGAAgG;IAChG,oFAAoF;IACpF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,IAAI,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAA;QAEhC,IAAI,IAAI,KAAK,WAAW,EAAE,CAAC;YAC1B,QAAQ,IAAI,CAAC,CAAA;YAEb,SAAQ,CAAC,yEAAyE;QACnF,CAAC;QAED,IAAI,IAAI,IAAI,eAAe,IAAI,IAAI,IAAI,aAAa,EAAE,CAAC;YACtD,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,GAAG,kBAAkB,CAAC,CAAC,CAAA;YACxD,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACX,MAAM,IAAI,CAAC,CAAA;YAEX,SAAQ;QACT,CAAC;QAED,IAAI,IAAI,KAAK,iBAAiB,EAAE,CAAC;YAChC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YACb,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACX,MAAM,IAAI,CAAC,CAAA;YAEX,SAAQ;QACT,CAAC;QAED,IAAI,mBAAmB,CAAC,IAAI,CAAC,EAAE,CAAC;YAC/B,MAAM,IAAI,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;YAEpC,MAAM,mBAAmB,GACxB,IAAI,GAAG,uBAAuB,IAAI,IAAI,IAAI,uBAAuB,IAAI,IAAI,IAAI,sBAAsB,CAAA;YAEpG,MAAM,KAAK,GAAG,mBAAmB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;YAEzC,MAAM,UAAU,GAAG,KAAK;iBACtB,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC;iBACnB,SAAS,CAAC,MAAM,CAAC;iBACjB,SAAS,CAAC,KAAK,CAAC,CAAA;YAElB,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;gBAC/B,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;gBACd,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACZ,CAAC;YAED,MAAM,IAAI,KAAK,CAAA;YACf,CAAC,IAAI,KAAK,GAAG,CAAC,CAAA;YAEd,SAAQ;QACT,CAAC;QAED,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,CAAA;QACnB,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACZ,CAAC;IAED,IAAI,MAAM,KAAK,CAAC,IAAI,QAAQ,KAAK,CAAC,EAAE,CAAC;QACpC,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAA;IAC/E,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAA;AACrD,CAAC"}

package/out/punctuation.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"punctuation.d.ts","sourceRoot":"","sources":["../punctuation.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAiBH,MAAM,WAAW,iBAAiB;IACjC,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,MAAM,EAAE,CAAA;IACb,YAAY,EAAE,MAAM,CAAA;CACpB;AAED,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,GAAG,iBAAiB,CA8BjE"}
+{"version":3,"file":"punctuation.d.ts","sourceRoot":"","sources":["../punctuation.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAmBH,MAAM,WAAW,iBAAiB;IACjC,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,MAAM,EAAE,CAAA;IACb,YAAY,EAAE,MAAM,CAAA;CACpB;AAED,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,GAAG,iBAAiB,CA8BjE"}

package/out/punctuation.js

@@ -14,8 +14,10 @@ const REPLACEMENTS = new Map([
     ["”", '"'], // ”
     ["–", "-"], // – en dash
     ["—", "-"], // — em dash
+    ["‐", "-"], // ‐ U+2010 hyphen
     ["−", "-"], // − U+2212 minus sign — Japanese IMEs emit this as the block separator (1−2−3)
     ["―", "-"], // ― U+2015 horizontal bar — another common JP block separator
+    ["﹣", "-"], // ﹣ U+FE63 small hyphen-minus
     ["…", "..."], // … expands; tracked specially
     [" ", " "], // non-breaking space
 ]);

package/out/punctuation.js.map

@@ -1 +1 @@
-{"version":3,"file":"punctuation.js","sourceRoot":"","sources":["../punctuation.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAA;AAE7C,MAAM,YAAY,GAAG,IAAI,GAAG,CAAiB;IAC5C,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,IAAI;IAChB,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,IAAI;IAChB,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,IAAI;IAChB,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,IAAI;IAChB,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,YAAY;IACxB,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,YAAY;IACxB,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,+EAA+E;IAC3F,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,8DAA8D;IAC1E,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,+BAA+B;IAC7C,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,qBAAqB;CACjC,CAAC,CAAA;AAQF,MAAM,UAAU,gBAAgB,CAAC,KAAa;IAC7C,IAAI,OAAO,GAAG,KAAK,CAAA;IACnB,IAAI,YAAY,GAAG,CAAC,CAAA;IACpB,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,MAAM,GAAG,GAAa,EAAE,CAAA;IAExB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,CAAE,CAAA;QACpB,MAAM,GAAG,GAAG,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QAEhC,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;YACvB,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;YACZ,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACZ,CAAC;aAAM,CAAC;YACP,OAAO,GAAG,IAAI,CAAA;YACd,YAAY,IAAI,CAAC,CAAA;YAEjB,mGAAmG;YACnG,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACrC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,CAAA;gBACjB,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACZ,CAAC;QACF,CAAC;IACF,CAAC;IAED,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,CAAA;IACxE,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,YAAY,EAAE,CAAA;AACjD,CAAC"}
+{"version":3,"file":"punctuation.js","sourceRoot":"","sources":["../punctuation.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAA;AAE7C,MAAM,YAAY,GAAG,IAAI,GAAG,CAAiB;IAC5C,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,IAAI;IAChB,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,IAAI;IAChB,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,IAAI;IAChB,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,IAAI;IAChB,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,YAAY;IACxB,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,YAAY;IACxB,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,kBAAkB;IAC9B,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,+EAA+E;IAC3F,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,8DAA8D;IAC1E,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,8BAA8B;IAC1C,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,+BAA+B;IAC7C,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,qBAAqB;CACjC,CAAC,CAAA;AAQF,MAAM,UAAU,gBAAgB,CAAC,KAAa;IAC7C,IAAI,OAAO,GAAG,KAAK,CAAA;IACnB,IAAI,YAAY,GAAG,CAAC,CAAA;IACpB,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,MAAM,GAAG,GAAa,EAAE,CAAA;IAExB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,CAAE,CAAA;QACpB,MAAM,GAAG,GAAG,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QAEhC,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;YACvB,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;YACZ,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACZ,CAAC;aAAM,CAAC;YACP,OAAO,GAAG,IAAI,CAAA;YACd,YAAY,IAAI,CAAC,CAAA;YAEjB,mGAAmG;YACnG,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACrC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,CAAA;gBACjB,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACZ,CAAC;QACF,CAAC;IACF,CAAC;IAED,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,CAAA;IACxE,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,YAAY,EAAE,CAAA;AACjD,CAAC"}

package/package.json

@@ -1,12 +1,12 @@
 {
   "name": "@mailwoman/normalize",
-  "version": "9.1.0",
+  "version": "9.2.0",
   "description": "Stage 1 of the runtime pipeline — deterministic input preprocessing (Unicode NFC, punctuation, whitespace, abbreviation). Pure functions, no ML.",
   "license": "AGPL-3.0-only OR LicenseRef-Commercial",
   "repository": {
     "type": "git",
     "url": "https://github.com/sister-software/mailwoman.git",
-    "directory": "normalize"
+    "directory": "packages/normalize"
   },
   "files": [
     "out/**/*.js",
@@ -20,7 +20,8 @@
     "!*.test.ts",
     "!*.test.tsx",
     "!**/*.test.ts",
-    "!**/*.test.tsx"
+    "!**/*.test.tsx",
+    "!test/**"
   ],
   "type": "module",
   "exports": {
@@ -28,6 +29,34 @@
     ".": {
       "types": "./out/index.d.ts",
       "default": "./out/index.js"
+    },
+    "./abbreviations": {
+      "types": "./out/abbreviations.d.ts",
+      "default": "./out/abbreviations.js"
+    },
+    "./cjk": {
+      "types": "./out/cjk.d.ts",
+      "default": "./out/cjk.js"
+    },
+    "./compute": {
+      "types": "./out/compute.d.ts",
+      "default": "./out/compute.js"
+    },
+    "./nfc": {
+      "types": "./out/nfc.d.ts",
+      "default": "./out/nfc.js"
+    },
+    "./offset-map": {
+      "types": "./out/offset-map.d.ts",
+      "default": "./out/offset-map.js"
+    },
+    "./punctuation": {
+      "types": "./out/punctuation.d.ts",
+      "default": "./out/punctuation.js"
+    },
+    "./whitespace": {
+      "types": "./out/whitespace.d.ts",
+      "default": "./out/whitespace.js"
     }
   },
   "publishConfig": {
@@ -37,6 +66,34 @@
       ".": {
         "types": "./out/index.d.ts",
         "default": "./out/index.js"
+      },
+      "./abbreviations": {
+        "types": "./out/abbreviations.d.ts",
+        "default": "./out/abbreviations.js"
+      },
+      "./cjk": {
+        "types": "./out/cjk.d.ts",
+        "default": "./out/cjk.js"
+      },
+      "./compute": {
+        "types": "./out/compute.d.ts",
+        "default": "./out/compute.js"
+      },
+      "./nfc": {
+        "types": "./out/nfc.d.ts",
+        "default": "./out/nfc.js"
+      },
+      "./offset-map": {
+        "types": "./out/offset-map.d.ts",
+        "default": "./out/offset-map.js"
+      },
+      "./punctuation": {
+        "types": "./out/punctuation.d.ts",
+        "default": "./out/punctuation.js"
+      },
+      "./whitespace": {
+        "types": "./out/whitespace.d.ts",
+        "default": "./out/whitespace.js"
       }
     }
   }

package/punctuation.ts

@@ -16,8 +16,10 @@ const REPLACEMENTS = new Map<string, string>([
 	["”", '"'], // ”
 	["–", "-"], // – en dash
 	["—", "-"], // — em dash
+	["‐", "-"], // ‐ U+2010 hyphen
 	["−", "-"], // − U+2212 minus sign — Japanese IMEs emit this as the block separator (1−2−3)
 	["―", "-"], // ― U+2015 horizontal bar — another common JP block separator
+	["﹣", "-"], // ﹣ U+FE63 small hyphen-minus
 	["…", "..."], // … expands; tracked specially
 	[" ", " "], // non-breaking space
 ])