@dev-pi2pie/word-counter 0.1.2 → 0.1.3-canary.1

package/README.md

@@ -69,6 +69,24 @@ word-counter --latin-language en "Hello world"
 word-counter --latin-tag en "Hello world"
 ```
 
+Add custom Latin hint rules (repeatable) or load from JSON:
+
+```bash
+word-counter --latin-hint 'pl=[ąćęłńóśźżĄĆĘŁŃÓŚŹŻ]' "Zażółć gęślą jaźń"
+word-counter --latin-hint 'tr=[çğıöşüÇĞİÖŞÜ]' --latin-hint 'ro=[ăâîșțĂÂÎȘȚ]' "șță"
+word-counter --latin-hints-file ./examples/latin-hints.json "Zażółć Știință Iğdır"
+word-counter --no-default-latin-hints --latin-hint 'pl=[ąćęłńóśźżĄĆĘŁŃÓŚŹŻ]' "Zażółć"
+```
+
+`examples/latin-hints.json` format:
+
+```json
+[
+  { "tag": "pl", "pattern": "[ąćęłńóśźżĄĆĘŁŃÓŚŹŻ]" },
+  { "tag": "tr", "pattern": "[çğıöşüÇĞİÖŞÜ]", "priority": 1 }
+]
+```
+
 Hint a language tag for Han fallback:
 
 ```bash
@@ -215,7 +233,7 @@ Skip details stay debug-gated and can still be suppressed with `--quiet-skips`.
 
 ## How It Works
 
-- The runtime inspects each character's Unicode script to infer its likely locale tag (e.g., `und-Latn`, `zh-Hani`, `ja`).
+- The runtime inspects each character's Unicode script to infer its likely locale tag (e.g., `und-Latn`, `und-Hani`, `ja`).
 - Adjacent characters that share the same locale tag are grouped into a chunk.
 - Each chunk is counted with `Intl.Segmenter` at `granularity: "word"`, caching segmenters to avoid re-instantiation.
 - Per-locale counts are summed into an overall total and printed to stdout.
@@ -223,7 +241,7 @@ Skip details stay debug-gated and can still be suppressed with `--quiet-skips`.
 ## Locale vs Language Code
 
 - Output keeps the field name `locale` for compatibility.
-- In this project, locale values are BCP 47 tags and are often language/script focused (for example: `en`, `und-Latn`, `zh-Hani`) rather than region-specific tags (for example: `en-US`, `zh-TW`).
+- In this project, locale values are BCP 47 tags and are often language/script focused (for example: `en`, `und-Latn`, `und-Hani`) rather than region-specific tags (for example: `en-US`, `zh-TW`).
 - Default detection prefers language/script tags to avoid incorrect region assumptions.
 - You can still provide region-specific locale tags through hint flags when needed.
 
@@ -245,6 +263,10 @@ import wordCounter, {
 
 wordCounter("Hello world", { latinLanguageHint: "en" });
 wordCounter("Hello world", { latinTagHint: "en" });
+wordCounter("Zażółć gęślą jaźń", {
+  latinHintRules: [{ tag: "pl", pattern: "[ąćęłńóśźżĄĆĘŁŃÓŚŹŻ]" }],
+});
+wordCounter("Über", { useDefaultLatinHints: false });
 wordCounter("漢字測試", { hanTagHint: "zh-Hant" });
 wordCounter("Hi 👋, world!", { nonWords: true });
 wordCounter("Hi 👋, world!", { mode: "char", nonWords: true });
@@ -295,6 +317,10 @@ const {
 
 wordCounter("Hello world", { latinLanguageHint: "en" });
 wordCounter("Hello world", { latinTagHint: "en" });
+wordCounter("Zażółć gęślą jaźń", {
+  latinHintRules: [{ tag: "pl", pattern: "[ąćęłńóśźżĄĆĘŁŃÓŚŹŻ]" }],
+});
+wordCounter("Über", { useDefaultLatinHints: false });
 wordCounter("漢字測試", { hanTagHint: "zh-Hant" });
 wordCounter("Hi 👋, world!", { nonWords: true });
 wordCounter("Hi 👋, world!", { mode: "char", nonWords: true });
@@ -563,13 +589,27 @@ Example JSON (trimmed):
 > [!Note]
 > Text-default symbols (e.g. ©) count as `symbols` unless explicitly emoji-presented (e.g. ©️ with VS16).
 
-## Locale Tag Detection Notes (Migration)
+## Locale Tag Detection Notes
 
+- Detection is regex/script based (Unicode script checks), not a statistical language-ID model.
 - Ambiguous Latin text uses `und-Latn` unless a Latin hint is provided.
-- Han-script fallback uses `zh-Hani` by default because regex script checks cannot natively distinguish `zh-Hans` vs `zh-Hant`.
+- Han-script fallback uses `und-Hani` by default because regex script checks cannot natively distinguish `zh-Hans` vs `zh-Hant`.
+- Current built-in Latin diacritic heuristics include:
+  - `de`: `äöüÄÖÜß`
+  - `es`: `ñÑ¿¡`
+  - `pt`: `ãõÃÕ`
+  - `fr`: `œŒæÆ`
+  - `pl`: `ąćęłńśźżĄĆĘŁŃŚŹŻ`
+  - `tr`: `ıİğĞşŞ`
+  - `ro`: `ăĂâÂîÎșȘțȚ`
+  - `hu`: `őŐűŰ`
+  - `is`: `ðÐþÞ`
+- Latin text with other European diacritics may still remain in `und-Latn` unless a hint is provided.
 - Use `--mode chunk`/`--mode segments` or `--format json` to see the exact locale tag assigned to each chunk.
 - Regex/script-only detection cannot reliably identify English vs. other Latin-script languages; 100% certainty requires explicit metadata (document language tags, user-provided locale, headers) or a language-ID model.
 - Use `--latin-language <tag>` or `--latin-tag <tag>` for ambiguous Latin text.
+- Use `--latin-hint <tag>=<pattern>` (repeatable) and `--latin-hints-file <path>` to add custom Latin rules.
+- Use `--no-default-latin-hints` to disable built-in Latin diacritic rules.
 - Use `--han-language <tag>` or `--han-tag <tag>` for Han-script fallback.
 - `--latin-locale` remains supported as a legacy alias for now and is planned for future deprecation.
 

package/dist/cjs/index.cjs

@@ -322,10 +322,53 @@ function resolveMode(input, fallback = "chunk") {
 	return normalizeMode(input) ?? fallback;
 }
 
+//#endregion
+//#region src/wc/latin-hints.ts
+const DEFAULT_LATIN_HINT_RULES_SOURCE = [
+	{
+		tag: "de",
+		pattern: "[äöüÄÖÜß]"
+	},
+	{
+		tag: "es",
+		pattern: "[ñÑ¿¡]"
+	},
+	{
+		tag: "pt",
+		pattern: "[ãõÃÕ]"
+	},
+	{
+		tag: "fr",
+		pattern: "[œŒæÆ]"
+	},
+	{
+		tag: "pl",
+		pattern: "[ąćęłńśźżĄĆĘŁŃŚŹŻ]"
+	},
+	{
+		tag: "tr",
+		pattern: "[ıİğĞşŞ]"
+	},
+	{
+		tag: "ro",
+		pattern: "[ăĂâÂîÎșȘțȚ]"
+	},
+	{
+		tag: "hu",
+		pattern: "[őŐűŰ]"
+	},
+	{
+		tag: "is",
+		pattern: "[ðÐþÞ]"
+	}
+];
+const DEFAULT_LATIN_HINT_RULES = Object.freeze(DEFAULT_LATIN_HINT_RULES_SOURCE.map((rule) => Object.freeze({ ...rule })));
+
 //#endregion
 //#region src/wc/locale-detect.ts
 const DEFAULT_LOCALE = "und-Latn";
-const DEFAULT_HAN_TAG = "zh-Hani";
+const DEFAULT_HAN_TAG = "und-Hani";
+const MAX_LATIN_HINT_PATTERN_LENGTH = 256;
 const regex = {
 	hiragana: /\p{Script=Hiragana}/u,
 	katakana: /\p{Script=Katakana}/u,
@@ -337,31 +380,10 @@ const regex = {
 	devanagari: /\p{Script=Devanagari}/u,
 	thai: /\p{Script=Thai}/u
 };
-const latinLocaleHints = [
-	{
-		locale: "de",
-		regex: /[äöüÄÖÜß]/
-	},
-	{
-		locale: "es",
-		regex: /[ñÑ¿¡]/
-	},
-	{
-		locale: "pt",
-		regex: /[ãõÃÕ]/
-	},
-	{
-		locale: "fr",
-		regex: /[œŒæÆ]/
-	}
-];
-const latinLocales = new Set([DEFAULT_LOCALE, ...latinLocaleHints.map((hint) => hint.locale)]);
-function isLatinLocale(locale) {
-	return latinLocales.has(locale);
-}
-function detectLatinLocale(char) {
-	for (const hint of latinLocaleHints) if (hint.regex.test(char)) return hint.locale;
-	return DEFAULT_LOCALE;
+const defaultLatinLocales = new Set([DEFAULT_LOCALE, ...DEFAULT_LATIN_HINT_RULES.map((hint) => hint.tag)]);
+function isLatinLocale(locale, context) {
+	if (context) return context.latinLocales.has(locale);
+	return defaultLatinLocales.has(locale);
 }
 function resolveLatinHint(options) {
 	const latinTagHint = options.latinTagHint?.trim();
@@ -377,7 +399,82 @@ function resolveHanHint(options) {
 	const hanLanguageHint = options.hanLanguageHint?.trim();
 	if (hanLanguageHint) return hanLanguageHint;
 }
-function detectLocaleForChar(char, previousLocale, options = {}) {
+function compileLatinHintPattern(pattern, label) {
+	const source = typeof pattern === "string" ? pattern : pattern.source;
+	const hasUnicodeMode = typeof pattern !== "string" && (pattern.flags.includes("u") || pattern.flags.includes("v"));
+	const flags = typeof pattern === "string" ? "u" : hasUnicodeMode ? pattern.flags : `${pattern.flags}u`;
+	if (source.length === 0) throw new Error(`${label}: pattern must not be empty.`);
+	if (source.length > MAX_LATIN_HINT_PATTERN_LENGTH) throw new Error(`${label}: pattern must be at most ${MAX_LATIN_HINT_PATTERN_LENGTH} characters.`);
+	try {
+		return new RegExp(source, flags);
+	} catch (error) {
+		const message = error instanceof Error ? error.message : String(error);
+		throw new Error(`${label}: invalid Unicode regex pattern (${message}).`);
+	}
+}
+function normalizeLatinHintPriority(priority, label) {
+	if (priority === void 0) return 0;
+	if (typeof priority !== "number" || !Number.isFinite(priority)) throw new Error(`${label}: priority must be a finite number when provided.`);
+	return priority;
+}
+function compileLatinHintRule(rule, order, label) {
+	const tag = typeof rule.tag === "string" ? rule.tag.trim() : "";
+	if (!tag) throw new Error(`${label}: tag must be a non-empty string.`);
+	return {
+		tag,
+		pattern: compileLatinHintPattern(rule.pattern, label),
+		priority: normalizeLatinHintPriority(rule.priority, label),
+		order
+	};
+}
+function resolveLatinHintRules(options) {
+	const useDefaultLatinHints = options.useDefaultLatinHints !== false;
+	const customRules = options.latinHintRules ?? [];
+	const combinedRules = [];
+	for (let index = 0; index < customRules.length; index += 1) {
+		const rule = customRules[index];
+		if (!rule) continue;
+		combinedRules.push({
+			rule,
+			label: `Invalid custom Latin hint rule at index ${index}`
+		});
+	}
+	if (useDefaultLatinHints) for (let index = 0; index < DEFAULT_LATIN_HINT_RULES.length; index += 1) {
+		const rule = DEFAULT_LATIN_HINT_RULES[index];
+		if (!rule) continue;
+		combinedRules.push({
+			rule,
+			label: `Invalid default Latin hint rule at index ${index}`
+		});
+	}
+	const resolvedRules = combinedRules.map((entry, index) => compileLatinHintRule(entry.rule, index, entry.label));
+	resolvedRules.sort((left, right) => {
+		if (left.priority !== right.priority) return right.priority - left.priority;
+		return left.order - right.order;
+	});
+	return resolvedRules;
+}
+function resolveLocaleDetectContext(options = {}) {
+	const latinHint = resolveLatinHint(options);
+	const latinHintRules = resolveLatinHintRules(options);
+	const latinLocales = new Set([DEFAULT_LOCALE]);
+	for (const rule of latinHintRules) latinLocales.add(rule.tag);
+	if (latinHint) latinLocales.add(latinHint);
+	return {
+		latinHint,
+		hanHint: resolveHanHint(options),
+		latinHintRules,
+		latinLocales
+	};
+}
+function detectLatinLocale(char, context) {
+	for (const hint of context.latinHintRules) {
+		hint.pattern.lastIndex = 0;
+		if (hint.pattern.test(char)) return hint.tag;
+	}
+	return DEFAULT_LOCALE;
+}
+function detectLocaleForChar(char, previousLocale, options = {}, context = resolveLocaleDetectContext(options)) {
 	if (regex.hiragana.test(char) || regex.katakana.test(char)) return "ja";
 	if (regex.hangul.test(char)) return "ko";
 	if (regex.arabic.test(char)) return "ar";
@@ -386,14 +483,13 @@ function detectLocaleForChar(char, previousLocale, options = {}) {
 	if (regex.thai.test(char)) return "th";
 	if (regex.han.test(char)) {
 		if (previousLocale && previousLocale.startsWith("ja")) return previousLocale;
-		return resolveHanHint(options) ?? DEFAULT_HAN_TAG;
+		return context.hanHint ?? DEFAULT_HAN_TAG;
 	}
 	if (regex.latin.test(char)) {
-		const hintedLocale = detectLatinLocale(char);
+		const hintedLocale = detectLatinLocale(char, context);
 		if (hintedLocale !== DEFAULT_LOCALE) return hintedLocale;
-		if (previousLocale && isLatinLocale(previousLocale) && previousLocale !== DEFAULT_LOCALE) return previousLocale;
-		const latinHint = resolveLatinHint(options);
-		if (latinHint) return latinHint;
+		if (previousLocale && isLatinLocale(previousLocale, context) && previousLocale !== DEFAULT_LOCALE) return previousLocale;
+		if (context.latinHint) return context.latinHint;
 		return DEFAULT_LOCALE;
 	}
 	return null;
@@ -402,12 +498,13 @@ function detectLocaleForChar(char, previousLocale, options = {}) {
 //#endregion
 //#region src/wc/segment.ts
 function segmentTextByLocale(text, options = {}) {
+	const context = resolveLocaleDetectContext(options);
 	const chunks = [];
 	let currentLocale = DEFAULT_LOCALE;
 	let buffer = "";
 	let bufferHasScript = false;
 	for (const char of text) {
-		const detected = detectLocaleForChar(char, currentLocale, options);
+		const detected = detectLocaleForChar(char, currentLocale, options, context);
 		const targetLocale = detected ?? currentLocale;
 		if (buffer === "") {
 			currentLocale = targetLocale;
@@ -422,7 +519,7 @@ function segmentTextByLocale(text, options = {}) {
 			continue;
 		}
 		if (targetLocale !== currentLocale && detected !== null) {
-			if (currentLocale === DEFAULT_LOCALE && isLatinLocale(targetLocale)) {
+			if (currentLocale === DEFAULT_LOCALE && isLatinLocale(targetLocale, context)) {
 				currentLocale = targetLocale;
 				buffer += char;
 				bufferHasScript = true;
@@ -475,6 +572,8 @@ function wordCounter(text, options = {}) {
 		latinLanguageHint: options.latinLanguageHint,
 		latinTagHint: options.latinTagHint,
 		latinLocaleHint: options.latinLocaleHint,
+		latinHintRules: options.latinHintRules,
+		useDefaultLatinHints: options.useDefaultLatinHints,
 		hanLanguageHint: options.hanLanguageHint,
 		hanTagHint: options.hanTagHint
 	});