@dev-pi2pie/word-counter 0.1.4 → 0.1.5-canary.2

package/README.md

@@ -37,6 +37,8 @@ For local development in this repository:
 ```bash
 git clone https://github.com/dev-pi2pie/word-counter.git
 cd word-counter
+rustup target add wasm32-unknown-unknown
+cargo install wasm-pack --locked
 bun install
 bun run build
 npm link
@@ -94,6 +96,22 @@ word-counter --han-language zh-Hant "漢字測試"
 word-counter --han-tag zh-Hans "汉字测试"
 ```
 
+Enable the optional WASM detector for ambiguous Latin and Han routes:
+
+```bash
+word-counter --detector wasm "This sentence should clearly be detected as English for the wasm detector path."
+word-counter --detector wasm "漢字測試需要更多內容才能觸發偵測"
+```
+
+Detector mode notes:
+
+- `--detector regex` is the default behavior.
+- `--detector wasm` only runs for ambiguous `und-Latn` and `und-Hani` chunks.
+- `--detector regex` keeps the original script/regex chunk-first detection path.
+- `--detector wasm` uses a detector-oriented ambiguous-window scoring pass before accepted tags are projected back onto the counting chunks.
+- Very short chunks stay on the original `und-*` fallback.
+- Low-confidence or unsupported detector results fall back to `und-*`.
+
 Collect non-words (emoji/symbols/punctuation):
 
 ```bash
@@ -164,6 +182,33 @@ word-counter --print-jobs-limit
 
 `--print-jobs-limit` must be used alone (no other inputs or runtime flags).
 
+### Doctor (`doctor`)
+
+Use `doctor` to verify whether the current host can run `word-counter` reliably:
+
+```bash
+word-counter doctor
+word-counter doctor --format json
+word-counter doctor --format json --pretty
+```
+
+Doctor scope in v1:
+
+- checks runtime support policy against Node.js `>=20`
+- verifies `Intl.Segmenter` availability plus word/grapheme constructor health
+- reports batch jobs host limits using the same heuristics as `--print-jobs-limit`
+- reports worker-route preflight signals and the worker-disable env toggle that affects worker availability
+
+Doctor output contract:
+
+- default output is human-readable text
+- `--format json` prints compact machine-readable JSON
+- `--format json --pretty` prints indented JSON
+- doctor exits with code `0` for `ok` / `warn`, `1` for invalid doctor usage, and `2` for runtime `fail`
+- doctor does not accept counting inputs, `--path`, `--jobs`, or other counting/debug flags
+
+For a field-by-field explanation of doctor text and JSON output, see [`docs/doctor-usage-guide.md`](docs/doctor-usage-guide.md).
+
 For full policy details, JSON parity expectations (`--misc`, `--total-of whitespace,words`), and benchmark standards, see [`docs/batch-jobs-usage-guide.md`](docs/batch-jobs-usage-guide.md).
 
 ### Stable Path Resolution Contract
@@ -266,6 +311,7 @@ Skip details stay debug-gated and can be suppressed with `--quiet-skips`.
 - Adjacent characters that share the same locale tag are grouped into a chunk.
 - Each chunk is counted with `Intl.Segmenter` at `granularity: "word"`, caching segmenters to avoid re-instantiation.
 - Per-locale counts are summed into an overall total and printed to stdout.
+- With `--detector wasm`, ambiguous `und-Latn` and `und-Hani` chunks can be relabeled through the optional WASM detector before counting.
 
 ## Locale vs Language Code
 
@@ -289,6 +335,10 @@ import wordCounter, {
   segmentTextByLocale,
   showSingularOrPluralWord,
 } from "@dev-pi2pie/word-counter";
+import {
+  wordCounterWithDetector,
+  segmentTextByLocaleWithDetector,
+} from "@dev-pi2pie/word-counter/detector";
 
 wordCounter("Hello world", { latinLanguageHint: "en" });
 wordCounter("Hello world", { latinTagHint: "en" });
@@ -302,6 +352,11 @@ wordCounter("Hi 👋, world!", { mode: "char", nonWords: true });
 wordCounter("飛鳥 bird 貓 cat", { mode: "char-collector" });
 wordCounter("Hi\tthere\n", { nonWords: true, includeWhitespace: true });
 countCharsForLocale("👋", "en");
+await wordCounterWithDetector(
+  "This sentence should clearly be detected as English for the wasm detector path.",
+  { detector: "wasm" },
+);
+await segmentTextByLocaleWithDetector("Hello 世界", { detector: "regex" });
 ```
 
 Note: `includeWhitespace` only affects results when `nonWords: true` is enabled.
@@ -335,6 +390,7 @@ Sample output (with `nonWords: true` and `includeWhitespace: true`):
 
 ```js
 const wordCounter = require("@dev-pi2pie/word-counter");
+const detector = require("@dev-pi2pie/word-counter/detector");
 const {
   countCharsForLocale,
   countWordsForLocale,
@@ -356,6 +412,10 @@ wordCounter("Hi 👋, world!", { mode: "char", nonWords: true });
 wordCounter("飛鳥 bird 貓 cat", { mode: "char-collector" });
 wordCounter("Hi\tthere\n", { nonWords: true, includeWhitespace: true });
 countCharsForLocale("👋", "en");
+await detector.wordCounterWithDetector(
+  "This sentence should clearly be detected as English for the wasm detector path.",
+  { detector: "wasm" },
+);
 ```
 
 Note: `includeWhitespace` only affects results when `nonWords: true` is enabled.
@@ -410,6 +470,18 @@ Sample output (with `nonWords: true` and `includeWhitespace: true`):
 | -------------------------- | -------- | ------------------------------ |
 | `showSingularOrPluralWord` | function | Formats singular/plural words. |
 
+#### Detector Subpath
+
+Import from `@dev-pi2pie/word-counter/detector` for the explicit detector-enabled API.
+
+| Export                        | Kind     | Notes                                           |
+| ----------------------------- | -------- | ----------------------------------------------- |
+| `wordCounterWithDetector`     | function | Async detector-aware counting entrypoint.       |
+| `segmentTextByLocaleWithDetector` | function | Async detector-aware locale segmentation.  |
+| `countSectionsWithDetector`   | function | Async detector-aware section counting.          |
+| `DEFAULT_DETECTOR_MODE`       | value    | Current default detector mode (`regex`).        |
+| `DETECTOR_MODES`              | value    | Supported detector modes.                       |
+
 #### Types
 
 | Export                 | Kind | Notes                                             |
@@ -623,6 +695,9 @@ Example JSON (trimmed):
 
 - Detection is regex/script based, not statistical language-ID.
 - Ambiguous Latin defaults to `und-Latn`; Han fallback defaults to `und-Hani`.
+- `--detector wasm` is optional and conservative; it only runs for ambiguous chunks that meet minimum script-bearing length thresholds.
+- The current first WASM engine is `whatlang`, remapped into this package's public tags.
+- The npm package ships one portable WASM artifact; users do not install per-OS detector packages.
 - Use explicit tag and hint flags when you need deterministic tagging.
 - Full notes (built-in heuristics, limitations, and override guidance) are tracked in `docs/locale-tag-detection-notes.md`.
 

package/dist/cjs/detector.cjs

@@ -0,0 +1,427 @@
+const require_markdown = require("./markdown.cjs");
+let node_fs = require("node:fs");
+let node_path = require("node:path");
+let node_module = require("node:module");
+let node_url = require("node:url");
+//#region src/detector/none.ts
+async function segmentTextByLocaleWithRegexDetector(text, options = {}) {
+	return require_markdown.segmentTextByLocale(text, options);
+}
+async function wordCounterWithRegexDetector(text, options = {}) {
+	return require_markdown.wc_default(text, options);
+}
+async function countSectionsWithRegexDetector(input, section, options = {}) {
+	return require_markdown.countSections(input, section, options);
+}
+//#endregion
+//#region src/detector/result-builder.ts
+function getNonWordTotal(nonWords) {
+	return nonWords.counts.emoji + nonWords.counts.symbols + nonWords.counts.punctuation + (nonWords.counts.whitespace ?? 0);
+}
+function collectNonWordsAggregate(analyzed, enabled) {
+	if (!enabled) return;
+	const collection = require_markdown.createNonWordCollection();
+	for (const chunk of analyzed) {
+		if (!chunk.nonWords) continue;
+		require_markdown.mergeNonWordCollections(collection, chunk.nonWords);
+	}
+	return collection;
+}
+function buildWordCounterResultFromChunks(chunks, options = {}) {
+	const mode = require_markdown.resolveMode(options.mode, "chunk");
+	const collectNonWords = Boolean(options.nonWords);
+	const includeWhitespace = Boolean(options.includeWhitespace);
+	if (mode === "char" || mode === "char-collector") {
+		const analyzed = chunks.map((chunk) => require_markdown.analyzeCharChunk(chunk, collectNonWords, includeWhitespace));
+		const total = analyzed.reduce((sum, chunk) => sum + chunk.chars, 0);
+		const counts = collectNonWords ? {
+			words: analyzed.reduce((sum, chunk) => sum + chunk.wordChars, 0),
+			nonWords: analyzed.reduce((sum, chunk) => sum + chunk.nonWordChars, 0),
+			total
+		} : void 0;
+		if (mode === "char") return {
+			total,
+			counts,
+			breakdown: {
+				mode,
+				items: analyzed.map((chunk) => ({
+					locale: chunk.locale,
+					text: chunk.text,
+					chars: chunk.chars,
+					nonWords: chunk.nonWords
+				}))
+			}
+		};
+		return {
+			total,
+			counts,
+			breakdown: {
+				mode,
+				items: require_markdown.aggregateCharsByLocale(analyzed).map((chunk) => ({
+					locale: chunk.locale,
+					chars: chunk.chars,
+					nonWords: chunk.nonWords
+				}))
+			}
+		};
+	}
+	const analyzed = chunks.map((chunk) => require_markdown.analyzeChunk(chunk, collectNonWords, includeWhitespace));
+	const wordsTotal = analyzed.reduce((sum, chunk) => sum + chunk.words, 0);
+	const nonWordsTotal = collectNonWords ? analyzed.reduce((sum, chunk) => {
+		if (!chunk.nonWords) return sum;
+		return sum + getNonWordTotal(chunk.nonWords);
+	}, 0) : 0;
+	const total = analyzed.reduce((sum, chunk) => {
+		let chunkTotal = chunk.words;
+		if (collectNonWords && chunk.nonWords) chunkTotal += getNonWordTotal(chunk.nonWords);
+		return sum + chunkTotal;
+	}, 0);
+	const counts = collectNonWords ? {
+		words: wordsTotal,
+		nonWords: nonWordsTotal,
+		total
+	} : void 0;
+	if (mode === "segments") return {
+		total,
+		counts,
+		breakdown: {
+			mode,
+			items: analyzed.map((chunk) => ({
+				locale: chunk.locale,
+				text: chunk.text,
+				words: chunk.words,
+				segments: chunk.segments,
+				nonWords: chunk.nonWords
+			}))
+		}
+	};
+	if (mode === "collector") return {
+		total,
+		counts,
+		breakdown: {
+			mode,
+			items: require_markdown.aggregateByLocale(analyzed),
+			nonWords: collectNonWordsAggregate(analyzed, collectNonWords)
+		}
+	};
+	return {
+		total,
+		counts,
+		breakdown: {
+			mode,
+			items: analyzed.map((chunk) => ({
+				locale: chunk.locale,
+				text: chunk.text,
+				words: chunk.words,
+				nonWords: chunk.nonWords
+			}))
+		}
+	};
+}
+//#endregion
+//#region src/detector/sections.ts
+function normalizeText(value) {
+	if (value == null) return "";
+	if (typeof value === "string") return value;
+	if (typeof value === "number" || typeof value === "boolean") return String(value);
+	try {
+		return JSON.stringify(value);
+	} catch {
+		return String(value);
+	}
+}
+async function buildPerKeyItems(data, options) {
+	if (!data || typeof data !== "object" || Array.isArray(data)) return [];
+	return Promise.all(Object.entries(data).map(async ([key, value]) => {
+		const valueText = normalizeText(value);
+		return {
+			name: key,
+			source: "frontmatter",
+			result: await wordCounterWithDetector(valueText ? `${key}: ${valueText}` : key, options)
+		};
+	}));
+}
+async function buildSingleItem(name, text, options, source) {
+	return [{
+		name,
+		source,
+		result: await wordCounterWithDetector(text, options)
+	}];
+}
+function sumTotals(items) {
+	return items.reduce((sum, item) => sum + item.result.total, 0);
+}
+async function countSectionsWithResolvedDetector(input, section, options = {}) {
+	options.mode;
+	if (section === "all") {
+		const result = await wordCounterWithDetector(input, options);
+		return {
+			section,
+			total: result.total,
+			frontmatterType: null,
+			items: [{
+				name: "all",
+				source: "content",
+				result
+			}]
+		};
+	}
+	const parsed = require_markdown.parseMarkdown(input);
+	const frontmatterText = parsed.frontmatter ?? "";
+	const contentText = parsed.content ?? "";
+	let items = [];
+	if (section === "frontmatter") items = await buildSingleItem("frontmatter", frontmatterText, options, "frontmatter");
+	else if (section === "content") items = await buildSingleItem("content", contentText, options, "content");
+	else if (section === "split") items = [...await buildSingleItem("frontmatter", frontmatterText, options, "frontmatter"), ...await buildSingleItem("content", contentText, options, "content")];
+	else if (section === "per-key") items = await buildPerKeyItems(parsed.data, options);
+	else if (section === "split-per-key") items = [...await buildPerKeyItems(parsed.data, options), ...await buildSingleItem("content", contentText, options, "content")];
+	return {
+		section,
+		total: sumTotals(items),
+		frontmatterType: parsed.frontmatterType,
+		items
+	};
+}
+const LATIN_WASM_MIN_CONFIDENCE = .75;
+const HANI_WASM_MIN_CONFIDENCE = .9;
+const LATIN_SCRIPT_REGEX = /\p{Script=Latin}/u;
+const HAN_SCRIPT_REGEX = /\p{Script=Han}/u;
+const DETECTOR_ROUTE_POLICIES = {
+	[require_markdown.DEFAULT_LOCALE]: {
+		routeTag: require_markdown.DEFAULT_LOCALE,
+		minScriptChars: 24,
+		minConfidence: LATIN_WASM_MIN_CONFIDENCE,
+		requireReliable: true
+	},
+	[require_markdown.DEFAULT_HAN_TAG]: {
+		routeTag: require_markdown.DEFAULT_HAN_TAG,
+		minScriptChars: 12,
+		minConfidence: HANI_WASM_MIN_CONFIDENCE,
+		requireReliable: true
+	}
+};
+function isAmbiguousDetectorRoute(locale) {
+	return locale === "und-Latn" || locale === "und-Hani";
+}
+function countScriptBearingCharsForRoute(text, routeTag) {
+	const matcher = routeTag === "und-Hani" ? HAN_SCRIPT_REGEX : LATIN_SCRIPT_REGEX;
+	let count = 0;
+	for (const char of text) if (matcher.test(char)) count += 1;
+	return count;
+}
+function shouldRunWasmDetector(text, routeTag) {
+	const policy = DETECTOR_ROUTE_POLICIES[routeTag];
+	return countScriptBearingCharsForRoute(text, routeTag) >= policy.minScriptChars;
+}
+function normalizeDetectorSampleForRoute(text, routeTag) {
+	const matcher = routeTag === "und-Hani" ? HAN_SCRIPT_REGEX : LATIN_SCRIPT_REGEX;
+	return [...text].map((char) => {
+		if (matcher.test(char)) return char;
+		if (/\s/u.test(char)) return " ";
+		return " ";
+	}).join("").replace(/\s+/g, " ").trim();
+}
+//#endregion
+//#region src/detector/whatlang-wasm.ts
+const GENERATED_FOLDER_NAME = "wasm-language-detector";
+const GENERATED_MODULE_FILE = "language_detector.js";
+const MAX_SEARCH_DEPTH = 8;
+const requireFromHere = (0, node_module.createRequire)(require("url").pathToFileURL(__filename).href);
+const WASM_DETECTOR_RUNTIME_UNAVAILABLE_MESSAGE = "WASM detector runtime is unavailable. Run `bun run build:wasm` to generate it.";
+let modulePromise = null;
+function resolveCandidateModulePaths() {
+	const moduleDir = (0, node_path.dirname)((0, node_url.fileURLToPath)(require("url").pathToFileURL(__filename).href));
+	const candidates = /* @__PURE__ */ new Set();
+	let currentDir = moduleDir;
+	for (let depth = 0; depth < MAX_SEARCH_DEPTH; depth += 1) {
+		candidates.add((0, node_path.join)(currentDir, GENERATED_FOLDER_NAME, GENERATED_MODULE_FILE));
+		candidates.add((0, node_path.join)(currentDir, "generated", GENERATED_FOLDER_NAME, GENERATED_MODULE_FILE));
+		const parentDir = (0, node_path.dirname)(currentDir);
+		if (parentDir === currentDir) break;
+		currentDir = parentDir;
+	}
+	return [...candidates];
+}
+function resolveWhatlangWasmModulePath() {
+	for (const candidate of resolveCandidateModulePaths()) if ((0, node_fs.existsSync)(candidate)) return candidate;
+	throw new Error(WASM_DETECTOR_RUNTIME_UNAVAILABLE_MESSAGE);
+}
+async function loadWhatlangWasmModule() {
+	if (!modulePromise) modulePromise = (async () => {
+		return requireFromHere(resolveWhatlangWasmModulePath());
+	})();
+	return modulePromise;
+}
+async function detectWithWhatlangWasm(text, routeTag) {
+	return (await loadWhatlangWasmModule()).detect_language(text, routeTag);
+}
+//#endregion
+//#region src/detector/whatlang-map.ts
+const LATIN_LANGUAGE_TAGS = {
+	cat: "ca",
+	ces: "cs",
+	dan: "da",
+	deu: "de",
+	eng: "en",
+	fin: "fi",
+	fra: "fr",
+	hun: "hu",
+	ita: "it",
+	lat: "la",
+	nld: "nl",
+	pol: "pl",
+	por: "pt",
+	ron: "ro",
+	spa: "es",
+	swe: "sv",
+	tur: "tr"
+};
+const HANI_LANGUAGE_TAGS = {
+	cmn: "zh",
+	jpn: "ja"
+};
+function hasSupportedScript(result, routeTag) {
+	if (routeTag === "und-Latn") return result.script === "Latin";
+	return result.script === "Mandarin";
+}
+function remapLanguageTag(lang, routeTag) {
+	if (routeTag === "und-Latn") return LATIN_LANGUAGE_TAGS[lang];
+	return HANI_LANGUAGE_TAGS[lang];
+}
+function remapWhatlangResult(result, routeTag) {
+	if (!hasSupportedScript(result, routeTag)) return null;
+	const tag = remapLanguageTag(result.lang, routeTag);
+	if (!tag) return null;
+	return {
+		tag,
+		confidence: result.confidence,
+		reliable: result.reliable,
+		source: "wasm"
+	};
+}
+function getDetectorFallbackTag(routeTag) {
+	return routeTag === "und-Hani" ? require_markdown.DEFAULT_HAN_TAG : require_markdown.DEFAULT_LOCALE;
+}
+//#endregion
+//#region src/detector/wasm.ts
+function shouldAcceptDetectorTag(routeTag, confidence, reliable) {
+	const policy = DETECTOR_ROUTE_POLICIES[routeTag];
+	if (policy.requireReliable && reliable !== true) return false;
+	if (confidence === void 0) return false;
+	return confidence >= policy.minConfidence;
+}
+function buildDetectorWindows(chunks) {
+	const windows = [];
+	for (let index = 0; index < chunks.length; index += 1) {
+		const chunk = chunks[index];
+		if (!chunk || !isAmbiguousDetectorRoute(chunk.locale)) continue;
+		const previousWindow = windows[windows.length - 1];
+		if (previousWindow && previousWindow.routeTag === chunk.locale && previousWindow.endIndex === index - 1) {
+			previousWindow.endIndex = index;
+			previousWindow.text += chunk.text;
+			continue;
+		}
+		windows.push({
+			routeTag: chunk.locale,
+			startIndex: index,
+			endIndex: index,
+			text: chunk.text
+		});
+	}
+	return windows;
+}
+async function resolveWindowLocale(window) {
+	if (!shouldRunWasmDetector(window.text, window.routeTag)) return window.routeTag;
+	const rawResult = await detectWithWhatlangWasm(window.text, window.routeTag);
+	const rawRemapped = rawResult ? remapWhatlangResult(rawResult, window.routeTag) : null;
+	const normalizedSample = normalizeDetectorSampleForRoute(window.text, window.routeTag);
+	const normalizedResult = normalizedSample.length > 0 && normalizedSample !== window.text ? await detectWithWhatlangWasm(normalizedSample, window.routeTag) : null;
+	const normalizedRemapped = normalizedResult ? remapWhatlangResult(normalizedResult, window.routeTag) : null;
+	const candidates = [rawRemapped, normalizedRemapped].filter((value) => value !== null);
+	if (candidates.length === 0) return getDetectorFallbackTag(window.routeTag);
+	const strongestCandidate = candidates.reduce((best, current) => {
+		if (!best) return current;
+		return (current.confidence ?? 0) > (best.confidence ?? 0) ? current : best;
+	}, candidates[0]);
+	if (strongestCandidate && shouldAcceptDetectorTag(window.routeTag, strongestCandidate.confidence, strongestCandidate.reliable)) return strongestCandidate.tag;
+	if (window.routeTag === "und-Latn" && rawRemapped && normalizedRemapped && rawRemapped.tag === normalizedRemapped.tag) {
+		if (Math.max(rawRemapped.confidence ?? 0, normalizedRemapped.confidence ?? 0) >= .7) return rawRemapped.tag;
+	}
+	return getDetectorFallbackTag(window.routeTag);
+}
+async function segmentTextByLocaleWithWasmDetector(text, options = {}) {
+	const chunks = require_markdown.segmentTextByLocale(text, options);
+	const resolved = [...chunks];
+	const windows = buildDetectorWindows(chunks);
+	for (const window of windows) {
+		const resolvedLocale = await resolveWindowLocale(window);
+		for (let index = window.startIndex; index <= window.endIndex; index += 1) {
+			const chunk = resolved[index];
+			if (!chunk) continue;
+			resolved[index] = {
+				...chunk,
+				locale: resolvedLocale
+			};
+		}
+	}
+	return resolved;
+}
+async function wordCounterWithWasmDetector(text, options = {}) {
+	return buildWordCounterResultFromChunks(await segmentTextByLocaleWithWasmDetector(text, options), options);
+}
+async function countSectionsWithWasmDetector(input, section, options = {}) {
+	return countSectionsWithResolvedDetector(input, section, options);
+}
+//#endregion
+//#region src/detector/index.ts
+const DETECTOR_MODES = ["regex", "wasm"];
+const DEFAULT_DETECTOR_MODE = "regex";
+function resolveDetectorMode(mode) {
+	return mode ?? "regex";
+}
+function assertDetectorModeImplemented(mode) {}
+async function segmentTextByLocaleWithDetector(text, options = {}) {
+	if (resolveDetectorMode(options.detector) === "wasm") return segmentTextByLocaleWithWasmDetector(text, options);
+	return segmentTextByLocaleWithRegexDetector(text, options);
+}
+async function wordCounterWithDetector(text, options = {}) {
+	if (resolveDetectorMode(options.detector) === "wasm") return wordCounterWithWasmDetector(text, options);
+	return wordCounterWithRegexDetector(text, options);
+}
+async function countSectionsWithDetector(input, section, options = {}) {
+	if (resolveDetectorMode(options.detector) === "wasm") return countSectionsWithWasmDetector(input, section, options);
+	return countSectionsWithRegexDetector(input, section, options);
+}
+const DETECTOR_SOURCES = [
+	"script",
+	"hint",
+	"wasm"
+];
+const DEFAULT_DETECTOR_RESULT_SOURCE = "script";
+function createDetectorResult(tag, source = DEFAULT_DETECTOR_RESULT_SOURCE, confidence, reliable) {
+	return {
+		tag,
+		source,
+		...confidence === void 0 ? {} : { confidence },
+		...reliable === void 0 ? {} : { reliable }
+	};
+}
+//#endregion
+//#region src/detector/index.cjs.ts
+const cjsExports = {
+	assertDetectorModeImplemented,
+	countSectionsWithDetector,
+	createDetectorResult,
+	DEFAULT_DETECTOR_MODE,
+	DEFAULT_DETECTOR_RESULT_SOURCE,
+	DETECTOR_MODES,
+	DETECTOR_SOURCES,
+	resolveDetectorMode,
+	segmentTextByLocaleWithDetector,
+	WASM_DETECTOR_RUNTIME_UNAVAILABLE_MESSAGE,
+	wordCounterWithDetector
+};
+module.exports = cjsExports;
+//#endregion
+
+//# sourceMappingURL=detector.cjs.map

package/dist/cjs/detector.cjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"detector.cjs","names":["segmentTextByLocale","wordCounter","countSections","createNonWordCollection","resolveMode","analyzeCharChunk","aggregateCharsByLocale","analyzeChunk","aggregateByLocale","parseMarkdown","DEFAULT_LOCALE","DEFAULT_HAN_TAG","DEFAULT_HAN_TAG","DEFAULT_LOCALE","segmentTextByLocale"],"sources":["../../src/detector/none.ts","../../src/detector/result-builder.ts","../../src/detector/sections.ts","../../src/detector/policy.ts","../../src/detector/whatlang-wasm.ts","../../src/detector/whatlang-map.ts","../../src/detector/wasm.ts","../../src/detector/index.ts","../../src/detector/index.cjs.ts"],"sourcesContent":["import { countSections } from \"../markdown\";\nimport wordCounter, { segmentTextByLocale } from \"../wc\";\nimport type {\n  DetectorCountSectionsOptions,\n  DetectorLocaleOptions,\n  DetectorWordCounterOptions,\n} from \"./types\";\n\nexport async function segmentTextByLocaleWithRegexDetector(\n  text: string,\n  options: DetectorLocaleOptions = {},\n) {\n  return segmentTextByLocale(text, options);\n}\n\nexport async function wordCounterWithRegexDetector(\n  text: string,\n  options: DetectorWordCounterOptions = {},\n) {\n  return wordCounter(text, options);\n}\n\nexport async function countSectionsWithRegexDetector(\n  input: string,\n  section: Parameters<typeof countSections>[1],\n  options: DetectorCountSectionsOptions = {},\n) {\n  return countSections(input, section, options);\n}\n","import {\n  analyzeCharChunk,\n  analyzeChunk,\n  aggregateByLocale,\n  aggregateCharsByLocale,\n} from \"../wc/analyze\";\nimport { resolveMode } from \"../wc/mode\";\nimport { createNonWordCollection, mergeNonWordCollections } from \"../wc/non-words\";\nimport type {\n  CharBreakdown,\n  CharCollectorBreakdown,\n  ChunkBreakdown,\n  ChunkWithSegments,\n  LocaleChunk,\n  NonWordCollection,\n  WordCounterMode,\n  WordCounterOptions,\n  WordCounterResult,\n} from \"../wc/types\";\n\nfunction getNonWordTotal(nonWords: NonWordCollection): number {\n  return (\n    nonWords.counts.emoji +\n    nonWords.counts.symbols +\n    nonWords.counts.punctuation +\n    (nonWords.counts.whitespace ?? 0)\n  );\n}\n\nfunction collectNonWordsAggregate(\n  analyzed: Array<{ nonWords?: NonWordCollection }>,\n  enabled: boolean,\n): NonWordCollection | undefined {\n  if (!enabled) {\n    return undefined;\n  }\n  const collection = createNonWordCollection();\n  for (const chunk of analyzed) {\n    if (!chunk.nonWords) {\n      continue;\n    }\n    mergeNonWordCollections(collection, chunk.nonWords);\n  }\n  return collection;\n}\n\nexport function buildWordCounterResultFromChunks(\n  chunks: LocaleChunk[],\n  options: WordCounterOptions = {},\n): WordCounterResult {\n  const mode: WordCounterMode = resolveMode(options.mode, \"chunk\");\n  const collectNonWords = Boolean(options.nonWords);\n  const includeWhitespace = Boolean(options.includeWhitespace);\n\n  if (mode === \"char\" || mode === \"char-collector\") {\n    const analyzed = chunks.map((chunk) =>\n      analyzeCharChunk(chunk, collectNonWords, includeWhitespace),\n    );\n    const total = analyzed.reduce((sum, chunk) => sum + chunk.chars, 0);\n    const counts = collectNonWords\n      ? {\n          words: analyzed.reduce((sum, chunk) => sum + chunk.wordChars, 0),\n          nonWords: analyzed.reduce((sum, chunk) => sum + chunk.nonWordChars, 0),\n          total,\n        }\n      : undefined;\n\n    if (mode === \"char\") {\n      const items: CharBreakdown[] = analyzed.map((chunk) => ({\n        locale: chunk.locale,\n        text: chunk.text,\n        chars: chunk.chars,\n        nonWords: chunk.nonWords,\n      }));\n      return {\n        total,\n        counts,\n        breakdown: {\n          mode,\n          items,\n        },\n      };\n    }\n\n    const aggregated = aggregateCharsByLocale(analyzed);\n    const items: CharCollectorBreakdown[] = aggregated.map((chunk) => ({\n      locale: chunk.locale,\n      chars: chunk.chars,\n      nonWords: chunk.nonWords,\n    }));\n    return {\n      total,\n      counts,\n      breakdown: {\n        mode,\n        items,\n      },\n    };\n  }\n\n  const analyzed = chunks.map((chunk) =>\n    analyzeChunk(chunk, collectNonWords, includeWhitespace),\n  );\n  const wordsTotal = analyzed.reduce((sum, chunk) => sum + chunk.words, 0);\n  const nonWordsTotal = collectNonWords\n    ? analyzed.reduce((sum, chunk) => {\n        if (!chunk.nonWords) {\n          return sum;\n        }\n        return sum + getNonWordTotal(chunk.nonWords);\n      }, 0)\n    : 0;\n  const total = analyzed.reduce((sum, chunk) => {\n    let chunkTotal = chunk.words;\n    if (collectNonWords && chunk.nonWords) {\n      chunkTotal += getNonWordTotal(chunk.nonWords);\n    }\n    return sum + chunkTotal;\n  }, 0);\n\n  const counts = collectNonWords ? { words: wordsTotal, nonWords: nonWordsTotal, total } : undefined;\n\n  if (mode === \"segments\") {\n    const items: ChunkWithSegments[] = analyzed.map((chunk) => ({\n      locale: chunk.locale,\n      text: chunk.text,\n      words: chunk.words,\n      segments: chunk.segments,\n      nonWords: chunk.nonWords,\n    }));\n    return {\n      total,\n      counts,\n      breakdown: {\n        mode,\n        items,\n      },\n    };\n  }\n\n  if (mode === \"collector\") {\n    const items = aggregateByLocale(analyzed);\n    const nonWords = collectNonWordsAggregate(analyzed, collectNonWords);\n    return {\n      total,\n      counts,\n      breakdown: {\n        mode,\n        items,\n        nonWords,\n      },\n    };\n  }\n\n  const items: ChunkBreakdown[] = analyzed.map((chunk) => ({\n    locale: chunk.locale,\n    text: chunk.text,\n    words: chunk.words,\n    nonWords: chunk.nonWords,\n  }));\n\n  return {\n    total,\n    counts,\n    breakdown: {\n      mode,\n      items,\n    },\n  };\n}\n","import { parseMarkdown } from \"../markdown\";\nimport type { SectionMode, SectionedResult } from \"../markdown\";\nimport type { WordCounterMode, WordCounterResult } from \"../wc/types\";\nimport type { DetectorCountSectionsOptions } from \"./types\";\nimport { wordCounterWithDetector } from \"./index\";\n\nfunction normalizeText(value: unknown): string {\n  if (value == null) {\n    return \"\";\n  }\n  if (typeof value === \"string\") {\n    return value;\n  }\n  if (typeof value === \"number\" || typeof value === \"boolean\") {\n    return String(value);\n  }\n  try {\n    return JSON.stringify(value);\n  } catch {\n    return String(value);\n  }\n}\n\nasync function buildPerKeyItems(\n  data: Record<string, unknown> | null,\n  options: DetectorCountSectionsOptions,\n): Promise<Array<{ name: string; source: \"frontmatter\"; result: WordCounterResult }>> {\n  if (!data || typeof data !== \"object\" || Array.isArray(data)) {\n    return [];\n  }\n\n  return Promise.all(\n    Object.entries(data).map(async ([key, value]) => {\n      const valueText = normalizeText(value);\n      const text = valueText ? `${key}: ${valueText}` : key;\n      return {\n        name: key,\n        source: \"frontmatter\" as const,\n        result: await wordCounterWithDetector(text, options),\n      };\n    }),\n  );\n}\n\nasync function buildSingleItem(\n  name: string,\n  text: string,\n  options: DetectorCountSectionsOptions,\n  source: \"frontmatter\" | \"content\",\n): Promise<Array<{ name: string; source: \"frontmatter\" | \"content\"; result: WordCounterResult }>> {\n  return [{ name, source, result: await wordCounterWithDetector(text, options) }];\n}\n\nfunction sumTotals(items: Array<{ result: WordCounterResult }>): number {\n  return items.reduce((sum, item) => sum + item.result.total, 0);\n}\n\nexport async function countSectionsWithResolvedDetector(\n  input: string,\n  section: SectionMode,\n  options: DetectorCountSectionsOptions = {},\n): Promise<SectionedResult> {\n  const mode: WordCounterMode = options.mode ?? \"chunk\";\n  if (section === \"all\") {\n    const result = await wordCounterWithDetector(input, options);\n    return {\n      section,\n      total: result.total,\n      frontmatterType: null,\n      items: [{ name: \"all\", source: \"content\", result }],\n    };\n  }\n\n  const parsed = parseMarkdown(input);\n  const frontmatterText = parsed.frontmatter ?? \"\";\n  const contentText = parsed.content ?? \"\";\n\n  let items: Array<{ name: string; source: \"frontmatter\" | \"content\"; result: WordCounterResult }> = [];\n\n  if (section === \"frontmatter\") {\n    items = await buildSingleItem(\"frontmatter\", frontmatterText, options, \"frontmatter\");\n  } else if (section === \"content\") {\n    items = await buildSingleItem(\"content\", contentText, options, \"content\");\n  } else if (section === \"split\") {\n    items = [\n      ...(await buildSingleItem(\"frontmatter\", frontmatterText, options, \"frontmatter\")),\n      ...(await buildSingleItem(\"content\", contentText, options, \"content\")),\n    ];\n  } else if (section === \"per-key\") {\n    items = await buildPerKeyItems(parsed.data, options);\n  } else if (section === \"split-per-key\") {\n    items = [\n      ...(await buildPerKeyItems(parsed.data, options)),\n      ...(await buildSingleItem(\"content\", contentText, options, \"content\")),\n    ];\n  }\n\n  return {\n    section,\n    total: sumTotals(items),\n    frontmatterType: parsed.frontmatterType,\n    items,\n  };\n}\n","import { DEFAULT_HAN_TAG, DEFAULT_LOCALE } from \"../wc/locale-detect\";\n\nexport const LATIN_WASM_MIN_SCRIPT_CHARS = 24;\nexport const HANI_WASM_MIN_SCRIPT_CHARS = 12;\nexport const LATIN_WASM_MIN_CONFIDENCE = 0.75;\nexport const HANI_WASM_MIN_CONFIDENCE = 0.9;\nexport const LATIN_WASM_CORROBORATED_MIN_CONFIDENCE = 0.7;\n\nconst LATIN_SCRIPT_REGEX = /\\p{Script=Latin}/u;\nconst HAN_SCRIPT_REGEX = /\\p{Script=Han}/u;\n\nexport type DetectorRouteTag = typeof DEFAULT_LOCALE | typeof DEFAULT_HAN_TAG;\n\nexport type DetectorRoutePolicy = {\n  routeTag: DetectorRouteTag;\n  minScriptChars: number;\n  minConfidence: number;\n  requireReliable: boolean;\n};\n\nexport const DETECTOR_ROUTE_POLICIES: Record<DetectorRouteTag, DetectorRoutePolicy> = {\n  [DEFAULT_LOCALE]: {\n    routeTag: DEFAULT_LOCALE,\n    minScriptChars: LATIN_WASM_MIN_SCRIPT_CHARS,\n    minConfidence: LATIN_WASM_MIN_CONFIDENCE,\n    requireReliable: true,\n  },\n  [DEFAULT_HAN_TAG]: {\n    routeTag: DEFAULT_HAN_TAG,\n    minScriptChars: HANI_WASM_MIN_SCRIPT_CHARS,\n    minConfidence: HANI_WASM_MIN_CONFIDENCE,\n    requireReliable: true,\n  },\n};\n\nexport function isAmbiguousDetectorRoute(locale: string): locale is DetectorRouteTag {\n  return locale === DEFAULT_LOCALE || locale === DEFAULT_HAN_TAG;\n}\n\nexport function countScriptBearingCharsForRoute(\n  text: string,\n  routeTag: DetectorRouteTag,\n): number {\n  const matcher = routeTag === DEFAULT_HAN_TAG ? HAN_SCRIPT_REGEX : LATIN_SCRIPT_REGEX;\n  let count = 0;\n  for (const char of text) {\n    if (matcher.test(char)) {\n      count += 1;\n    }\n  }\n  return count;\n}\n\nexport function shouldRunWasmDetector(text: string, routeTag: DetectorRouteTag): boolean {\n  const policy = DETECTOR_ROUTE_POLICIES[routeTag];\n  return countScriptBearingCharsForRoute(text, routeTag) >= policy.minScriptChars;\n}\n\nexport function normalizeDetectorSampleForRoute(\n  text: string,\n  routeTag: DetectorRouteTag,\n): string {\n  const matcher = routeTag === DEFAULT_HAN_TAG ? HAN_SCRIPT_REGEX : LATIN_SCRIPT_REGEX;\n  return [...text]\n    .map((char) => {\n      if (matcher.test(char)) {\n        return char;\n      }\n      if (/\\s/u.test(char)) {\n        return \" \";\n      }\n      return \" \";\n    })\n    .join(\"\")\n    .replace(/\\s+/g, \" \")\n    .trim();\n}\n","import { existsSync } from \"node:fs\";\nimport { dirname, join } from \"node:path\";\nimport { createRequire } from \"node:module\";\nimport { fileURLToPath } from \"node:url\";\nimport type { DetectorRouteTag } from \"./policy\";\nimport type { WhatlangWasmResult } from \"./whatlang-map\";\n\nconst GENERATED_FOLDER_NAME = \"wasm-language-detector\";\nconst GENERATED_MODULE_FILE = \"language_detector.js\";\nconst MAX_SEARCH_DEPTH = 8;\nconst requireFromHere = createRequire(import.meta.url);\n\nexport const WASM_DETECTOR_RUNTIME_UNAVAILABLE_MESSAGE =\n  \"WASM detector runtime is unavailable. Run `bun run build:wasm` to generate it.\";\n\ntype WhatlangWasmModule = {\n  detect_language: (text: string, routeTag: string) => WhatlangWasmResult | null;\n};\n\nlet modulePromise: Promise<WhatlangWasmModule> | null = null;\n\nfunction resolveCandidateModulePaths(): string[] {\n  const moduleDir = dirname(fileURLToPath(import.meta.url));\n  const candidates = new Set<string>();\n  let currentDir = moduleDir;\n\n  for (let depth = 0; depth < MAX_SEARCH_DEPTH; depth += 1) {\n    candidates.add(join(currentDir, GENERATED_FOLDER_NAME, GENERATED_MODULE_FILE));\n    candidates.add(join(currentDir, \"generated\", GENERATED_FOLDER_NAME, GENERATED_MODULE_FILE));\n\n    const parentDir = dirname(currentDir);\n    if (parentDir === currentDir) {\n      break;\n    }\n    currentDir = parentDir;\n  }\n\n  return [...candidates];\n}\n\nfunction resolveWhatlangWasmModulePath(): string {\n  for (const candidate of resolveCandidateModulePaths()) {\n    if (existsSync(candidate)) {\n      return candidate;\n    }\n  }\n\n  throw new Error(WASM_DETECTOR_RUNTIME_UNAVAILABLE_MESSAGE);\n}\n\nasync function loadWhatlangWasmModule(): Promise<WhatlangWasmModule> {\n  if (!modulePromise) {\n    modulePromise = (async () => {\n      const modulePath = resolveWhatlangWasmModulePath();\n      return requireFromHere(modulePath) as WhatlangWasmModule;\n    })();\n  }\n\n  return modulePromise;\n}\n\nexport async function detectWithWhatlangWasm(\n  text: string,\n  routeTag: DetectorRouteTag,\n): Promise<WhatlangWasmResult | null> {\n  const wasmModule = await loadWhatlangWasmModule();\n  return wasmModule.detect_language(text, routeTag);\n}\n","import { DEFAULT_HAN_TAG, DEFAULT_LOCALE } from \"../wc/locale-detect\";\nimport type { DetectorRouteTag } from \"./policy\";\nimport type { DetectorResult } from \"./types\";\n\nexport interface WhatlangWasmResult {\n  lang: string;\n  script: string;\n  confidence: number;\n  reliable: boolean;\n}\n\nconst LATIN_LANGUAGE_TAGS: Record<string, string> = {\n  cat: \"ca\",\n  ces: \"cs\",\n  dan: \"da\",\n  deu: \"de\",\n  eng: \"en\",\n  fin: \"fi\",\n  fra: \"fr\",\n  hun: \"hu\",\n  ita: \"it\",\n  lat: \"la\",\n  nld: \"nl\",\n  pol: \"pl\",\n  por: \"pt\",\n  ron: \"ro\",\n  spa: \"es\",\n  swe: \"sv\",\n  tur: \"tr\",\n};\n\nconst HANI_LANGUAGE_TAGS: Record<string, string> = {\n  cmn: \"zh\",\n  jpn: \"ja\",\n};\n\nfunction hasSupportedScript(result: WhatlangWasmResult, routeTag: DetectorRouteTag): boolean {\n  if (routeTag === DEFAULT_LOCALE) {\n    return result.script === \"Latin\";\n  }\n\n  return result.script === \"Mandarin\";\n}\n\nfunction remapLanguageTag(\n  lang: string,\n  routeTag: DetectorRouteTag,\n): string | undefined {\n  if (routeTag === DEFAULT_LOCALE) {\n    return LATIN_LANGUAGE_TAGS[lang];\n  }\n\n  return HANI_LANGUAGE_TAGS[lang];\n}\n\nexport function remapWhatlangResult(\n  result: WhatlangWasmResult,\n  routeTag: DetectorRouteTag,\n): DetectorResult | null {\n  if (!hasSupportedScript(result, routeTag)) {\n    return null;\n  }\n\n  const tag = remapLanguageTag(result.lang, routeTag);\n  if (!tag) {\n    return null;\n  }\n\n  return {\n    tag,\n    confidence: result.confidence,\n    reliable: result.reliable,\n    source: \"wasm\",\n  };\n}\n\nexport function getDetectorFallbackTag(routeTag: DetectorRouteTag): string {\n  return routeTag === DEFAULT_HAN_TAG ? DEFAULT_HAN_TAG : DEFAULT_LOCALE;\n}\n","import { DEFAULT_HAN_TAG, DEFAULT_LOCALE } from \"../wc/locale-detect\";\nimport { segmentTextByLocale } from \"../wc\";\nimport type { LocaleChunk } from \"../wc/types\";\nimport { buildWordCounterResultFromChunks } from \"./result-builder\";\nimport { countSectionsWithResolvedDetector } from \"./sections\";\nimport {\n  DETECTOR_ROUTE_POLICIES,\n  LATIN_WASM_CORROBORATED_MIN_CONFIDENCE,\n  isAmbiguousDetectorRoute,\n  normalizeDetectorSampleForRoute,\n  shouldRunWasmDetector,\n  type DetectorRouteTag,\n} from \"./policy\";\nimport { detectWithWhatlangWasm, WASM_DETECTOR_RUNTIME_UNAVAILABLE_MESSAGE } from \"./whatlang-wasm\";\nimport { getDetectorFallbackTag, remapWhatlangResult } from \"./whatlang-map\";\nimport type {\n  DetectorCountSectionsOptions,\n  DetectorLocaleOptions,\n  DetectorWordCounterOptions,\n} from \"./types\";\n\nfunction shouldAcceptDetectorTag(\n  routeTag: DetectorRouteTag,\n  confidence: number | undefined,\n  reliable: boolean | undefined,\n): boolean {\n  const policy = DETECTOR_ROUTE_POLICIES[routeTag];\n  if (policy.requireReliable && reliable !== true) {\n    return false;\n  }\n\n  if (confidence === undefined) {\n    return false;\n  }\n\n  return confidence >= policy.minConfidence;\n}\n\ntype DetectorWindow = {\n  routeTag: DetectorRouteTag;\n  startIndex: number;\n  endIndex: number;\n  text: string;\n};\n\nfunction buildDetectorWindows(chunks: LocaleChunk[]): DetectorWindow[] {\n  const windows: DetectorWindow[] = [];\n\n  for (let index = 0; index < chunks.length; index += 1) {\n    const chunk = chunks[index];\n    if (!chunk || !isAmbiguousDetectorRoute(chunk.locale)) {\n      continue;\n    }\n\n    const previousWindow = windows[windows.length - 1];\n    if (\n      previousWindow &&\n      previousWindow.routeTag === chunk.locale &&\n      previousWindow.endIndex === index - 1\n    ) {\n      previousWindow.endIndex = index;\n      previousWindow.text += chunk.text;\n      continue;\n    }\n\n    windows.push({\n      routeTag: chunk.locale,\n      startIndex: index,\n      endIndex: index,\n      text: chunk.text,\n    });\n  }\n\n  return windows;\n}\n\nasync function resolveWindowLocale(window: DetectorWindow): Promise<string> {\n  if (!shouldRunWasmDetector(window.text, window.routeTag)) {\n    return window.routeTag;\n  }\n\n  const rawResult = await detectWithWhatlangWasm(window.text, window.routeTag);\n  const rawRemapped = rawResult ? remapWhatlangResult(rawResult, window.routeTag) : null;\n\n  const normalizedSample = normalizeDetectorSampleForRoute(window.text, window.routeTag);\n  const normalizedResult =\n    normalizedSample.length > 0 && normalizedSample !== window.text\n      ? await detectWithWhatlangWasm(normalizedSample, window.routeTag)\n      : null;\n  const normalizedRemapped = normalizedResult\n    ? remapWhatlangResult(normalizedResult, window.routeTag)\n    : null;\n\n  const candidates = [rawRemapped, normalizedRemapped].filter((value) => value !== null);\n  if (candidates.length === 0) {\n    return getDetectorFallbackTag(window.routeTag);\n  }\n\n  const strongestCandidate = candidates.reduce((best, current) => {\n    if (!best) {\n      return current;\n    }\n    return (current.confidence ?? 0) > (best.confidence ?? 0) ? current : best;\n  }, candidates[0]);\n\n  if (\n    strongestCandidate &&\n    shouldAcceptDetectorTag(\n      window.routeTag,\n      strongestCandidate.confidence,\n      strongestCandidate.reliable,\n    )\n  ) {\n    return strongestCandidate.tag;\n  }\n\n  if (\n    window.routeTag === DEFAULT_LOCALE &&\n    rawRemapped &&\n    normalizedRemapped &&\n    rawRemapped.tag === normalizedRemapped.tag\n  ) {\n    const corroboratedConfidence = Math.max(\n      rawRemapped.confidence ?? 0,\n      normalizedRemapped.confidence ?? 0,\n    );\n    if (corroboratedConfidence >= LATIN_WASM_CORROBORATED_MIN_CONFIDENCE) {\n      return rawRemapped.tag;\n    }\n  }\n\n  return getDetectorFallbackTag(window.routeTag);\n}\n\nexport { WASM_DETECTOR_RUNTIME_UNAVAILABLE_MESSAGE };\n\nexport async function segmentTextByLocaleWithWasmDetector(\n  text: string,\n  options: DetectorLocaleOptions = {},\n) {\n  const chunks = segmentTextByLocale(text, options);\n  const resolved = [...chunks];\n  const windows = buildDetectorWindows(chunks);\n\n  for (const window of windows) {\n    const resolvedLocale = await resolveWindowLocale(window);\n    for (let index = window.startIndex; index <= window.endIndex; index += 1) {\n      const chunk = resolved[index];\n      if (!chunk) {\n        continue;\n      }\n      resolved[index] = {\n        ...chunk,\n        locale: resolvedLocale,\n      };\n    }\n  }\n\n  return resolved;\n}\n\nexport async function wordCounterWithWasmDetector(\n  text: string,\n  options: DetectorWordCounterOptions = {},\n) {\n  const chunks = await segmentTextByLocaleWithWasmDetector(text, options);\n  return buildWordCounterResultFromChunks(chunks, options);\n}\n\nexport async function countSectionsWithWasmDetector(\n  input: string,\n  section: Parameters<typeof countSectionsWithResolvedDetector>[1],\n  options: DetectorCountSectionsOptions = {},\n) {\n  return countSectionsWithResolvedDetector(input, section, options);\n}\n","import type { SectionMode } from \"../markdown\";\nimport type { LocaleChunk } from \"../wc/types\";\nimport {\n  countSectionsWithRegexDetector,\n  segmentTextByLocaleWithRegexDetector,\n  wordCounterWithRegexDetector,\n} from \"./none\";\nimport {\n  countSectionsWithWasmDetector,\n  segmentTextByLocaleWithWasmDetector,\n  WASM_DETECTOR_RUNTIME_UNAVAILABLE_MESSAGE,\n  wordCounterWithWasmDetector,\n} from \"./wasm\";\nimport type {\n  DetectorCountSectionsOptions,\n  DetectorLocaleOptions,\n  DetectorMode,\n  DetectorResult,\n  DetectorSource,\n  DetectorWordCounterOptions,\n} from \"./types\";\n\nexport type {\n  DetectorCountSections,\n  DetectorCountSectionsOptions,\n  DetectorCountResult,\n  DetectorLocaleOptions,\n  DetectorMode,\n  DetectorResult,\n  DetectorRuntimeOptions,\n  DetectorSource,\n  DetectorWordCounterOptions,\n} from \"./types\";\n\nexport const DETECTOR_MODES: DetectorMode[] = [\"regex\", \"wasm\"];\nexport const DEFAULT_DETECTOR_MODE: DetectorMode = \"regex\";\n\nexport function resolveDetectorMode(mode?: DetectorMode): DetectorMode {\n  return mode ?? DEFAULT_DETECTOR_MODE;\n}\n\nexport function assertDetectorModeImplemented(mode?: DetectorMode): void {\n  void mode;\n}\n\nexport async function segmentTextByLocaleWithDetector(\n  text: string,\n  options: DetectorLocaleOptions = {},\n): Promise<LocaleChunk[]> {\n  const mode = resolveDetectorMode(options.detector);\n  if (mode === \"wasm\") {\n    return segmentTextByLocaleWithWasmDetector(text, options);\n  }\n  return segmentTextByLocaleWithRegexDetector(text, options);\n}\n\nexport async function wordCounterWithDetector(\n  text: string,\n  options: DetectorWordCounterOptions = {},\n) {\n  const mode = resolveDetectorMode(options.detector);\n  if (mode === \"wasm\") {\n    return wordCounterWithWasmDetector(text, options);\n  }\n  return wordCounterWithRegexDetector(text, options);\n}\n\nexport async function countSectionsWithDetector(\n  input: string,\n  section: SectionMode,\n  options: DetectorCountSectionsOptions = {},\n) {\n  const mode = resolveDetectorMode(options.detector);\n  if (mode === \"wasm\") {\n    return countSectionsWithWasmDetector(input, section, options);\n  }\n  return countSectionsWithRegexDetector(input, section, options);\n}\n\nexport const DETECTOR_SOURCES: DetectorSource[] = [\"script\", \"hint\", \"wasm\"];\nexport const DEFAULT_DETECTOR_RESULT_SOURCE: DetectorSource = \"script\";\nexport { WASM_DETECTOR_RUNTIME_UNAVAILABLE_MESSAGE };\n\nexport function createDetectorResult(\n  tag: string,\n  source: DetectorSource = DEFAULT_DETECTOR_RESULT_SOURCE,\n  confidence?: number,\n  reliable?: boolean,\n): DetectorResult {\n  return {\n    tag,\n    source,\n    ...(confidence === undefined ? {} : { confidence }),\n    ...(reliable === undefined ? {} : { reliable }),\n  };\n}\n","import {\n  assertDetectorModeImplemented,\n  countSectionsWithDetector,\n  createDetectorResult,\n  DEFAULT_DETECTOR_MODE,\n  DEFAULT_DETECTOR_RESULT_SOURCE,\n  DETECTOR_MODES,\n  DETECTOR_SOURCES,\n  resolveDetectorMode,\n  segmentTextByLocaleWithDetector,\n  WASM_DETECTOR_RUNTIME_UNAVAILABLE_MESSAGE,\n  wordCounterWithDetector,\n} from \"./index\";\n\nconst cjsExports = {\n  assertDetectorModeImplemented,\n  countSectionsWithDetector,\n  createDetectorResult,\n  DEFAULT_DETECTOR_MODE,\n  DEFAULT_DETECTOR_RESULT_SOURCE,\n  DETECTOR_MODES,\n  DETECTOR_SOURCES,\n  resolveDetectorMode,\n  segmentTextByLocaleWithDetector,\n  WASM_DETECTOR_RUNTIME_UNAVAILABLE_MESSAGE,\n  wordCounterWithDetector,\n};\n\nexport = cjsExports;\n"],"mappings":";;;;;;AAQA,eAAsB,qCACpB,MACA,UAAiC,EAAE,EACnC;AACA,QAAOA,iBAAAA,oBAAoB,MAAM,QAAQ;;AAG3C,eAAsB,6BACpB,MACA,UAAsC,EAAE,EACxC;AACA,QAAOC,iBAAAA,WAAY,MAAM,QAAQ;;AAGnC,eAAsB,+BACpB,OACA,SACA,UAAwC,EAAE,EAC1C;AACA,QAAOC,iBAAAA,cAAc,OAAO,SAAS,QAAQ;;;;ACP/C,SAAS,gBAAgB,UAAqC;AAC5D,QACE,SAAS,OAAO,QAChB,SAAS,OAAO,UAChB,SAAS,OAAO,eACf,SAAS,OAAO,cAAc;;AAInC,SAAS,yBACP,UACA,SAC+B;AAC/B,KAAI,CAAC,QACH;CAEF,MAAM,aAAaC,iBAAAA,yBAAyB;AAC5C,MAAK,MAAM,SAAS,UAAU;AAC5B,MAAI,CAAC,MAAM,SACT;AAEF,mBAAA,wBAAwB,YAAY,MAAM,SAAS;;AAErD,QAAO;;AAGT,SAAgB,iCACd,QACA,UAA8B,EAAE,EACb;CACnB,MAAM,OAAwBC,iBAAAA,YAAY,QAAQ,MAAM,QAAQ;CAChE,MAAM,kBAAkB,QAAQ,QAAQ,SAAS;CACjD,MAAM,oBAAoB,QAAQ,QAAQ,kBAAkB;AAE5D,KAAI,SAAS,UAAU,SAAS,kBAAkB;EAChD,MAAM,WAAW,OAAO,KAAK,UAC3BC,iBAAAA,iBAAiB,OAAO,iBAAiB,kBAAkB,CAC5D;EACD,MAAM,QAAQ,SAAS,QAAQ,KAAK,UAAU,MAAM,MAAM,OAAO,EAAE;EACnE,MAAM,SAAS,kBACX;GACE,OAAO,SAAS,QAAQ,KAAK,UAAU,MAAM,MAAM,WAAW,EAAE;GAChE,UAAU,SAAS,QAAQ,KAAK,UAAU,MAAM,MAAM,cAAc,EAAE;GACtE;GACD,GACD,KAAA;AAEJ,MAAI,SAAS,OAOX,QAAO;GACL;GACA;GACA,WAAW;IACT;IACA,OAX2B,SAAS,KAAK,WAAW;KACtD,QAAQ,MAAM;KACd,MAAM,MAAM;KACZ,OAAO,MAAM;KACb,UAAU,MAAM;KACjB,EAAE;IAOA;GACF;AASH,SAAO;GACL;GACA;GACA,WAAW;IACT;IACA,OAXeC,iBAAAA,uBAAuB,SAAS,CACA,KAAK,WAAW;KACjE,QAAQ,MAAM;KACd,OAAO,MAAM;KACb,UAAU,MAAM;KACjB,EAAE;IAOA;GACF;;CAGH,MAAM,WAAW,OAAO,KAAK,UAC3BC,iBAAAA,aAAa,OAAO,iBAAiB,kBAAkB,CACxD;CACD,MAAM,aAAa,SAAS,QAAQ,KAAK,UAAU,MAAM,MAAM,OAAO,EAAE;CACxE,MAAM,gBAAgB,kBAClB,SAAS,QAAQ,KAAK,UAAU;AAC9B,MAAI,CAAC,MAAM,SACT,QAAO;AAET,SAAO,MAAM,gBAAgB,MAAM,SAAS;IAC3C,EAAE,GACL;CACJ,MAAM,QAAQ,SAAS,QAAQ,KAAK,UAAU;EAC5C,IAAI,aAAa,MAAM;AACvB,MAAI,mBAAmB,MAAM,SAC3B,eAAc,gBAAgB,MAAM,SAAS;AAE/C,SAAO,MAAM;IACZ,EAAE;CAEL,MAAM,SAAS,kBAAkB;EAAE,OAAO;EAAY,UAAU;EAAe;EAAO,GAAG,KAAA;AAEzF,KAAI,SAAS,WAQX,QAAO;EACL;EACA;EACA,WAAW;GACT;GACA,OAZ+B,SAAS,KAAK,WAAW;IAC1D,QAAQ,MAAM;IACd,MAAM,MAAM;IACZ,OAAO,MAAM;IACb,UAAU,MAAM;IAChB,UAAU,MAAM;IACjB,EAAE;GAOA;EACF;AAGH,KAAI,SAAS,YAGX,QAAO;EACL;EACA;EACA,WAAW;GACT;GACA,OAPUC,iBAAAA,kBAAkB,SAAS;GAQrC,UAPa,yBAAyB,UAAU,gBAAgB;GAQjE;EACF;AAUH,QAAO;EACL;EACA;EACA,WAAW;GACT;GACA,OAZ4B,SAAS,KAAK,WAAW;IACvD,QAAQ,MAAM;IACd,MAAM,MAAM;IACZ,OAAO,MAAM;IACb,UAAU,MAAM;IACjB,EAAE;GAQA;EACF;;;;AClKH,SAAS,cAAc,OAAwB;AAC7C,KAAI,SAAS,KACX,QAAO;AAET,KAAI,OAAO,UAAU,SACnB,QAAO;AAET,KAAI,OAAO,UAAU,YAAY,OAAO,UAAU,UAChD,QAAO,OAAO,MAAM;AAEtB,KAAI;AACF,SAAO,KAAK,UAAU,MAAM;SACtB;AACN,SAAO,OAAO,MAAM;;;AAIxB,eAAe,iBACb,MACA,SACoF;AACpF,KAAI,CAAC,QAAQ,OAAO,SAAS,YAAY,MAAM,QAAQ,KAAK,CAC1D,QAAO,EAAE;AAGX,QAAO,QAAQ,IACb,OAAO,QAAQ,KAAK,CAAC,IAAI,OAAO,CAAC,KAAK,WAAW;EAC/C,MAAM,YAAY,cAAc,MAAM;AAEtC,SAAO;GACL,MAAM;GACN,QAAQ;GACR,QAAQ,MAAM,wBAJH,YAAY,GAAG,IAAI,IAAI,cAAc,KAIJ,QAAQ;GACrD;GACD,CACH;;AAGH,eAAe,gBACb,MACA,MACA,SACA,QACgG;AAChG,QAAO,CAAC;EAAE;EAAM;EAAQ,QAAQ,MAAM,wBAAwB,MAAM,QAAQ;EAAE,CAAC;;AAGjF,SAAS,UAAU,OAAqD;AACtE,QAAO,MAAM,QAAQ,KAAK,SAAS,MAAM,KAAK,OAAO,OAAO,EAAE;;AAGhE,eAAsB,kCACpB,OACA,SACA,UAAwC,EAAE,EAChB;AACI,SAAQ;AACtC,KAAI,YAAY,OAAO;EACrB,MAAM,SAAS,MAAM,wBAAwB,OAAO,QAAQ;AAC5D,SAAO;GACL;GACA,OAAO,OAAO;GACd,iBAAiB;GACjB,OAAO,CAAC;IAAE,MAAM;IAAO,QAAQ;IAAW;IAAQ,CAAC;GACpD;;CAGH,MAAM,SAASC,iBAAAA,cAAc,MAAM;CACnC,MAAM,kBAAkB,OAAO,eAAe;CAC9C,MAAM,cAAc,OAAO,WAAW;CAEtC,IAAI,QAA+F,EAAE;AAErG,KAAI,YAAY,cACd,SAAQ,MAAM,gBAAgB,eAAe,iBAAiB,SAAS,cAAc;UAC5E,YAAY,UACrB,SAAQ,MAAM,gBAAgB,WAAW,aAAa,SAAS,UAAU;UAChE,YAAY,QACrB,SAAQ,CACN,GAAI,MAAM,gBAAgB,eAAe,iBAAiB,SAAS,cAAc,EACjF,GAAI,MAAM,gBAAgB,WAAW,aAAa,SAAS,UAAU,CACtE;UACQ,YAAY,UACrB,SAAQ,MAAM,iBAAiB,OAAO,MAAM,QAAQ;UAC3C,YAAY,gBACrB,SAAQ,CACN,GAAI,MAAM,iBAAiB,OAAO,MAAM,QAAQ,EAChD,GAAI,MAAM,gBAAgB,WAAW,aAAa,SAAS,UAAU,CACtE;AAGH,QAAO;EACL;EACA,OAAO,UAAU,MAAM;EACvB,iBAAiB,OAAO;EACxB;EACD;;AClGH,MAAa,4BAA4B;AACzC,MAAa,2BAA2B;AAGxC,MAAM,qBAAqB;AAC3B,MAAM,mBAAmB;AAWzB,MAAa,0BAAyE;EACnFC,iBAAAA,iBAAiB;EAChB,UAAUA,iBAAAA;EACV,gBAAA;EACA,eAAe;EACf,iBAAiB;EAClB;EACAC,iBAAAA,kBAAkB;EACjB,UAAUA,iBAAAA;EACV,gBAAA;EACA,eAAe;EACf,iBAAiB;EAClB;CACF;AAED,SAAgB,yBAAyB,QAA4C;AACnF,QAAO,WAAA,cAA6B,WAAA;;AAGtC,SAAgB,gCACd,MACA,UACQ;CACR,MAAM,UAAU,aAAA,aAA+B,mBAAmB;CAClE,IAAI,QAAQ;AACZ,MAAK,MAAM,QAAQ,KACjB,KAAI,QAAQ,KAAK,KAAK,CACpB,UAAS;AAGb,QAAO;;AAGT,SAAgB,sBAAsB,MAAc,UAAqC;CACvF,MAAM,SAAS,wBAAwB;AACvC,QAAO,gCAAgC,MAAM,SAAS,IAAI,OAAO;;AAGnE,SAAgB,gCACd,MACA,UACQ;CACR,MAAM,UAAU,aAAA,aAA+B,mBAAmB;AAClE,QAAO,CAAC,GAAG,KAAK,CACb,KAAK,SAAS;AACb,MAAI,QAAQ,KAAK,KAAK,CACpB,QAAO;AAET,MAAI,MAAM,KAAK,KAAK,CAClB,QAAO;AAET,SAAO;GACP,CACD,KAAK,GAAG,CACR,QAAQ,QAAQ,IAAI,CACpB,MAAM;;;;ACpEX,MAAM,wBAAwB;AAC9B,MAAM,wBAAwB;AAC9B,MAAM,mBAAmB;AACzB,MAAM,mBAAA,GAAA,YAAA,eAAA,QAAA,MAAA,CAAA,cAAA,WAAA,CAAA,KAAgD;AAEtD,MAAa,4CACX;AAMF,IAAI,gBAAoD;AAExD,SAAS,8BAAwC;CAC/C,MAAM,aAAA,GAAA,UAAA,UAAA,GAAA,SAAA,eAAA,QAAA,MAAA,CAAA,cAAA,WAAA,CAAA,KAAkD,CAAC;CACzD,MAAM,6BAAa,IAAI,KAAa;CACpC,IAAI,aAAa;AAEjB,MAAK,IAAI,QAAQ,GAAG,QAAQ,kBAAkB,SAAS,GAAG;AACxD,aAAW,KAAA,GAAA,UAAA,MAAS,YAAY,uBAAuB,sBAAsB,CAAC;AAC9E,aAAW,KAAA,GAAA,UAAA,MAAS,YAAY,aAAa,uBAAuB,sBAAsB,CAAC;EAE3F,MAAM,aAAA,GAAA,UAAA,SAAoB,WAAW;AACrC,MAAI,cAAc,WAChB;AAEF,eAAa;;AAGf,QAAO,CAAC,GAAG,WAAW;;AAGxB,SAAS,gCAAwC;AAC/C,MAAK,MAAM,aAAa,6BAA6B,CACnD,MAAA,GAAA,QAAA,YAAe,UAAU,CACvB,QAAO;AAIX,OAAM,IAAI,MAAM,0CAA0C;;AAG5D,eAAe,yBAAsD;AACnE,KAAI,CAAC,cACH,kBAAiB,YAAY;AAE3B,SAAO,gBADY,+BAA+B,CAChB;KAChC;AAGN,QAAO;;AAGT,eAAsB,uBACpB,MACA,UACoC;AAEpC,SADmB,MAAM,wBAAwB,EAC/B,gBAAgB,MAAM,SAAS;;;;ACvDnD,MAAM,sBAA8C;CAClD,KAAK;CACL,KAAK;CACL,KAAK;CACL,KAAK;CACL,KAAK;CACL,KAAK;CACL,KAAK;CACL,KAAK;CACL,KAAK;CACL,KAAK;CACL,KAAK;CACL,KAAK;CACL,KAAK;CACL,KAAK;CACL,KAAK;CACL,KAAK;CACL,KAAK;CACN;AAED,MAAM,qBAA6C;CACjD,KAAK;CACL,KAAK;CACN;AAED,SAAS,mBAAmB,QAA4B,UAAqC;AAC3F,KAAI,aAAA,WACF,QAAO,OAAO,WAAW;AAG3B,QAAO,OAAO,WAAW;;AAG3B,SAAS,iBACP,MACA,UACoB;AACpB,KAAI,aAAA,WACF,QAAO,oBAAoB;AAG7B,QAAO,mBAAmB;;AAG5B,SAAgB,oBACd,QACA,UACuB;AACvB,KAAI,CAAC,mBAAmB,QAAQ,SAAS,CACvC,QAAO;CAGT,MAAM,MAAM,iBAAiB,OAAO,MAAM,SAAS;AACnD,KAAI,CAAC,IACH,QAAO;AAGT,QAAO;EACL;EACA,YAAY,OAAO;EACnB,UAAU,OAAO;EACjB,QAAQ;EACT;;AAGH,SAAgB,uBAAuB,UAAoC;AACzE,QAAO,aAAA,aAA+BC,iBAAAA,kBAAkBC,iBAAAA;;;;ACxD1D,SAAS,wBACP,UACA,YACA,UACS;CACT,MAAM,SAAS,wBAAwB;AACvC,KAAI,OAAO,mBAAmB,aAAa,KACzC,QAAO;AAGT,KAAI,eAAe,KAAA,EACjB,QAAO;AAGT,QAAO,cAAc,OAAO;;AAU9B,SAAS,qBAAqB,QAAyC;CACrE,MAAM,UAA4B,EAAE;AAEpC,MAAK,IAAI,QAAQ,GAAG,QAAQ,OAAO,QAAQ,SAAS,GAAG;EACrD,MAAM,QAAQ,OAAO;AACrB,MAAI,CAAC,SAAS,CAAC,yBAAyB,MAAM,OAAO,CACnD;EAGF,MAAM,iBAAiB,QAAQ,QAAQ,SAAS;AAChD,MACE,kBACA,eAAe,aAAa,MAAM,UAClC,eAAe,aAAa,QAAQ,GACpC;AACA,kBAAe,WAAW;AAC1B,kBAAe,QAAQ,MAAM;AAC7B;;AAGF,UAAQ,KAAK;GACX,UAAU,MAAM;GAChB,YAAY;GACZ,UAAU;GACV,MAAM,MAAM;GACb,CAAC;;AAGJ,QAAO;;AAGT,eAAe,oBAAoB,QAAyC;AAC1E,KAAI,CAAC,sBAAsB,OAAO,MAAM,OAAO,SAAS,CACtD,QAAO,OAAO;CAGhB,MAAM,YAAY,MAAM,uBAAuB,OAAO,MAAM,OAAO,SAAS;CAC5E,MAAM,cAAc,YAAY,oBAAoB,WAAW,OAAO,SAAS,GAAG;CAElF,MAAM,mBAAmB,gCAAgC,OAAO,MAAM,OAAO,SAAS;CACtF,MAAM,mBACJ,iBAAiB,SAAS,KAAK,qBAAqB,OAAO,OACvD,MAAM,uBAAuB,kBAAkB,OAAO,SAAS,GAC/D;CACN,MAAM,qBAAqB,mBACvB,oBAAoB,kBAAkB,OAAO,SAAS,GACtD;CAEJ,MAAM,aAAa,CAAC,aAAa,mBAAmB,CAAC,QAAQ,UAAU,UAAU,KAAK;AACtF,KAAI,WAAW,WAAW,EACxB,QAAO,uBAAuB,OAAO,SAAS;CAGhD,MAAM,qBAAqB,WAAW,QAAQ,MAAM,YAAY;AAC9D,MAAI,CAAC,KACH,QAAO;AAET,UAAQ,QAAQ,cAAc,MAAM,KAAK,cAAc,KAAK,UAAU;IACrE,WAAW,GAAG;AAEjB,KACE,sBACA,wBACE,OAAO,UACP,mBAAmB,YACnB,mBAAmB,SACpB,CAED,QAAO,mBAAmB;AAG5B,KACE,OAAO,aAAA,cACP,eACA,sBACA,YAAY,QAAQ,mBAAmB;MAER,KAAK,IAClC,YAAY,cAAc,GAC1B,mBAAmB,cAAc,EAClC,IAAA,GAEC,QAAO,YAAY;;AAIvB,QAAO,uBAAuB,OAAO,SAAS;;AAKhD,eAAsB,oCACpB,MACA,UAAiC,EAAE,EACnC;CACA,MAAM,SAASC,iBAAAA,oBAAoB,MAAM,QAAQ;CACjD,MAAM,WAAW,CAAC,GAAG,OAAO;CAC5B,MAAM,UAAU,qBAAqB,OAAO;AAE5C,MAAK,MAAM,UAAU,SAAS;EAC5B,MAAM,iBAAiB,MAAM,oBAAoB,OAAO;AACxD,OAAK,IAAI,QAAQ,OAAO,YAAY,SAAS,OAAO,UAAU,SAAS,GAAG;GACxE,MAAM,QAAQ,SAAS;AACvB,OAAI,CAAC,MACH;AAEF,YAAS,SAAS;IAChB,GAAG;IACH,QAAQ;IACT;;;AAIL,QAAO;;AAGT,eAAsB,4BACpB,MACA,UAAsC,EAAE,EACxC;AAEA,QAAO,iCADQ,MAAM,oCAAoC,MAAM,QAAQ,EACvB,QAAQ;;AAG1D,eAAsB,8BACpB,OACA,SACA,UAAwC,EAAE,EAC1C;AACA,QAAO,kCAAkC,OAAO,SAAS,QAAQ;;;;AC5InE,MAAa,iBAAiC,CAAC,SAAS,OAAO;AAC/D,MAAa,wBAAsC;AAEnD,SAAgB,oBAAoB,MAAmC;AACrE,QAAO,QAAA;;AAGT,SAAgB,8BAA8B,MAA2B;AAIzE,eAAsB,gCACpB,MACA,UAAiC,EAAE,EACX;AAExB,KADa,oBAAoB,QAAQ,SAAS,KACrC,OACX,QAAO,oCAAoC,MAAM,QAAQ;AAE3D,QAAO,qCAAqC,MAAM,QAAQ;;AAG5D,eAAsB,wBACpB,MACA,UAAsC,EAAE,EACxC;AAEA,KADa,oBAAoB,QAAQ,SAAS,KACrC,OACX,QAAO,4BAA4B,MAAM,QAAQ;AAEnD,QAAO,6BAA6B,MAAM,QAAQ;;AAGpD,eAAsB,0BACpB,OACA,SACA,UAAwC,EAAE,EAC1C;AAEA,KADa,oBAAoB,QAAQ,SAAS,KACrC,OACX,QAAO,8BAA8B,OAAO,SAAS,QAAQ;AAE/D,QAAO,+BAA+B,OAAO,SAAS,QAAQ;;AAGhE,MAAa,mBAAqC;CAAC;CAAU;CAAQ;CAAO;AAC5E,MAAa,iCAAiD;AAG9D,SAAgB,qBACd,KACA,SAAyB,gCACzB,YACA,UACgB;AAChB,QAAO;EACL;EACA;EACA,GAAI,eAAe,KAAA,IAAY,EAAE,GAAG,EAAE,YAAY;EAClD,GAAI,aAAa,KAAA,IAAY,EAAE,GAAG,EAAE,UAAU;EAC/C;;;;AChFH,MAAM,aAAa;CACjB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACD;iBAEQ"}