langtell 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -13
- package/dist/chrome-ai.d.ts +1 -1
- package/dist/{chunk-6PWEE3SR.js → chunk-JXGCCDSV.js} +3 -2
- package/dist/chunk-JXGCCDSV.js.map +1 -0
- package/dist/classify.d.ts +1 -1
- package/dist/cyrillic.d.ts +12 -5
- package/dist/cyrillic.js +7 -2
- package/dist/cyrillic.js.map +1 -1
- package/dist/franc.d.ts +1 -1
- package/dist/fuse.d.ts +1 -1
- package/dist/fuse.js +1 -1
- package/dist/headers.d.ts +1 -1
- package/dist/html.d.ts +1 -1
- package/dist/index.d.ts +37 -3
- package/dist/index.js +39 -5
- package/dist/index.js.map +1 -1
- package/dist/profiles.d.ts +5 -2
- package/dist/profiles.js +431 -2
- package/dist/profiles.js.map +1 -1
- package/dist/text.d.ts +1 -1
- package/dist/{types-BIXrkuAr.d.ts → types-DfHZ5hkh.d.ts} +19 -3
- package/package.json +1 -1
- package/dist/chunk-6PWEE3SR.js.map +0 -1
package/README.md
CHANGED
|
@@ -97,21 +97,47 @@ This powers per-rung safety gates ("act only when a _weak_ rung clears a high
|
|
|
97
97
|
margin") and diagnostics — uses a single confidence number can't serve. The
|
|
98
98
|
high-level `compile`/`detect`/`fuse` output is unchanged; this is purely additive.
|
|
99
99
|
|
|
100
|
+
### Roster-free Cyrillic fast-path
|
|
101
|
+
|
|
102
|
+
`langtell/classify` scores a snippet _relative to a roster you pass in_.
|
|
103
|
+
`langtell/cyrillic` is the opposite trade: a fixed, zero-config discriminator for
|
|
104
|
+
the four Cyrillic languages langtell profiles — Ukrainian, Russian, Belarusian,
|
|
105
|
+
Bulgarian — decided purely by distinctive letters, with no roster, no
|
|
106
|
+
tokenization, and no franc. Reach for it when you just need _"is this Russian? is
|
|
107
|
+
this Ukrainian?"_ on a hot path and don't want to assemble a candidate set.
|
|
108
|
+
|
|
109
|
+
```ts
|
|
110
|
+
import { detectCyrillicLanguage, isUkrainian } from "langtell/cyrillic";
|
|
111
|
+
|
|
112
|
+
detectCyrillicLanguage("Їжак"); // → { language: "uk", … } ї is uniquely Ukrainian
|
|
113
|
+
detectCyrillicLanguage("жёлтый"); // → { language: "ru", … } ё / ы are Russian
|
|
114
|
+
detectCyrillicLanguage("съм българин"); // → { language: "bg", … } ъ used as a vowel, repeated
|
|
115
|
+
detectCyrillicLanguage("подъезд"); // → { language: "ru", … } lone ъ in a short word
|
|
116
|
+
isUkrainian("Слава Україні"); // → true
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
It returns `"unknown"` rather than guessing when the signals are insufficient — no
|
|
120
|
+
Cyrillic at all, a uk/ru tie, or only an ambiguous `э`. The `CyrillicVerdict` also
|
|
121
|
+
carries the raw `ukScore` / `ruScore` tallies behind the call. Zero-dependency and
|
|
122
|
+
side-effect-free; escalate to `classifyBySnippet` or a franc-backed source when
|
|
123
|
+
letter signals aren't enough.
|
|
124
|
+
|
|
100
125
|
## API at a glance
|
|
101
126
|
|
|
102
|
-
| Export | Role
|
|
103
|
-
| ------------------------------------- |
|
|
104
|
-
| `compile(config)` | Build a configured `detect` function (does the precompute once).
|
|
105
|
-
| `detect(input)` | The compiled detector. Sync or `Promise`, by config — see below.
|
|
106
|
-
| `evidenceFromText(text, candidates?)` | Producer: roster-relative script + distinctive-letter signals. Zero-dep, sync.
|
|
107
|
-
| `evidenceFromHtml(html)` | Producer: `<html lang>`, meta content-language, `og:locale`. Zero-dep, sync.
|
|
108
|
-
| `evidenceFromHeaders(h)` | Producer: HTTP `Content-Language`. Zero-dep, sync.
|
|
109
|
-
| `normalizeBCP47(tag)` | Normalize a BCP-47 tag/alias to a canonical code (`uk-UA`/`ua` → `uk`).
|
|
110
|
-
| `fuse(evidence, opts?)` | Weighted blend + "context never overrides clear script" guard.
|
|
111
|
-
| `langtell/profiles` | Ready-made `LanguageProfile` data (uk/ru/be/bg/en). Opt-in (carries word data).
|
|
112
|
-
| `langtell/classify` | Opt-in structured snippet verdict (`{ language, margin, rung }`). Zero-dep.
|
|
113
|
-
| `langtell/
|
|
114
|
-
| `langtell/
|
|
127
|
+
| Export | Role |
|
|
128
|
+
| ------------------------------------- | ------------------------------------------------------------------------------------------------------------ |
|
|
129
|
+
| `compile(config)` | Build a configured `detect` function (does the precompute once). |
|
|
130
|
+
| `detect(input)` | The compiled detector. Sync or `Promise`, by config — see below. |
|
|
131
|
+
| `evidenceFromText(text, candidates?)` | Producer: roster-relative script + distinctive-letter signals. Zero-dep, sync. |
|
|
132
|
+
| `evidenceFromHtml(html)` | Producer: `<html lang>`, meta content-language, `og:locale`. Zero-dep, sync. |
|
|
133
|
+
| `evidenceFromHeaders(h)` | Producer: HTTP `Content-Language`. Zero-dep, sync. |
|
|
134
|
+
| `normalizeBCP47(tag)` | Normalize a BCP-47 tag/alias to a canonical code (`uk-UA`/`ua` → `uk`). |
|
|
135
|
+
| `fuse(evidence, opts?)` | Weighted blend + "context never overrides clear script" guard. |
|
|
136
|
+
| `langtell/profiles` | Ready-made `LanguageProfile` data (uk/ru/be/bg/en). Opt-in (carries word data). |
|
|
137
|
+
| `langtell/classify` | Opt-in structured snippet verdict (`{ language, margin, rung }`). Zero-dep. |
|
|
138
|
+
| `langtell/cyrillic` | Opt-in roster-free Cyrillic fast-path (`detectCyrillicLanguage`, `isRussian`/`isUkrainian`). Zero-dep, sync. |
|
|
139
|
+
| `langtell/franc` | Opt-in franc engine (pulls trigram tables). Sync. |
|
|
140
|
+
| `langtell/chrome-ai` | Opt-in on-device Chrome AI engine (browser). Async. |
|
|
115
141
|
|
|
116
142
|
`detect` returns a plain `Classification` when every registered source is
|
|
117
143
|
synchronous, and `Promise<Classification>` the moment an async engine is in the
|
package/dist/chrome-ai.d.ts
CHANGED
|
@@ -8,6 +8,7 @@ var DEFAULT_KIND_WEIGHT = {
|
|
|
8
8
|
"chrome-ai": 1,
|
|
9
9
|
"source-prior": 0.7,
|
|
10
10
|
franc: 0.7,
|
|
11
|
+
"node-lang": 0.65,
|
|
11
12
|
"http-content-language": 0.6,
|
|
12
13
|
"meta-content-language": 0.55,
|
|
13
14
|
"meta-og-locale": 0.55,
|
|
@@ -136,5 +137,5 @@ function clamp01(value) {
|
|
|
136
137
|
}
|
|
137
138
|
|
|
138
139
|
export { fuse };
|
|
139
|
-
//# sourceMappingURL=chunk-
|
|
140
|
-
//# sourceMappingURL=chunk-
|
|
140
|
+
//# sourceMappingURL=chunk-JXGCCDSV.js.map
|
|
141
|
+
//# sourceMappingURL=chunk-JXGCCDSV.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/fuse.ts"],"names":[],"mappings":";;;;AA2BA,IAAM,mBAAA,GAA8C;AAAA,EAClD,cAAA,EAAgB,CAAA;AAAA,EAChB,iBAAA,EAAmB,CAAA;AAAA,EACnB,WAAA,EAAa,CAAA;AAAA,EACb,cAAA,EAAgB,GAAA;AAAA,EAChB,KAAA,EAAO,GAAA;AAAA,EACP,WAAA,EAAa,IAAA;AAAA,EACb,uBAAA,EAAyB,GAAA;AAAA,EACzB,uBAAA,EAAyB,IAAA;AAAA,EACzB,gBAAA,EAAkB,IAAA;AAAA,EAClB,WAAA,EAAa;AACf,CAAA;AAKA,IAAM,+BAAe,IAAI,GAAA,CAAY,CAAC,cAAA,EAAgB,OAAA,EAAS,WAAW,CAAC,CAAA;AAI3E,IAAM,uBAAA,GAA0B,GAAA;AAEhC,IAAM,iBAAA,GAAoB,IAAA;AAC1B,IAAM,UAAA,GAAa,IAAA;AAcZ,SAAS,IAAA,CACd,QAAA,EACA,OAAA,GAAuB,EAAC,EACR;AAChB,EAAA,MAAM,OAAA,GAAU,OAAA,CAAQ,OAAA,IAAW,EAAC;AACpC,EAAA,MAAM,UAAA,GAAa,iBAAA,CAAkB,QAAA,EAAU,OAAA,CAAQ,UAAU,CAAA;AASjE,EAAA,MAAM,OAAA,GACJ,QAAQ,uBAAA,KAA4B,SAAA,GAChC,qBAAqB,UAAA,EAAY,OAAA,CAAQ,UAAU,CAAA,GACnD,UAAA;AAEN,EAAA,MAAM,MAAA,uBAAa,GAAA,EAAoB;AACvC,EAAA,KAAA,MAAW,QAAQ,OAAA,EAAS;AAC1B,IAAA,IAAI,IAAA,CAAK,aAAa,SAAA,EAAW;AACjC,IAAA,MAAM,MAAA,GACJ,OAAA,CAAQ,IAAA,CAAK,MAAM,CAAA,IAAK,OAAA,CAAQ,IAAA,CAAK,IAAI,CAAA,IAAK,mBAAA,CAAoB,IAAA,CAAK,IAAI,CAAA,IAAK,GAAA;AAClF,IAAA,MAAA,CAAO,GAAA,CAAI,IAAA,CAAK,QAAA,EAAA,CAAW,MAAA,CAAO,GAAA,CAAI,IAAA,CAAK,QAAQ,CAAA,IAAK,CAAA,IAAK,OAAA,CAAQ,IAAA,CAAK,UAAU,IAAI,MAAM,CAAA;AAAA,EAChG;AAGA,EAAA,MAAM,MAAA,GAAS,wBAAwB,OAAO,CAAA;AAE9C,EAAA,MAAM,EAAE,IAAA,EAAM,SAAA,EAAW,aAAY,GAAI,MAAA,CAAO,QAAQ,MAAM,CAAA;AAE9D,EAAA,IAAI,SAAS,IAAA,IAAQ,SAAA,GAAY,iBAAA,IAAqB,SAAA,GAAY,cAAc,UAAA,EAAY;AAG1F,IAAA,IAAI,MAAA,KAAW,IAAA,IAAQ,MAAA,CAAO,GAAA,CAAI,MAAM,CAAA,EAAG;AACzC,MAAA,MAAM,KAAA,GAAQ,MAAA,CAAO,GAAA,CAAI,MAAM,CAAA,IAAK,CAAA;AACpC,MAAA,OAAO;AAAA,QACL,QAAA,EAAU,MAAA;AAAA,QACV,UAAA,EAAY,OAAA,CAAQ,KAAA,IAAS,KAAA,GAAQ,IAAA,CAAK,CAAA;AAAA,QAC1C,QAAA,EAAU,CAAC,GAAG,UAAU;AAAA,OAC1B;AAAA,IACF;AACA,IAAA,OAAO,EAAE,QAAA,EAAU,SAAA,EAAW,UAAA,EAAY,OAAA,CAAQ,SAAS,CAAA,EAAG,QAAA,EAAU,CAAC,GAAG,UAAU,CAAA,EAAE;AAAA,EAC1F;AAEA,EAAA,OAAO;AAAA,IACL,QAAA,EAAU,IAAA;AAAA,IACV,UAAA,EAAY,OAAA,CAAQ,SAAA,IAAa,SAAA,GAAY,cAAc,IAAA,CAAK,CAAA;AAAA,IAChE,QAAA,EAAU,CAAC,GAAG,UAAU;AAAA,GAC1B;AACF;AAYA,SAAS,iBAAA,CACP,UACA,WAAA,EACoB;AACpB,EAAA,OAAO,QAAA,CAAS,GAAA,CAAI,CAAC,IAAA,KAAS;AAC5B,IAAA,IAAI,IAAA,CAAK,QAAA,KAAa,SAAA,EAAW,OAAO,IAAA;AACxC,IAAA,MAAM,UAAA,GAAa,cAAA,CAAe,IAAA,CAAK,QAAQ,KAAK,IAAA,CAAK,QAAA;AACzD,IAAA,IAAI,UAAA,KAAe,IAAA,CAAK,QAAA,EAAU,OAAO,IAAA;AACzC,IAAA,OAAO,EAAE,GAAG,IAAA,EAAM,QAAA,EAAU,UAAA,EAAW;AAAA,EACzC,CAAC,CAAA;AACH;AAoBA,SAAS,oBAAA,CACP,YACA,UAAA,EACoB;AACpB,EAAA,MAAM,SAAA,GAAY,WAAW,MAAA,CAAO,CAAC,SAAS,CAAC,aAAA,CAAc,IAAA,EAAM,UAAU,CAAC,CAAA;AAE9E,EAAA,MAAM,WAAA,GAAc,4BAAA,CAA6B,UAAA,EAAY,UAAU,CAAA;AACvE,EAAA,IAAI,WAAA,KAAgB,MAAM,OAAO,SAAA;AAEjC,EAAA,MAAM,QAAA,GAAW,YAAA,CAAa,UAAA,IAAc,EAAE,CAAA;AAC9C,EAAA,OAAO,SAAA,CAAU,MAAA,CAAO,CAAC,IAAA,KAAS;AAGhC,IAAA,IAAI,YAAA,CAAa,IAAI,IAAA,CAAK,IAAI,KAAK,IAAA,CAAK,QAAA,KAAa,WAAW,OAAO,IAAA;AACvE,IAAA,MAAM,UAAA,GAAa,QAAA,CAAS,GAAA,CAAI,IAAA,CAAK,QAAQ,CAAA;AAC7C,IAAA,OAAO,UAAA,KAAe,UAAa,UAAA,KAAe,WAAA;AAAA,EACpD,CAAC,CAAA;AACH;AAKA,SAAS,4BAAA,CACP,YACA,UAAA,EACmB;AACnB,EAAA,IAAI,UAAA,KAAe,QAAW,OAAO,IAAA;AACrC,EAAA,MAAM,QAAA,GAAW,aAAa,UAAU,CAAA;AACxC,EAAA,KAAA,MAAW,QAAQ,UAAA,EAAY;AAC7B,IAAA,IAAI,aAAA,CAAc,IAAA,EAAM,UAAU,CAAA,EAAG;AACnC,MAAA,MAAM,MAAA,GAAS,QAAA,CAAS,GAAA,CAAI,IAAA,CAAK,QAAQ,CAAA;AACzC,MAAA,IAAI,MAAA,KAAW,QAAW,OAAO,MAAA;AAAA,IACnC;AAAA,EACF;AACA,EAAA,OAAO,IAAA;AACT;AAIA,SAAS,aAAa,UAAA,EAAiE;AACrF,EAAA,MAAM,GAAA,uBAAU,GAAA,EAAwB;AACxC,EAAA,KAAA,MAAW,KAAK,UAAA,EAAY;AAC1B,IAAA,MAAM,MAAA,GAAS,gBAAgB,CAAC,CAAA;AAChC,IAAA,IAAI,WAAW,IAAA,EAAM,GAAA,CAAI,GAAA,CAAI,CAAA,CAAE,MAAM,MAAM,CAAA;AAAA,EAC7C;AACA,EAAA,OAAO,GAAA;AACT;AAUA,SAAS,aAAA,CAAc,MAAwB,GAAA,EAA2C;AACxF,EAAA,IAAI,IAAA,CAAK,mBAAmB,KAAA,IAAS,CAAC,aAAa,GAAA,CAAI,IAAA,CAAK,IAAI,CAAA,EAAG,OAAO,KAAA;AAC1E,EAAA,OAAO,CAAC,GAAA,CAAI,IAAA;AAAA,IACV,CAAC,KAAA,KACC,KAAA,CAAM,QAAA,KAAa,IAAA,CAAK,QAAA,IACxB,KAAA,CAAM,QAAA,KAAa,SAAA,IACnB,CAAC,YAAA,CAAa,GAAA,CAAI,MAAM,IAAI;AAAA,GAChC;AACF;AAKA,SAAS,wBAAwB,QAAA,EAAsD;AACrF,EAAA,IAAI,IAAA,GAAsB,IAAA;AAC1B,EAAA,IAAI,cAAA,GAAiB,CAAA;AACrB,EAAA,KAAA,MAAW,QAAQ,QAAA,EAAU;AAC3B,IAAA,IAAI,IAAA,CAAK,aAAa,SAAA,IAAa,CAAC,aAAa,GAAA,CAAI,IAAA,CAAK,IAAI,CAAA,EAAG;AACjE,IAAA,MAAM,CAAA,GAAI,OAAA,CAAQ,IAAA,CAAK,UAAU,CAAA;AACjC,IAAA,IAAI,IAAI,uBAAA,EAAyB;AACjC,IAAA,IAAI,IAAI,cAAA,EAAgB;AACtB,MAAA,cAAA,GAAiB,CAAA;AACjB,MAAA,IAAA,GAAO,IAAA,CAAK,QAAA;AAAA,IACd,CAAA,MAAA,IAAW,CAAA,KAAM,cAAA,IAAkB,IAAA,CAAK,aAAa,IAAA,EAAM;AAEzD,MAAA,IAAA,GAAO,IAAA;AAAA,IACT;AAAA,EACF;AACA,EAAA,OAAO,IAAA;AACT;AAQA,SAAS,MAAA,CACP,QACA,MAAA,EACiE;AACjE,EAAA,IAAI,IAAA,GAAsB,IAAA;AAC1B,EAAA,IAAI,SAAA,GAAY,CAAA;AAChB,EAAA,IAAI,WAAA,GAAc,CAAA;AAClB,EAAA,MAAM,cAAc,MAAA,KAAW,IAAA,GAAQ,OAAO,GAAA,CAAI,MAAM,KAAK,CAAA,GAAK,CAAA;AAElE,EAAA,KAAA,MAAW,CAAC,QAAA,EAAU,GAAG,CAAA,IAAK,MAAA,EAAQ;AAEpC,IAAA,MAAM,KAAA,GAAQ,WAAW,IAAA,IAAQ,QAAA,KAAa,SAAS,IAAA,CAAK,GAAA,CAAI,GAAA,EAAK,WAAW,CAAA,GAAI,GAAA;AACpF,IAAA,IAAI,QAAQ,SAAA,EAAW;AACrB,MAAA,WAAA,GAAc,SAAA;AACd,MAAA,SAAA,GAAY,KAAA;AACZ,MAAA,IAAA,GAAO,QAAA;AAAA,IACT,CAAA,MAAA,IAAW,QAAQ,WAAA,EAAa;AAC9B,MAAA,WAAA,GAAc,KAAA;AAAA,IAChB;AAAA,EACF;AAEA,EAAA,IAAI,WAAW,IAAA,IAAQ,IAAA,KAAS,UAAU,SAAA,KAAc,WAAA,IAAe,cAAc,CAAA,EAAG;AACtF,IAAA,WAAA,GAAc,SAAA;AACd,IAAA,IAAA,GAAO,MAAA;AACP,IAAA,SAAA,GAAY,WAAA;AAAA,EACd;AACA,EAAA,OAAO,EAAE,IAAA,EAAM,SAAA,EAAW,WAAA,EAAY;AACxC;AAEA,SAAS,QAAQ,KAAA,EAAuB;AACtC,EAAA,IAAI,CAAC,MAAA,CAAO,QAAA,CAAS,KAAK,GAAG,OAAO,CAAA;AACpC,EAAA,IAAI,KAAA,GAAQ,GAAG,OAAO,CAAA;AACtB,EAAA,IAAI,KAAA,GAAQ,GAAG,OAAO,CAAA;AACtB,EAAA,OAAO,KAAA;AACT","file":"chunk-JXGCCDSV.js","sourcesContent":["import type {\n Classification,\n LanguageEvidence,\n LanguageProfile,\n NonDiscriminatingScript,\n Weights,\n} from \"./types.js\";\nimport { normalizeBCP47 } from \"./internal/bcp47.js\";\nimport { scriptOfProfile, type ScriptName } from \"./internal/classify.js\";\n\nexport interface FuseOptions {\n weights?: Weights;\n /** The candidate roster. When present, incoming evidence tags are normalized\n * into it (`uk-UA` → `uk`, `ua` → `uk`) so context signals (page/header\n * locale) land on the same code the text rungs use. */\n candidates?: readonly LanguageProfile[];\n /** How to resolve a *non-discriminating* script read (one flagged\n * `discriminating: false` — its winning script owned by ≤1 roster candidate).\n * Default `\"candidate\"` keeps current behavior; `\"unknown\"` drops such a read\n * unless non-script evidence corroborates the same language. See\n * {@link NonDiscriminatingScript}. */\n nonDiscriminatingScript?: NonDiscriminatingScript;\n}\n\n/** Default per-kind weights. Clear lexical signal (script, explicit locale)\n * outweighs contextual signal (page tags, headers). Callers override per\n * `source` id or `kind` via {@link FuseOptions.weights}. */\nconst DEFAULT_KIND_WEIGHT: Record<string, number> = {\n \"title-script\": 1,\n \"explicit-locale\": 1,\n \"chrome-ai\": 1,\n \"source-prior\": 0.7,\n franc: 0.7,\n \"node-lang\": 0.65,\n \"http-content-language\": 0.6,\n \"meta-content-language\": 0.55,\n \"meta-og-locale\": 0.55,\n \"html-lang\": 0.5,\n};\n\n/** Evidence kinds that constitute *clear script evidence* — a verdict the text\n * classifier or an on-device model reached by actually reading the string. The\n * guard below forbids weaker page/header *context* from flipping these. */\nconst SCRIPT_KINDS = new Set<string>([\"title-script\", \"franc\", \"chrome-ai\"]);\n\n/** A script verdict this confident is treated as settled — context may add to it\n * but must not flip the winner to a different language. */\nconst SCRIPT_CONFIDENCE_FLOOR = 0.6;\n\nconst MIN_WINNING_SCORE = 0.35;\nconst MIN_MARGIN = 0.12;\n\n/**\n * Combine evidence into a single weighted verdict with an audit trail.\n *\n * Three steps:\n * 1. Normalize each item's language tag into the candidate roster (BCP-47:\n * `uk-UA`/`ua` → `uk`) so text, page, and header signals agree on a code.\n * 2. Weighted argmax over languages (caller weights override per `source`/`kind`).\n * 3. Apply the guard **context must never override clear script evidence**: when\n * the text classifier (or an on-device model) confidently read one language,\n * weaker page/header context for a *different* language cannot win — a\n * Ukrainian page chrome does not make a Latin/English title Ukrainian.\n */\nexport function fuse(\n evidence: readonly LanguageEvidence[],\n options: FuseOptions = {},\n): Classification {\n const weights = options.weights ?? {};\n const normalized = normalizeEvidence(evidence, options.candidates);\n\n // Under `\"unknown\"`, a non-discriminating script read scores nothing on its own\n // — it's dropped from the tally and the pin below — but stays in the trail. AND\n // context written in a *different script* than that title is dropped too: a\n // foreign-script title's language is never named by page/transport context in\n // another script (a Latin title on a Ukrainian page is a foreign title in a\n // Ukrainian UI, not a Ukrainian title). The full `normalized` set is still\n // returned as evidence.\n const scoring =\n options.nonDiscriminatingScript === \"unknown\"\n ? filterForUnknownMode(normalized, options.candidates)\n : normalized;\n\n const scores = new Map<string, number>();\n for (const item of scoring) {\n if (item.language === \"unknown\") continue;\n const weight =\n weights[item.source] ?? weights[item.kind] ?? DEFAULT_KIND_WEIGHT[item.kind] ?? 0.5;\n scores.set(item.language, (scores.get(item.language) ?? 0) + clamp01(item.confidence) * weight);\n }\n\n // The context-vs-script guard: a confident script read pins the winner.\n const pinned = confidentScriptLanguage(scoring);\n\n const { best, bestScore, secondScore } = argmax(scores, pinned);\n\n if (best === null || bestScore < MIN_WINNING_SCORE || bestScore - secondScore < MIN_MARGIN) {\n // A pinned script language still wins even on a thin margin — clear script\n // evidence is never demoted to \"unknown\" by competing context.\n if (pinned !== null && scores.has(pinned)) {\n const score = scores.get(pinned) ?? 0;\n return {\n language: pinned,\n confidence: clamp01(score / (score + 0.15)),\n evidence: [...normalized],\n };\n }\n return { language: \"unknown\", confidence: clamp01(bestScore), evidence: [...normalized] };\n }\n\n return {\n language: best,\n confidence: clamp01(bestScore / (bestScore + secondScore + 0.15)),\n evidence: [...normalized],\n };\n}\n\n/** Normalize each item's tag into the roster's code space (BCP-47-aware). Items\n * already `\"unknown\"` pass through untouched. Tags are BCP-47-normalized\n * (`en-US` → `en`, `ua` → `uk`) so text, page, and header signals land on the\n * same code. The normalized code is kept even when it falls outside the roster —\n * argmax simply won't favor an out-of-roster context tag, but it stays in the\n * audit trail.\n *\n * The roster is accepted (and reserved) so a future revision can fold roster\n * aliasing in without a signature change; today BCP-47 normalization alone\n * reconciles the codes the producers emit. */\nfunction normalizeEvidence(\n evidence: readonly LanguageEvidence[],\n _candidates: readonly LanguageProfile[] | undefined,\n): LanguageEvidence[] {\n return evidence.map((item) => {\n if (item.language === \"unknown\") return item;\n const normalized = normalizeBCP47(item.language) ?? item.language;\n if (normalized === item.language) return item;\n return { ...item, language: normalized };\n });\n}\n\n/**\n * The scoring set under `nonDiscriminatingScript: \"unknown\"`. Two cuts:\n *\n * 1. Drop every *neutralized* non-discriminating script read (see\n * {@link isNeutralized}) — it names a language only by being the lone\n * candidate in its script, with nothing corroborating it.\n * 2. Drop context (page/transport) evidence whose language is in a **different\n * script** than such a neutralized title. A foreign-script title's language\n * is not the page's language: a Latin title on a `lang=\"uk\"` page must not\n * resolve to `uk`. Same-script context (an explicit `en` `Content-Language`\n * for a Latin title) survives and may still name — or, among same-script\n * candidates, disambiguate — the title.\n *\n * The second cut needs each language's script, which is derived from the\n * candidate roster's alphabets. When `candidates` is absent the scripts can't be\n * derived, so the cut is skipped and behavior falls back to cut 1 alone (the\n * 0.3.0 behavior) — never throwing.\n */\nfunction filterForUnknownMode(\n normalized: readonly LanguageEvidence[],\n candidates: readonly LanguageProfile[] | undefined,\n): LanguageEvidence[] {\n const surviving = normalized.filter((item) => !isNeutralized(item, normalized));\n\n const titleScript = nonDiscriminatingTitleScript(normalized, candidates);\n if (titleScript === null) return surviving;\n\n const scriptOf = scriptByCode(candidates ?? []);\n return surviving.filter((item) => {\n // Keep the script reads themselves and anything whose script we can't place;\n // only cross-script *context* in a known, different script is excluded.\n if (SCRIPT_KINDS.has(item.kind) || item.language === \"unknown\") return true;\n const itemScript = scriptOf.get(item.language);\n return itemScript === undefined || itemScript === titleScript;\n });\n}\n\n/** The script of the title under `\"unknown\"` mode, or `null` when there is no\n * neutralized non-discriminating script read to anchor on (so no cross-script\n * cut applies) or the roster can't place that read's language. */\nfunction nonDiscriminatingTitleScript(\n normalized: readonly LanguageEvidence[],\n candidates: readonly LanguageProfile[] | undefined,\n): ScriptName | null {\n if (candidates === undefined) return null;\n const scriptOf = scriptByCode(candidates);\n for (const item of normalized) {\n if (isNeutralized(item, normalized)) {\n const script = scriptOf.get(item.language);\n if (script !== undefined) return script;\n }\n }\n return null;\n}\n\n/** Map each roster code to the script of its alphabet (Cyrillic/Latin). Codes\n * whose alphabet carries no Cyrillic/Latin letter are omitted. */\nfunction scriptByCode(candidates: readonly LanguageProfile[]): Map<string, ScriptName> {\n const map = new Map<string, ScriptName>();\n for (const c of candidates) {\n const script = scriptOfProfile(c);\n if (script !== null) map.set(c.code, script);\n }\n return map;\n}\n\n/**\n * Whether a non-discriminating script read should score nothing (mode\n * `\"unknown\"`). True when `item` is a script kind flagged `discriminating:\n * false` (its winning script is owned by ≤1 roster candidate) AND no *non-script*\n * evidence corroborates its language. Corroboration must come from context kinds\n * (page tags, headers): two lone-candidate script reads agreeing is still two\n * defaults, not real evidence — so script kinds never corroborate one another.\n */\nfunction isNeutralized(item: LanguageEvidence, all: readonly LanguageEvidence[]): boolean {\n if (item.discriminating !== false || !SCRIPT_KINDS.has(item.kind)) return false;\n return !all.some(\n (other) =>\n other.language === item.language &&\n other.language !== \"unknown\" &&\n !SCRIPT_KINDS.has(other.kind),\n );\n}\n\n/** The language of a *clear script* read confident enough to pin the verdict, or\n * `null` when none qualifies. When two script reads disagree, the higher-\n * confidence one pins (a tie leaves nothing pinned — argmax decides normally). */\nfunction confidentScriptLanguage(evidence: readonly LanguageEvidence[]): string | null {\n let best: string | null = null;\n let bestConfidence = 0;\n for (const item of evidence) {\n if (item.language === \"unknown\" || !SCRIPT_KINDS.has(item.kind)) continue;\n const c = clamp01(item.confidence);\n if (c < SCRIPT_CONFIDENCE_FLOOR) continue;\n if (c > bestConfidence) {\n bestConfidence = c;\n best = item.language;\n } else if (c === bestConfidence && item.language !== best) {\n // Two equally-confident script reads for different languages — ambiguous.\n best = null;\n }\n }\n return best;\n}\n\n/**\n * Weighted argmax. When `pinned` is set (a confident script language), any\n * *other* language's score may only come from context kinds; that score is\n * capped so it can never exceed the pinned language. This enforces the guard\n * without discarding the context from the audit trail.\n */\nfunction argmax(\n scores: Map<string, number>,\n pinned: string | null,\n): { best: string | null; bestScore: number; secondScore: number } {\n let best: string | null = null;\n let bestScore = 0;\n let secondScore = 0;\n const pinnedScore = pinned !== null ? (scores.get(pinned) ?? 0) : 0;\n\n for (const [language, raw] of scores) {\n // Guard: a non-pinned language cannot out-score the pinned one.\n const score = pinned !== null && language !== pinned ? Math.min(raw, pinnedScore) : raw;\n if (score > bestScore) {\n secondScore = bestScore;\n bestScore = score;\n best = language;\n } else if (score > secondScore) {\n secondScore = score;\n }\n }\n // On a pinned tie (pinned capped equal to a context language), prefer pinned.\n if (pinned !== null && best !== pinned && bestScore === pinnedScore && pinnedScore > 0) {\n secondScore = bestScore;\n best = pinned;\n bestScore = pinnedScore;\n }\n return { best, bestScore, secondScore };\n}\n\nfunction clamp01(value: number): number {\n if (!Number.isFinite(value)) return 0;\n if (value < 0) return 0;\n if (value > 1) return 1;\n return value;\n}\n"]}
|
package/dist/classify.d.ts
CHANGED
package/dist/cyrillic.d.ts
CHANGED
|
@@ -3,11 +3,18 @@
|
|
|
3
3
|
*
|
|
4
4
|
* Where {@link classifyBySnippet} (`langtell/classify`) scores a snippet
|
|
5
5
|
* *relative to a candidate roster you pass in*, this is the opposite trade: a
|
|
6
|
-
* fixed, zero-config discriminator for the four Cyrillic languages
|
|
7
|
-
*
|
|
8
|
-
* letters distinctive to each, with no profiles, no tokenization, and
|
|
9
|
-
* Reach for it when you just need "is this Russian / is this
|
|
10
|
-
* hot path and don't want to assemble a candidate set.
|
|
6
|
+
* fixed, zero-config discriminator for the four Cyrillic languages this fast-path
|
|
7
|
+
* positively identifies — Ukrainian, Russian, Belarusian, Bulgarian — decided
|
|
8
|
+
* purely by letters distinctive to each, with no profiles, no tokenization, and
|
|
9
|
+
* no franc. Reach for it when you just need "is this Russian / is this
|
|
10
|
+
* Ukrainian?" on a hot path and don't want to assemble a candidate set.
|
|
11
|
+
*
|
|
12
|
+
* Other Cyrillic languages langtell now profiles (Serbian, Macedonian, Kazakh)
|
|
13
|
+
* are deliberately NOT positively detected here — that is the candidate-relative
|
|
14
|
+
* classifier's job. This module only takes care not to MISLABEL them: text
|
|
15
|
+
* carrying letters distinctive to sr/mk/kk (ђ ћ џ ѓ ќ ѕ љ њ ј, or the Kazakh
|
|
16
|
+
* Turkic set ә ғ қ ң ө ұ ү һ) returns `"unknown"` so the snippet escalates to
|
|
17
|
+
* the classifier instead of being silently called Russian by the fallbacks.
|
|
11
18
|
*
|
|
12
19
|
* Each language carries letters the others (mostly) don't:
|
|
13
20
|
* Ukrainian — і ї є ґ
|
package/dist/cyrillic.js
CHANGED
|
@@ -4,6 +4,7 @@ var RU_DISTINCTIVE = /[ыё]/gi;
|
|
|
4
4
|
var BE_DISTINCTIVE = /ў/gi;
|
|
5
5
|
var HARD_SIGN = /ъ/gi;
|
|
6
6
|
var E_OBOROT = /э/gi;
|
|
7
|
+
var SIBLING_DISTINCTIVE = /[ђћџѓќѕљњјәғқңөұүһ]/gi;
|
|
7
8
|
var CYRILLIC = /[\u0400-\u04FF]/g;
|
|
8
9
|
var MIN_CYRILLIC_FOR_FALLBACK = 10;
|
|
9
10
|
var MIN_LEN_FOR_BG = 10;
|
|
@@ -17,14 +18,18 @@ function countSignals(text) {
|
|
|
17
18
|
beScore: count(text, BE_DISTINCTIVE),
|
|
18
19
|
hardSigns: count(text, HARD_SIGN),
|
|
19
20
|
eOborot: count(text, E_OBOROT),
|
|
20
|
-
cyrillicCount: count(text, CYRILLIC)
|
|
21
|
+
cyrillicCount: count(text, CYRILLIC),
|
|
22
|
+
siblingDistinctive: count(text, SIBLING_DISTINCTIVE)
|
|
21
23
|
};
|
|
22
24
|
}
|
|
23
25
|
function detectCyrillicLanguage(text) {
|
|
24
|
-
const { ukScore, ruDistinctive, beScore, hardSigns, eOborot, cyrillicCount } = countSignals(text);
|
|
26
|
+
const { ukScore, ruDistinctive, beScore, hardSigns, eOborot, cyrillicCount, siblingDistinctive } = countSignals(text);
|
|
25
27
|
if (beScore > 0) {
|
|
26
28
|
return { language: "be", ukScore, ruScore: ruDistinctive };
|
|
27
29
|
}
|
|
30
|
+
if (siblingDistinctive > 0) {
|
|
31
|
+
return { language: "unknown", ukScore, ruScore: ruDistinctive };
|
|
32
|
+
}
|
|
28
33
|
if (ukScore > 0 && ruDistinctive > 0) {
|
|
29
34
|
if (ukScore === ruDistinctive) {
|
|
30
35
|
return { language: "unknown", ukScore, ruScore: ruDistinctive };
|
package/dist/cyrillic.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/cyrillic.ts"],"names":[],"mappings":";AAiCA,IAAM,cAAA,GAAiB,UAAA;AACvB,IAAM,cAAA,GAAiB,QAAA;AACvB,IAAM,cAAA,GAAiB,KAAA;AACvB,IAAM,SAAA,GAAY,KAAA;AAClB,IAAM,QAAA,GAAW,KAAA;AAGjB,IAAM,QAAA,GAAW,kBAAA;AAIjB,IAAM,yBAAA,GAA4B,EAAA;AAGlC,IAAM,cAAA,GAAiB,EAAA;AAWvB,SAAS,KAAA,CAAM,MAAc,EAAA,EAAoB;AAC/C,EAAA,OAAA,CAAQ,IAAA,CAAK,KAAA,CAAM,EAAE,CAAA,IAAK,EAAC,EAAG,MAAA;AAChC;AAcA,SAAS,aAAa,IAAA,EAAuB;AAC3C,EAAA,OAAO;AAAA,IACL,OAAA,EAAS,KAAA,CAAM,IAAA,EAAM,cAAc,CAAA;AAAA,IACnC,aAAA,EAAe,KAAA,CAAM,IAAA,EAAM,cAAc,CAAA;AAAA,IACzC,OAAA,EAAS,KAAA,CAAM,IAAA,EAAM,cAAc,CAAA;AAAA,IACnC,SAAA,EAAW,KAAA,CAAM,IAAA,EAAM,SAAS,CAAA;AAAA,IAChC,OAAA,EAAS,KAAA,CAAM,IAAA,EAAM,QAAQ,CAAA;AAAA,IAC7B,aAAA,EAAe,KAAA,CAAM,IAAA,EAAM,QAAQ;AAAA,GACrC;AACF;AAOO,SAAS,uBAAuB,IAAA,EAA+B;AACpE,EAAA,MAAM,EAAE,SAAS,aAAA,EAAe,OAAA,EAAS,WAAW,OAAA,EAAS,aAAA,EAAc,GAAI,YAAA,CAAa,IAAI,CAAA;AAGhG,EAAA,IAAI,UAAU,CAAA,EAAG;AACf,IAAA,OAAO,EAAE,QAAA,EAAU,IAAA,EAAM,OAAA,EAAS,SAAS,aAAA,EAAc;AAAA,EAC3D;AAIA,EAAA,IAAI,OAAA,GAAU,CAAA,IAAK,aAAA,GAAgB,CAAA,EAAG;AACpC,IAAA,IAAI,YAAY,aAAA,EAAe;AAC7B,MAAA,OAAO,EAAE,QAAA,EAAU,SAAA,EAAW,OAAA,EAAS,SAAS,aAAA,EAAc;AAAA,IAChE;AACA,IAAA,OAAO;AAAA,MACL,QAAA,EAAU,OAAA,GAAU,aAAA,GAAgB,IAAA,GAAO,IAAA;AAAA,MAC3C,OAAA;AAAA,MACA,OAAA,EAAS;AAAA,KACX;AAAA,EACF;AAGA,EAAA,IAAI,gBAAgB,CAAA,EAAG;AACrB,IAAA,OAAO,EAAE,QAAA,EAAU,IAAA,EAAM,OAAA,EAAS,SAAS,aAAA,EAAc;AAAA,EAC3D;AAGA,EAAA,IAAI,UAAU,CAAA,EAAG;AACf,IAAA,OAAO,EAAE,QAAA,EAAU,IAAA,EAAM,OAAA,EAAS,SAAS,CAAA,EAAE;AAAA,EAC/C;AAKA,EAAA,IAAI,SAAA,IAAa,CAAA,IAAK,IAAA,CAAK,MAAA,IAAU,cAAA,EAAgB;AACnD,IAAA,OAAO,EAAE,QAAA,EAAU,IAAA,EAAM,OAAA,EAAS,CAAA,EAAG,SAAS,SAAA,EAAU;AAAA,EAC1D;AAKA,EAAA,IAAI,aAAA,IAAiB,yBAAA,IAA6B,OAAA,KAAY,CAAA,EAAG;AAC/D,IAAA,OAAO,EAAE,QAAA,EAAU,IAAA,EAAM,OAAA,EAAS,CAAA,EAAG,SAAS,CAAA,EAAE;AAAA,EAClD;AAIA,EAAA,IAAI,SAAA,GAAY,CAAA,IAAK,OAAA,KAAY,CAAA,EAAG;AAClC,IAAA,OAAO,EAAE,QAAA,EAAU,IAAA,EAAM,OAAA,EAAS,CAAA,EAAG,SAAS,SAAA,EAAU;AAAA,EAC1D;AAEA,EAAA,OAAO,EAAE,QAAA,EAAU,SAAA,EAAW,OAAA,EAAS,CAAA,EAAG,SAAS,CAAA,EAAE;AACvD;AAIO,SAAS,UAAU,IAAA,EAAuB;AAC/C,EAAA,OAAO,sBAAA,CAAuB,IAAI,CAAA,CAAE,QAAA,KAAa,IAAA;AACnD;AAIO,SAAS,YAAY,IAAA,EAAuB;AACjD,EAAA,OAAO,sBAAA,CAAuB,IAAI,CAAA,CAAE,QAAA,KAAa,IAAA;AACnD","file":"cyrillic.js","sourcesContent":["/**\n * `langtell/cyrillic` — a cheap, roster-free Cyrillic language fast-path.\n *\n * Where {@link classifyBySnippet} (`langtell/classify`) scores a snippet\n * *relative to a candidate roster you pass in*, this is the opposite trade: a\n * fixed, zero-config discriminator for the four Cyrillic languages langtell\n * profiles — Ukrainian, Russian, Belarusian, Bulgarian — decided purely by\n * letters distinctive to each, with no profiles, no tokenization, and no franc.\n * Reach for it when you just need \"is this Russian / is this Ukrainian?\" on a\n * hot path and don't want to assemble a candidate set.\n *\n * Each language carries letters the others (mostly) don't:\n * Ukrainian — і ї є ґ\n * Russian — ы ё (ъ э are shared with bg/be and handled separately)\n * Belarusian — ў (uniquely Belarusian)\n * Bulgarian — ъ used as a vowel in nearly every word\n *\n * The shared letters `ъ` and `э` need care: `подъезд` is Russian, `съм\n * българин` is Bulgarian — both contain `ъ`. We disambiguate by density and\n * length: a single `ъ` in a short snippet leans RU; multiple `ъ` in longer text\n * leans BG. A lone `э` with no other distinctives stays `\"unknown\"` rather than\n * silently guessing (Russian uses it in loanwords/`Это`, Belarusian in\n * `гэта`/`сэрца`).\n *\n * Zero-dependency and side-effect-free. The cheap heuristic stays cheap; if a\n * use case needs more than letter signals can give, escalate to\n * {@link classifyBySnippet} or a franc-backed source.\n */\n\n/** The four Cyrillic languages this fast-path tells apart, plus the `\"unknown\"`\n * sentinel when letter signals are insufficient. */\nexport type CyrillicLanguage = \"uk\" | \"ru\" | \"be\" | \"bg\" | \"unknown\";\n\nconst UK_DISTINCTIVE = /[іїєґ]/gi;\nconst RU_DISTINCTIVE = /[ыё]/gi;\nconst BE_DISTINCTIVE = /ў/gi;\nconst HARD_SIGN = /ъ/gi;\nconst E_OBOROT = /э/gi;\n// U+0400–U+04FF is the Cyrillic Unicode block; written as explicit \\u escapes\n// so the range bounds are unambiguous (regexp/no-obscure-range).\nconst CYRILLIC = /[\\u0400-\\u04FF]/g;\n\n/** Minimum Cyrillic-letter count before the fallback guesses a language. Below\n * this, a short snippet (`Привет`, `Хочу`) is too ambiguous to act on. */\nconst MIN_CYRILLIC_FOR_FALLBACK = 10;\n/** Minimum text length before `ъ`-density is read as Bulgarian. Single-word\n * Russian samples like `подъезд` (7 chars) would otherwise misclassify. */\nconst MIN_LEN_FOR_BG = 10;\n\n/** The verdict from {@link detectCyrillicLanguage}: the chosen language plus the\n * raw distinctive-letter tallies that drove a uk-vs-ru decision (informational;\n * `ruScore` carries the deciding count for the ru/bg fallbacks too). */\nexport interface CyrillicVerdict {\n language: CyrillicLanguage;\n ukScore: number;\n ruScore: number;\n}\n\nfunction count(text: string, re: RegExp): number {\n return (text.match(re) ?? []).length;\n}\n\n/** Raw letter-signal tallies over `text`, one pass per distinctive class.\n * Gathered up front so {@link detectCyrillicLanguage} reads as a pure decision\n * cascade over these counts rather than interleaving counting and branching. */\ninterface Signals {\n ukScore: number;\n ruDistinctive: number;\n beScore: number;\n hardSigns: number;\n eOborot: number;\n cyrillicCount: number;\n}\n\nfunction countSignals(text: string): Signals {\n return {\n ukScore: count(text, UK_DISTINCTIVE),\n ruDistinctive: count(text, RU_DISTINCTIVE),\n beScore: count(text, BE_DISTINCTIVE),\n hardSigns: count(text, HARD_SIGN),\n eOborot: count(text, E_OBOROT),\n cyrillicCount: count(text, CYRILLIC),\n };\n}\n\n/**\n * Identify the Cyrillic language of `text` by distinctive letters, returning the\n * chosen language and the uk/ru tallies behind it. `\"unknown\"` when there is no\n * Cyrillic evidence, on a uk/ru tie, or when only an ambiguous `э` is present.\n */\nexport function detectCyrillicLanguage(text: string): CyrillicVerdict {\n const { ukScore, ruDistinctive, beScore, hardSigns, eOborot, cyrillicCount } = countSignals(text);\n\n // ў is uniquely Belarusian — strongest single signal.\n if (beScore > 0) {\n return { language: \"be\", ukScore, ruScore: ruDistinctive };\n }\n\n // Both UA and RU evidence present — a tie is \"unknown\", not a silent UA call\n // (which would misclassify Belarusian/Bulgarian whenever і or ё balances out).\n if (ukScore > 0 && ruDistinctive > 0) {\n if (ukScore === ruDistinctive) {\n return { language: \"unknown\", ukScore, ruScore: ruDistinctive };\n }\n return {\n language: ukScore > ruDistinctive ? \"uk\" : \"ru\",\n ukScore,\n ruScore: ruDistinctive,\n };\n }\n\n // Distinctive RU letters (ы, ё) with no UA evidence — unambiguously RU.\n if (ruDistinctive > 0) {\n return { language: \"ru\", ukScore, ruScore: ruDistinctive };\n }\n\n // Distinctive UA letters with no RU distinctives — UA.\n if (ukScore > 0) {\n return { language: \"uk\", ukScore, ruScore: 0 };\n }\n\n // Bulgarian: ъ as a vowel — multiple occurrences in non-trivial text, no UA /\n // RU distinctives. Length guard keeps short Russian compounds like `подъезд`\n // from sliding into BG.\n if (hardSigns >= 2 && text.length >= MIN_LEN_FOR_BG) {\n return { language: \"bg\", ukScore: 0, ruScore: hardSigns };\n }\n\n // Russian fallback: substantial Cyrillic with no UA, no BE distinctives, and\n // no э (which signals possible Belarusian). Catches short-but-clear Russian\n // text with none of ы/ё/ъ/э (`Привет, мир`, `Здравствуйте, меня зовут …`).\n if (cyrillicCount >= MIN_CYRILLIC_FOR_FALLBACK && eOborot === 0) {\n return { language: \"ru\", ukScore: 0, ruScore: 0 };\n }\n\n // Lone ъ (no э, no other distinctives) in shorter text — RU compound-word\n // pattern (подъезд, объект, съезд).\n if (hardSigns > 0 && eOborot === 0) {\n return { language: \"ru\", ukScore: 0, ruScore: hardSigns };\n }\n\n return { language: \"unknown\", ukScore: 0, ruScore: 0 };\n}\n\n/** Convenience predicate: `true` iff {@link detectCyrillicLanguage} calls `text`\n * Russian. */\nexport function isRussian(text: string): boolean {\n return detectCyrillicLanguage(text).language === \"ru\";\n}\n\n/** Convenience predicate: `true` iff {@link detectCyrillicLanguage} calls `text`\n * Ukrainian. */\nexport function isUkrainian(text: string): boolean {\n return detectCyrillicLanguage(text).language === \"uk\";\n}\n"]}
|
|
1
|
+
{"version":3,"sources":["../src/cyrillic.ts"],"names":[],"mappings":";AAwCA,IAAM,cAAA,GAAiB,UAAA;AACvB,IAAM,cAAA,GAAiB,QAAA;AACvB,IAAM,cAAA,GAAiB,KAAA;AACvB,IAAM,SAAA,GAAY,KAAA;AAClB,IAAM,QAAA,GAAW,KAAA;AAOjB,IAAM,mBAAA,GAAsB,uBAAA;AAG5B,IAAM,QAAA,GAAW,kBAAA;AAIjB,IAAM,yBAAA,GAA4B,EAAA;AAGlC,IAAM,cAAA,GAAiB,EAAA;AAWvB,SAAS,KAAA,CAAM,MAAc,EAAA,EAAoB;AAC/C,EAAA,OAAA,CAAQ,IAAA,CAAK,KAAA,CAAM,EAAE,CAAA,IAAK,EAAC,EAAG,MAAA;AAChC;AAiBA,SAAS,aAAa,IAAA,EAAuB;AAC3C,EAAA,OAAO;AAAA,IACL,OAAA,EAAS,KAAA,CAAM,IAAA,EAAM,cAAc,CAAA;AAAA,IACnC,aAAA,EAAe,KAAA,CAAM,IAAA,EAAM,cAAc,CAAA;AAAA,IACzC,OAAA,EAAS,KAAA,CAAM,IAAA,EAAM,cAAc,CAAA;AAAA,IACnC,SAAA,EAAW,KAAA,CAAM,IAAA,EAAM,SAAS,CAAA;AAAA,IAChC,OAAA,EAAS,KAAA,CAAM,IAAA,EAAM,QAAQ,CAAA;AAAA,IAC7B,aAAA,EAAe,KAAA,CAAM,IAAA,EAAM,QAAQ,CAAA;AAAA,IACnC,kBAAA,EAAoB,KAAA,CAAM,IAAA,EAAM,mBAAmB;AAAA,GACrD;AACF;AAOO,SAAS,uBAAuB,IAAA,EAA+B;AACpE,EAAA,MAAM,EAAE,OAAA,EAAS,aAAA,EAAe,OAAA,EAAS,SAAA,EAAW,SAAS,aAAA,EAAe,kBAAA,EAAmB,GAC7F,YAAA,CAAa,IAAI,CAAA;AAGnB,EAAA,IAAI,UAAU,CAAA,EAAG;AACf,IAAA,OAAO,EAAE,QAAA,EAAU,IAAA,EAAM,OAAA,EAAS,SAAS,aAAA,EAAc;AAAA,EAC3D;AAUA,EAAA,IAAI,qBAAqB,CAAA,EAAG;AAC1B,IAAA,OAAO,EAAE,QAAA,EAAU,SAAA,EAAW,OAAA,EAAS,SAAS,aAAA,EAAc;AAAA,EAChE;AAIA,EAAA,IAAI,OAAA,GAAU,CAAA,IAAK,aAAA,GAAgB,CAAA,EAAG;AACpC,IAAA,IAAI,YAAY,aAAA,EAAe;AAC7B,MAAA,OAAO,EAAE,QAAA,EAAU,SAAA,EAAW,OAAA,EAAS,SAAS,aAAA,EAAc;AAAA,IAChE;AACA,IAAA,OAAO;AAAA,MACL,QAAA,EAAU,OAAA,GAAU,aAAA,GAAgB,IAAA,GAAO,IAAA;AAAA,MAC3C,OAAA;AAAA,MACA,OAAA,EAAS;AAAA,KACX;AAAA,EACF;AAGA,EAAA,IAAI,gBAAgB,CAAA,EAAG;AACrB,IAAA,OAAO,EAAE,QAAA,EAAU,IAAA,EAAM,OAAA,EAAS,SAAS,aAAA,EAAc;AAAA,EAC3D;AAGA,EAAA,IAAI,UAAU,CAAA,EAAG;AACf,IAAA,OAAO,EAAE,QAAA,EAAU,IAAA,EAAM,OAAA,EAAS,SAAS,CAAA,EAAE;AAAA,EAC/C;AAKA,EAAA,IAAI,SAAA,IAAa,CAAA,IAAK,IAAA,CAAK,MAAA,IAAU,cAAA,EAAgB;AACnD,IAAA,OAAO,EAAE,QAAA,EAAU,IAAA,EAAM,OAAA,EAAS,CAAA,EAAG,SAAS,SAAA,EAAU;AAAA,EAC1D;AAKA,EAAA,IAAI,aAAA,IAAiB,yBAAA,IAA6B,OAAA,KAAY,CAAA,EAAG;AAC/D,IAAA,OAAO,EAAE,QAAA,EAAU,IAAA,EAAM,OAAA,EAAS,CAAA,EAAG,SAAS,CAAA,EAAE;AAAA,EAClD;AAIA,EAAA,IAAI,SAAA,GAAY,CAAA,IAAK,OAAA,KAAY,CAAA,EAAG;AAClC,IAAA,OAAO,EAAE,QAAA,EAAU,IAAA,EAAM,OAAA,EAAS,CAAA,EAAG,SAAS,SAAA,EAAU;AAAA,EAC1D;AAEA,EAAA,OAAO,EAAE,QAAA,EAAU,SAAA,EAAW,OAAA,EAAS,CAAA,EAAG,SAAS,CAAA,EAAE;AACvD;AAIO,SAAS,UAAU,IAAA,EAAuB;AAC/C,EAAA,OAAO,sBAAA,CAAuB,IAAI,CAAA,CAAE,QAAA,KAAa,IAAA;AACnD;AAIO,SAAS,YAAY,IAAA,EAAuB;AACjD,EAAA,OAAO,sBAAA,CAAuB,IAAI,CAAA,CAAE,QAAA,KAAa,IAAA;AACnD","file":"cyrillic.js","sourcesContent":["/**\n * `langtell/cyrillic` — a cheap, roster-free Cyrillic language fast-path.\n *\n * Where {@link classifyBySnippet} (`langtell/classify`) scores a snippet\n * *relative to a candidate roster you pass in*, this is the opposite trade: a\n * fixed, zero-config discriminator for the four Cyrillic languages this fast-path\n * positively identifies — Ukrainian, Russian, Belarusian, Bulgarian — decided\n * purely by letters distinctive to each, with no profiles, no tokenization, and\n * no franc. Reach for it when you just need \"is this Russian / is this\n * Ukrainian?\" on a hot path and don't want to assemble a candidate set.\n *\n * Other Cyrillic languages langtell now profiles (Serbian, Macedonian, Kazakh)\n * are deliberately NOT positively detected here — that is the candidate-relative\n * classifier's job. This module only takes care not to MISLABEL them: text\n * carrying letters distinctive to sr/mk/kk (ђ ћ џ ѓ ќ ѕ љ њ ј, or the Kazakh\n * Turkic set ә ғ қ ң ө ұ ү һ) returns `\"unknown\"` so the snippet escalates to\n * the classifier instead of being silently called Russian by the fallbacks.\n *\n * Each language carries letters the others (mostly) don't:\n * Ukrainian — і ї є ґ\n * Russian — ы ё (ъ э are shared with bg/be and handled separately)\n * Belarusian — ў (uniquely Belarusian)\n * Bulgarian — ъ used as a vowel in nearly every word\n *\n * The shared letters `ъ` and `э` need care: `подъезд` is Russian, `съм\n * българин` is Bulgarian — both contain `ъ`. We disambiguate by density and\n * length: a single `ъ` in a short snippet leans RU; multiple `ъ` in longer text\n * leans BG. A lone `э` with no other distinctives stays `\"unknown\"` rather than\n * silently guessing (Russian uses it in loanwords/`Это`, Belarusian in\n * `гэта`/`сэрца`).\n *\n * Zero-dependency and side-effect-free. The cheap heuristic stays cheap; if a\n * use case needs more than letter signals can give, escalate to\n * {@link classifyBySnippet} or a franc-backed source.\n */\n\n/** The four Cyrillic languages this fast-path tells apart, plus the `\"unknown\"`\n * sentinel when letter signals are insufficient. */\nexport type CyrillicLanguage = \"uk\" | \"ru\" | \"be\" | \"bg\" | \"unknown\";\n\nconst UK_DISTINCTIVE = /[іїєґ]/gi;\nconst RU_DISTINCTIVE = /[ыё]/gi;\nconst BE_DISTINCTIVE = /ў/gi;\nconst HARD_SIGN = /ъ/gi;\nconst E_OBOROT = /э/gi;\n// Letters distinctive to Serbian / Macedonian / Kazakh — none of which this\n// fast-path positively detects. Their presence only triggers a bail to\n// `\"unknown\"` (escalate to the classifier) so the Russian fallbacks below never\n// mislabel them. ј/љ/њ/џ are shared by sr+mk; ђ/ћ are Serbian, ѓ/ќ/ѕ Macedonian,\n// and ә/ғ/қ/ң/ө/ұ/ү/һ the distinctive Kazakh Turkic set (і is excluded here — it\n// is already Ukrainian-distinctive and handled above).\nconst SIBLING_DISTINCTIVE = /[ђћџѓќѕљњјәғқңөұүһ]/gi;\n// U+0400–U+04FF is the Cyrillic Unicode block; written as explicit \\u escapes\n// so the range bounds are unambiguous (regexp/no-obscure-range).\nconst CYRILLIC = /[\\u0400-\\u04FF]/g;\n\n/** Minimum Cyrillic-letter count before the fallback guesses a language. Below\n * this, a short snippet (`Привет`, `Хочу`) is too ambiguous to act on. */\nconst MIN_CYRILLIC_FOR_FALLBACK = 10;\n/** Minimum text length before `ъ`-density is read as Bulgarian. Single-word\n * Russian samples like `подъезд` (7 chars) would otherwise misclassify. */\nconst MIN_LEN_FOR_BG = 10;\n\n/** The verdict from {@link detectCyrillicLanguage}: the chosen language plus the\n * raw distinctive-letter tallies that drove a uk-vs-ru decision (informational;\n * `ruScore` carries the deciding count for the ru/bg fallbacks too). */\nexport interface CyrillicVerdict {\n language: CyrillicLanguage;\n ukScore: number;\n ruScore: number;\n}\n\nfunction count(text: string, re: RegExp): number {\n return (text.match(re) ?? []).length;\n}\n\n/** Raw letter-signal tallies over `text`, one pass per distinctive class.\n * Gathered up front so {@link detectCyrillicLanguage} reads as a pure decision\n * cascade over these counts rather than interleaving counting and branching. */\ninterface Signals {\n ukScore: number;\n ruDistinctive: number;\n beScore: number;\n hardSigns: number;\n eOborot: number;\n cyrillicCount: number;\n /** Count of letters distinctive to sr/mk/kk (see {@link SIBLING_DISTINCTIVE}).\n * Non-zero ⇒ bail to `\"unknown\"` before the Russian fallbacks. */\n siblingDistinctive: number;\n}\n\nfunction countSignals(text: string): Signals {\n return {\n ukScore: count(text, UK_DISTINCTIVE),\n ruDistinctive: count(text, RU_DISTINCTIVE),\n beScore: count(text, BE_DISTINCTIVE),\n hardSigns: count(text, HARD_SIGN),\n eOborot: count(text, E_OBOROT),\n cyrillicCount: count(text, CYRILLIC),\n siblingDistinctive: count(text, SIBLING_DISTINCTIVE),\n };\n}\n\n/**\n * Identify the Cyrillic language of `text` by distinctive letters, returning the\n * chosen language and the uk/ru tallies behind it. `\"unknown\"` when there is no\n * Cyrillic evidence, on a uk/ru tie, or when only an ambiguous `э` is present.\n */\nexport function detectCyrillicLanguage(text: string): CyrillicVerdict {\n const { ukScore, ruDistinctive, beScore, hardSigns, eOborot, cyrillicCount, siblingDistinctive } =\n countSignals(text);\n\n // ў is uniquely Belarusian — strongest single signal.\n if (beScore > 0) {\n return { language: \"be\", ukScore, ruScore: ruDistinctive };\n }\n\n // Mislabel guard: letters distinctive to Serbian/Macedonian/Kazakh, none of\n // which this fast-path positively detects. Bail to `\"unknown\"` so the snippet\n // escalates to the candidate-relative classifier instead of being silently\n // labelled by the uk/ru/bg logic below. This fires BEFORE the uk/ru checks on\n // purpose: Kazakh shares і with Ukrainian and ы with Russian (тілі, тым), so a\n // Kazakh Turkic letter (ә/ғ/қ/…) must outrank a stray і/ы rather than letting\n // the text be called Ukrainian or Russian. Genuine uk/ru/be/bg text carries\n // none of these letters, so their detection above/below is unaffected.\n if (siblingDistinctive > 0) {\n return { language: \"unknown\", ukScore, ruScore: ruDistinctive };\n }\n\n // Both UA and RU evidence present — a tie is \"unknown\", not a silent UA call\n // (which would misclassify Belarusian/Bulgarian whenever і or ё balances out).\n if (ukScore > 0 && ruDistinctive > 0) {\n if (ukScore === ruDistinctive) {\n return { language: \"unknown\", ukScore, ruScore: ruDistinctive };\n }\n return {\n language: ukScore > ruDistinctive ? \"uk\" : \"ru\",\n ukScore,\n ruScore: ruDistinctive,\n };\n }\n\n // Distinctive RU letters (ы, ё) with no UA evidence — unambiguously RU.\n if (ruDistinctive > 0) {\n return { language: \"ru\", ukScore, ruScore: ruDistinctive };\n }\n\n // Distinctive UA letters with no RU distinctives — UA.\n if (ukScore > 0) {\n return { language: \"uk\", ukScore, ruScore: 0 };\n }\n\n // Bulgarian: ъ as a vowel — multiple occurrences in non-trivial text, no UA /\n // RU distinctives. Length guard keeps short Russian compounds like `подъезд`\n // from sliding into BG.\n if (hardSigns >= 2 && text.length >= MIN_LEN_FOR_BG) {\n return { language: \"bg\", ukScore: 0, ruScore: hardSigns };\n }\n\n // Russian fallback: substantial Cyrillic with no UA, no BE distinctives, and\n // no э (which signals possible Belarusian). Catches short-but-clear Russian\n // text with none of ы/ё/ъ/э (`Привет, мир`, `Здравствуйте, меня зовут …`).\n if (cyrillicCount >= MIN_CYRILLIC_FOR_FALLBACK && eOborot === 0) {\n return { language: \"ru\", ukScore: 0, ruScore: 0 };\n }\n\n // Lone ъ (no э, no other distinctives) in shorter text — RU compound-word\n // pattern (подъезд, объект, съезд).\n if (hardSigns > 0 && eOborot === 0) {\n return { language: \"ru\", ukScore: 0, ruScore: hardSigns };\n }\n\n return { language: \"unknown\", ukScore: 0, ruScore: 0 };\n}\n\n/** Convenience predicate: `true` iff {@link detectCyrillicLanguage} calls `text`\n * Russian. */\nexport function isRussian(text: string): boolean {\n return detectCyrillicLanguage(text).language === \"ru\";\n}\n\n/** Convenience predicate: `true` iff {@link detectCyrillicLanguage} calls `text`\n * Ukrainian. */\nexport function isUkrainian(text: string): boolean {\n return detectCyrillicLanguage(text).language === \"uk\";\n}\n"]}
|
package/dist/franc.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { L as LanguageProfile, S as SyncSource, a as LanguageEvidence } from './types-
|
|
1
|
+
import { L as LanguageProfile, S as SyncSource, a as LanguageEvidence } from './types-DfHZ5hkh.js';
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
4
|
* Producer: the franc trigram backstop over `text`, scoped to `candidates`.
|
package/dist/fuse.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { W as Weights, L as LanguageProfile, N as NonDiscriminatingScript, a as LanguageEvidence, C as Classification } from './types-
|
|
1
|
+
import { W as Weights, L as LanguageProfile, N as NonDiscriminatingScript, a as LanguageEvidence, C as Classification } from './types-DfHZ5hkh.js';
|
|
2
2
|
|
|
3
3
|
interface FuseOptions {
|
|
4
4
|
weights?: Weights;
|
package/dist/fuse.js
CHANGED
package/dist/headers.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { H as HeaderBag, a as LanguageEvidence } from './types-
|
|
1
|
+
import { H as HeaderBag, a as LanguageEvidence } from './types-DfHZ5hkh.js';
|
|
2
2
|
|
|
3
3
|
/** Producer: the HTTP `Content-Language` response header. */
|
|
4
4
|
declare function evidenceFromHeaders(headers: HeaderBag | undefined): LanguageEvidence[];
|
package/dist/html.d.ts
CHANGED
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { E as EvidenceSource, D as DetectorConfig, c as DetectFn } from './types-
|
|
2
|
-
export { A as AsyncSource, C as Classification,
|
|
1
|
+
import { E as EvidenceSource, D as DetectorConfig, c as DetectFn, a as LanguageEvidence } from './types-DfHZ5hkh.js';
|
|
2
|
+
export { A as AsyncSource, d as AttrBag, C as Classification, e as DetectContext, f as DetectInput, g as EarlyExit, h as EvidenceKind, i as HasAsync, H as HeaderBag, b as LanguageCode, L as LanguageProfile, N as NonDiscriminatingScript, j as SourceInput, S as SyncSource, W as Weights } from './types-DfHZ5hkh.js';
|
|
3
3
|
export { FuseOptions, fuse } from './fuse.js';
|
|
4
4
|
export { evidenceFromText } from './text.js';
|
|
5
5
|
export { evidenceFromHtml } from './html.js';
|
|
@@ -14,6 +14,40 @@ import './classify.js';
|
|
|
14
14
|
*/
|
|
15
15
|
declare function compile<const E extends readonly EvidenceSource[] = []>(config?: DetectorConfig<E>): DetectFn<E>;
|
|
16
16
|
|
|
17
|
+
/** The pipeline's default declaration attributes: only the standardized
|
|
18
|
+
* global `lang`. Vendor labels (Google's `data-rl` and kin) are page-private
|
|
19
|
+
* `data-*` conventions and must be opted into per detector — see
|
|
20
|
+
* {@link DetectorConfig.nodeLangAttributes}. */
|
|
21
|
+
declare const DEFAULT_NODE_LANG_ATTRIBUTES: readonly string[];
|
|
22
|
+
/**
|
|
23
|
+
* Producer: a per-node declared language — the page's own label for one
|
|
24
|
+
* content node, read from an attribute. The one STANDARDIZED bearer is the
|
|
25
|
+
* global `lang` attribute (BCP-47, inheritable); everything else is a vendor
|
|
26
|
+
* convention the caller opts into — e.g. Google's `data-rl` response-language
|
|
27
|
+
* label on AI-generated answers. No attribute name is built in here: which
|
|
28
|
+
* attributes count as declarations is entirely the caller's choice.
|
|
29
|
+
*
|
|
30
|
+
* The document-scope siblings of this signal live in {@link evidenceFromHtml}
|
|
31
|
+
* (`<html lang>`, `content-language`, `og:locale`); this is the same
|
|
32
|
+
* declaration concept one scope down. Like them it is *context* evidence: the
|
|
33
|
+
* fuser weighs it against text reads, and its script guard keeps a
|
|
34
|
+
* declaration from overriding a confident text read — a mislabeled node loses
|
|
35
|
+
* to clear script evidence, while weak or absent text lets the declaration
|
|
36
|
+
* decide.
|
|
37
|
+
*
|
|
38
|
+
* DOM-free by design — this module never touches an Element. The caller
|
|
39
|
+
* passes the attribute values it extracted, keyed by attribute name
|
|
40
|
+
* (`{ lang: el.getAttribute("lang") }`, `{ "data-rl": … }`). Each recognized
|
|
41
|
+
* value becomes one evidence item whose `source` is the attribute name under
|
|
42
|
+
* the `node-lang:` namespace (`node-lang:data-rl`) — namespaced so an
|
|
43
|
+
* attribute that happens to share a name with an engine id or evidence kind
|
|
44
|
+
* can never collide in the {@link FuseOptions.weights} keyspace. That lets
|
|
45
|
+
* weights target one attribute — trust a curated vendor label above an
|
|
46
|
+
* often-stale inherited `lang`, or silence an attribute entirely — instead
|
|
47
|
+
* of the whole kind. Empty and unrecognized values are dropped.
|
|
48
|
+
*/
|
|
49
|
+
declare function evidenceFromNodeLang(attrs: Readonly<Record<string, string | null | undefined>>): LanguageEvidence[];
|
|
50
|
+
|
|
17
51
|
/**
|
|
18
52
|
* BCP-47 / language-code normalization.
|
|
19
53
|
*
|
|
@@ -74,4 +108,4 @@ declare function normalizeBCP47(input: string | undefined | null, options?: Norm
|
|
|
74
108
|
*/
|
|
75
109
|
declare function primarySubtag(value: string | undefined | null): string | null;
|
|
76
110
|
|
|
77
|
-
export { DetectFn, DetectorConfig, EvidenceSource, type NormalizeBCP47Options, compile, normalizeBCP47, normalizeLanguageCode, primarySubtag };
|
|
111
|
+
export { DEFAULT_NODE_LANG_ATTRIBUTES, DetectFn, DetectorConfig, EvidenceSource, LanguageEvidence, type NormalizeBCP47Options, compile, evidenceFromNodeLang, normalizeBCP47, normalizeLanguageCode, primarySubtag };
|
package/dist/index.js
CHANGED
|
@@ -4,13 +4,34 @@ import { evidenceFromHtml } from './chunk-K4MXTIY7.js';
|
|
|
4
4
|
export { evidenceFromHtml } from './chunk-K4MXTIY7.js';
|
|
5
5
|
import { evidenceFromHeaders } from './chunk-FEKBPTHK.js';
|
|
6
6
|
export { evidenceFromHeaders } from './chunk-FEKBPTHK.js';
|
|
7
|
-
import { fuse } from './chunk-
|
|
8
|
-
export { fuse } from './chunk-
|
|
7
|
+
import { fuse } from './chunk-JXGCCDSV.js';
|
|
8
|
+
export { fuse } from './chunk-JXGCCDSV.js';
|
|
9
|
+
import { normalizeBCP47 } from './chunk-YCUSX3GG.js';
|
|
9
10
|
export { normalizeBCP47, normalizeLanguageCode, primarySubtag } from './chunk-YCUSX3GG.js';
|
|
10
11
|
import './chunk-BL627TWI.js';
|
|
11
12
|
|
|
13
|
+
// src/node-lang.ts
|
|
14
|
+
var NODE_LANG_CONFIDENCE = 0.7;
|
|
15
|
+
var DEFAULT_NODE_LANG_ATTRIBUTES = ["lang"];
|
|
16
|
+
function evidenceFromNodeLang(attrs) {
|
|
17
|
+
const out = [];
|
|
18
|
+
for (const [attribute, raw] of Object.entries(attrs)) {
|
|
19
|
+
if (raw == null || raw === "") continue;
|
|
20
|
+
const language = normalizeBCP47(raw);
|
|
21
|
+
if (language === null) continue;
|
|
22
|
+
out.push({
|
|
23
|
+
kind: "node-lang",
|
|
24
|
+
language,
|
|
25
|
+
confidence: NODE_LANG_CONFIDENCE,
|
|
26
|
+
source: `node-lang:${attribute}`,
|
|
27
|
+
value: raw
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
return out;
|
|
31
|
+
}
|
|
32
|
+
|
|
12
33
|
// src/compile.ts
|
|
13
|
-
function builtIns(candidates) {
|
|
34
|
+
function builtIns(candidates, nodeLangAttributes) {
|
|
14
35
|
return [
|
|
15
36
|
{
|
|
16
37
|
id: "text",
|
|
@@ -24,6 +45,16 @@ function builtIns(candidates) {
|
|
|
24
45
|
sync: true,
|
|
25
46
|
inputs: ["headers"],
|
|
26
47
|
detect: (i) => evidenceFromHeaders(i.headers)
|
|
48
|
+
},
|
|
49
|
+
{
|
|
50
|
+
id: "node-lang",
|
|
51
|
+
sync: true,
|
|
52
|
+
inputs: ["attrs"],
|
|
53
|
+
detect: (i) => {
|
|
54
|
+
const declared = {};
|
|
55
|
+
for (const name of nodeLangAttributes) declared[name] = i.attrs?.[name];
|
|
56
|
+
return evidenceFromNodeLang(declared);
|
|
57
|
+
}
|
|
27
58
|
}
|
|
28
59
|
];
|
|
29
60
|
}
|
|
@@ -31,7 +62,10 @@ function applicable(source, input) {
|
|
|
31
62
|
return source.inputs.every((key) => input[key] !== void 0);
|
|
32
63
|
}
|
|
33
64
|
function compile(config = {}) {
|
|
34
|
-
const sources = [
|
|
65
|
+
const sources = [
|
|
66
|
+
...builtIns(config.candidates, config.nodeLangAttributes ?? DEFAULT_NODE_LANG_ATTRIBUTES),
|
|
67
|
+
...config.engines ?? []
|
|
68
|
+
];
|
|
35
69
|
const hasAsync = sources.some((source) => !source.sync);
|
|
36
70
|
const fuseOptions = {
|
|
37
71
|
weights: config.weights,
|
|
@@ -62,6 +96,6 @@ function compile(config = {}) {
|
|
|
62
96
|
return detect;
|
|
63
97
|
}
|
|
64
98
|
|
|
65
|
-
export { compile };
|
|
99
|
+
export { DEFAULT_NODE_LANG_ATTRIBUTES, compile, evidenceFromNodeLang };
|
|
66
100
|
//# sourceMappingURL=index.js.map
|
|
67
101
|
//# sourceMappingURL=index.js.map
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/compile.ts"],"names":["detect"],"mappings":"
|
|
1
|
+
{"version":3,"sources":["../src/node-lang.ts","../src/compile.ts"],"names":["detect"],"mappings":";;;;;;;;;;;;;AAOA,IAAM,oBAAA,GAAuB,GAAA;AAMtB,IAAM,4BAAA,GAAkD,CAAC,MAAM;AA6B/D,SAAS,qBACd,KAAA,EACoB;AACpB,EAAA,MAAM,MAA0B,EAAC;AACjC,EAAA,KAAA,MAAW,CAAC,SAAA,EAAW,GAAG,KAAK,MAAA,CAAO,OAAA,CAAQ,KAAK,CAAA,EAAG;AACpD,IAAA,IAAI,GAAA,IAAO,IAAA,IAAQ,GAAA,KAAQ,EAAA,EAAI;AAC/B,IAAA,MAAM,QAAA,GAAW,eAAe,GAAG,CAAA;AACnC,IAAA,IAAI,aAAa,IAAA,EAAM;AACvB,IAAA,GAAA,CAAI,IAAA,CAAK;AAAA,MACP,IAAA,EAAM,WAAA;AAAA,MACN,QAAA;AAAA,MACA,UAAA,EAAY,oBAAA;AAAA,MACZ,MAAA,EAAQ,aAAa,SAAS,CAAA,CAAA;AAAA,MAC9B,KAAA,EAAO;AAAA,KACR,CAAA;AAAA,EACH;AACA,EAAA,OAAO,GAAA;AACT;;;ACpCA,SAAS,QAAA,CACP,YACA,kBAAA,EACc;AACd,EAAA,OAAO;AAAA,IACL;AAAA,MACE,EAAA,EAAI,MAAA;AAAA,MACJ,IAAA,EAAM,IAAA;AAAA,MACN,MAAA,EAAQ,CAAC,MAAM,CAAA;AAAA,MACf,QAAQ,CAAC,CAAA,KAAM,gBAAA,CAAiB,CAAA,CAAE,MAAM,UAAU;AAAA,KACpD;AAAA,IACA,EAAE,EAAA,EAAI,MAAA,EAAQ,IAAA,EAAM,MAAM,MAAA,EAAQ,CAAC,MAAM,CAAA,EAAG,QAAQ,CAAC,CAAA,KAAM,gBAAA,CAAiB,CAAA,CAAE,IAAI,CAAA,EAAE;AAAA,IACpF;AAAA,MACE,EAAA,EAAI,SAAA;AAAA,MACJ,IAAA,EAAM,IAAA;AAAA,MACN,MAAA,EAAQ,CAAC,SAAS,CAAA;AAAA,MAClB,MAAA,EAAQ,CAAC,CAAA,KAAM,mBAAA,CAAoB,EAAE,OAAO;AAAA,KAC9C;AAAA,IACA;AAAA,MACE,EAAA,EAAI,WAAA;AAAA,MACJ,IAAA,EAAM,IAAA;AAAA,MACN,MAAA,EAAQ,CAAC,OAAO,CAAA;AAAA,MAChB,MAAA,EAAQ,CAAC,CAAA,KAAM;AACb,QAAA,MAAM,WAAsD,EAAC;AAC7D,QAAA,KAAA,MAAW,QAAQ,kBAAA,EAAoB,QAAA,CAAS,IAAI,CAAA,GAAI,CAAA,CAAE,QAAQ,IAAI,CAAA;AACtE,QAAA,OAAO,qBAAqB,QAAQ,CAAA;AAAA,MACtC;AAAA;AACF,GACF;AACF;AAGA,SAAS,UAAA,CAAW,QAAwB,KAAA,EAA6B;AACvE,EAAA,OAAO,MAAA,CAAO,OAAO,KAAA,CAAM,CAAC,QAAQ,KAAA,CAAM,GAAG,MAAM,MAAS,CAAA;AAC9D;AAQO,SAAS,OAAA,CACd,MAAA,GAA4B,EAAC,EAChB;AACb,EAAA,MAAM,OAAA,GAA4B;AAAA,IAChC,GAAG,QAAA,CAAS,MAAA,CAAO,UAAA,EAAY,MAAA,CAAO,sBAAsB,4BAA4B,CAAA;AAAA,IACxF,GAAI,MAAA,CAAO,OAAA,IAAW;AAAC,GACzB;AACA,EAAA,MAAM,WAAW,OAAA,CAAQ,IAAA,CAAK,CAAC,MAAA,KAAW,CAAC,OAAO,IAAI,CAAA;AACtD,EAAA,MAAM,WAAA,GAA2B;AAAA,IAC/B,SAAS,MAAA,CAAO,OAAA;AAAA,IAChB,YAAY,MAAA,CAAO,UAAA;AAAA,IACnB,yBAAyB,MAAA,CAAO;AAAA,GAClC;AAEA,EAAA,IAAI,CAAC,QAAA,EAAU;AACb,IAAA,MAAMA,OAAAA,GAAS,CAAC,KAAA,KAAuC;AACrD,MAAA,MAAM,WAA+B,EAAC;AACtC,MAAA,KAAA,MAAW,UAAU,OAAA,EAAS;AAC5B,QAAA,IAAI,MAAA,CAAO,IAAA,IAAQ,UAAA,CAAW,MAAA,EAAQ,KAAK,CAAA,EAAG,QAAA,CAAS,IAAA,CAAK,GAAG,MAAA,CAAO,MAAA,CAAO,KAAK,CAAC,CAAA;AAAA,MACrF;AACA,MAAA,OAAO,IAAA,CAAK,UAAU,WAAW,CAAA;AAAA,IACnC,CAAA;AACA,IAAA,OAAOA,OAAAA;AAAA,EACT;AAEA,EAAA,MAAM,MAAA,GAAS,OAAO,KAAA,EAAoB,GAAA,GAAqB,EAAC,KAA+B;AAC7F,IAAA,MAAM,WAA+B,EAAC;AACtC,IAAA,MAAM,UAAyC,EAAC;AAChD,IAAA,KAAA,MAAW,UAAU,OAAA,EAAS;AAC5B,MAAA,IAAI,CAAC,UAAA,CAAW,MAAA,EAAQ,KAAK,CAAA,EAAG;AAChC,MAAA,IAAI,MAAA,CAAO,MAAM,QAAA,CAAS,IAAA,CAAK,GAAG,MAAA,CAAO,MAAA,CAAO,KAAK,CAAC,CAAA;AAAA,WACjD,OAAA,CAAQ,IAAA,CAAK,OAAA,CAAQ,OAAA,CAAQ,OAAO,MAAA,CAAO,KAAA,EAAO,GAAG,CAAC,CAAA,CAAE,KAAA,CAAM,MAAM,EAAE,CAAC,CAAA;AAAA,IAC9E;AACA,IAAA,KAAA,MAAW,KAAA,IAAS,MAAM,OAAA,CAAQ,GAAA,CAAI,OAAO,CAAA,EAAG,QAAA,CAAS,IAAA,CAAK,GAAG,KAAK,CAAA;AACtE,IAAA,OAAO,IAAA,CAAK,UAAU,WAAW,CAAA;AAAA,EACnC,CAAA;AACA,EAAA,OAAO,MAAA;AACT","file":"index.js","sourcesContent":["import type { LanguageEvidence } from \"./types.js\";\nimport { normalizeBCP47 } from \"./internal/bcp47.js\";\n\n/** Confidence for a node-level declaration: above the page-scope tags (a label\n * on the node itself sits closer to the content than `<html lang>` does), but\n * still context evidence — {@link fuse}'s guard keeps it from flipping clear\n * script evidence. */\nconst NODE_LANG_CONFIDENCE = 0.7;\n\n/** The pipeline's default declaration attributes: only the standardized\n * global `lang`. Vendor labels (Google's `data-rl` and kin) are page-private\n * `data-*` conventions and must be opted into per detector — see\n * {@link DetectorConfig.nodeLangAttributes}. */\nexport const DEFAULT_NODE_LANG_ATTRIBUTES: readonly string[] = [\"lang\"];\n\n/**\n * Producer: a per-node declared language — the page's own label for one\n * content node, read from an attribute. The one STANDARDIZED bearer is the\n * global `lang` attribute (BCP-47, inheritable); everything else is a vendor\n * convention the caller opts into — e.g. Google's `data-rl` response-language\n * label on AI-generated answers. No attribute name is built in here: which\n * attributes count as declarations is entirely the caller's choice.\n *\n * The document-scope siblings of this signal live in {@link evidenceFromHtml}\n * (`<html lang>`, `content-language`, `og:locale`); this is the same\n * declaration concept one scope down. Like them it is *context* evidence: the\n * fuser weighs it against text reads, and its script guard keeps a\n * declaration from overriding a confident text read — a mislabeled node loses\n * to clear script evidence, while weak or absent text lets the declaration\n * decide.\n *\n * DOM-free by design — this module never touches an Element. The caller\n * passes the attribute values it extracted, keyed by attribute name\n * (`{ lang: el.getAttribute(\"lang\") }`, `{ \"data-rl\": … }`). Each recognized\n * value becomes one evidence item whose `source` is the attribute name under\n * the `node-lang:` namespace (`node-lang:data-rl`) — namespaced so an\n * attribute that happens to share a name with an engine id or evidence kind\n * can never collide in the {@link FuseOptions.weights} keyspace. That lets\n * weights target one attribute — trust a curated vendor label above an\n * often-stale inherited `lang`, or silence an attribute entirely — instead\n * of the whole kind. Empty and unrecognized values are dropped.\n */\nexport function evidenceFromNodeLang(\n attrs: Readonly<Record<string, string | null | undefined>>,\n): LanguageEvidence[] {\n const out: LanguageEvidence[] = [];\n for (const [attribute, raw] of Object.entries(attrs)) {\n if (raw == null || raw === \"\") continue;\n const language = normalizeBCP47(raw);\n if (language === null) continue;\n out.push({\n kind: \"node-lang\",\n language,\n confidence: NODE_LANG_CONFIDENCE,\n source: `node-lang:${attribute}`,\n value: raw,\n });\n }\n return out;\n}\n","import { evidenceFromHeaders } from \"./headers.js\";\nimport { evidenceFromHtml } from \"./html.js\";\nimport { DEFAULT_NODE_LANG_ATTRIBUTES, evidenceFromNodeLang } from \"./node-lang.js\";\nimport { evidenceFromText } from \"./text.js\";\nimport { fuse, type FuseOptions } from \"./fuse.js\";\nimport type {\n Classification,\n DetectContext,\n DetectFn,\n DetectInput,\n DetectorConfig,\n EvidenceSource,\n LanguageEvidence,\n LanguageProfile,\n SyncSource,\n} from \"./types.js\";\n\n/** The always-on, zero-dependency producers. The text producer is bound to the\n * configured candidate roster so its scoring is roster-relative (and so it\n * abstains when no roster was supplied — its signals need candidates). The\n * node-lang producer is bound to the configured declaration-attribute list\n * (standard `lang` only, unless the config opts into vendor attributes), and\n * reads them from `input.attrs` in configured order. */\nfunction builtIns(\n candidates: readonly LanguageProfile[] | undefined,\n nodeLangAttributes: readonly string[],\n): SyncSource[] {\n return [\n {\n id: \"text\",\n sync: true,\n inputs: [\"text\"],\n detect: (i) => evidenceFromText(i.text, candidates),\n },\n { id: \"html\", sync: true, inputs: [\"html\"], detect: (i) => evidenceFromHtml(i.html) },\n {\n id: \"headers\",\n sync: true,\n inputs: [\"headers\"],\n detect: (i) => evidenceFromHeaders(i.headers),\n },\n {\n id: \"node-lang\",\n sync: true,\n inputs: [\"attrs\"],\n detect: (i) => {\n const declared: Record<string, string | null | undefined> = {};\n for (const name of nodeLangAttributes) declared[name] = i.attrs?.[name];\n return evidenceFromNodeLang(declared);\n },\n },\n ];\n}\n\n/** Run a source only when every input it declares is present. */\nfunction applicable(source: EvidenceSource, input: DetectInput): boolean {\n return source.inputs.every((key) => input[key] !== undefined);\n}\n\n/**\n * Build a configured detector. Does the per-roster setup once and returns a\n * `detect` function whose sync/async shape is fixed by the registered engines\n * (see {@link DetectFn}). The built-in producers are always registered; opt-in\n * engines (franc, chrome-ai) are added via `config.engines`.\n */\nexport function compile<const E extends readonly EvidenceSource[] = []>(\n config: DetectorConfig<E> = {},\n): DetectFn<E> {\n const sources: EvidenceSource[] = [\n ...builtIns(config.candidates, config.nodeLangAttributes ?? DEFAULT_NODE_LANG_ATTRIBUTES),\n ...(config.engines ?? []),\n ];\n const hasAsync = sources.some((source) => !source.sync);\n const fuseOptions: FuseOptions = {\n weights: config.weights,\n candidates: config.candidates,\n nonDiscriminatingScript: config.nonDiscriminatingScript,\n };\n\n if (!hasAsync) {\n const detect = (input: DetectInput): Classification => {\n const evidence: LanguageEvidence[] = [];\n for (const source of sources) {\n if (source.sync && applicable(source, input)) evidence.push(...source.detect(input));\n }\n return fuse(evidence, fuseOptions);\n };\n return detect as DetectFn<E>;\n }\n\n const detect = async (input: DetectInput, ctx: DetectContext = {}): Promise<Classification> => {\n const evidence: LanguageEvidence[] = [];\n const pending: Promise<LanguageEvidence[]>[] = [];\n for (const source of sources) {\n if (!applicable(source, input)) continue;\n if (source.sync) evidence.push(...source.detect(input));\n else pending.push(Promise.resolve(source.detect(input, ctx)).catch(() => []));\n }\n for (const batch of await Promise.all(pending)) evidence.push(...batch);\n return fuse(evidence, fuseOptions);\n };\n return detect as DetectFn<E>;\n}\n"]}
|
package/dist/profiles.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { b as LanguageCode, L as LanguageProfile } from './types-
|
|
1
|
+
import { b as LanguageCode, L as LanguageProfile } from './types-DfHZ5hkh.js';
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
4
|
* `langtell/profiles` — ready-to-use {@link LanguageProfile} data.
|
|
@@ -30,6 +30,9 @@ declare const ru: LanguageProfile;
|
|
|
30
30
|
declare const be: LanguageProfile;
|
|
31
31
|
declare const bg: LanguageProfile;
|
|
32
32
|
declare const en: LanguageProfile;
|
|
33
|
+
declare const sr: LanguageProfile;
|
|
34
|
+
declare const mk: LanguageProfile;
|
|
35
|
+
declare const kk: LanguageProfile;
|
|
33
36
|
|
|
34
37
|
/** Registry of shipped profiles, keyed by BCP-47 code. */
|
|
35
38
|
declare const PROFILES: Readonly<Record<LanguageCode, LanguageProfile>>;
|
|
@@ -44,4 +47,4 @@ declare function hasProfile(code: LanguageCode): boolean;
|
|
|
44
47
|
/** Resolve profiles for the given codes, skipping any without a shipped profile. */
|
|
45
48
|
declare function getProfiles(codes: readonly LanguageCode[]): LanguageProfile[];
|
|
46
49
|
|
|
47
|
-
export { PROFILED_CODES, PROFILES, be, bg, en, getProfiles, hasProfile, ru, uk };
|
|
50
|
+
export { PROFILED_CODES, PROFILES, be, bg, en, getProfiles, hasProfile, kk, mk, ru, sr, uk };
|