langtell 0.4.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -13
- package/dist/chrome-ai.d.ts +1 -1
- package/dist/{chunk-KCK3XWCJ.js → chunk-7TDGJOIJ.js} +3 -3
- package/dist/{chunk-KCK3XWCJ.js.map → chunk-7TDGJOIJ.js.map} +1 -1
- package/dist/{chunk-U34Z3ZSV.js → chunk-BL627TWI.js} +4 -3
- package/dist/chunk-BL627TWI.js.map +1 -0
- package/dist/{chunk-3LDE35U2.js → chunk-FEKBPTHK.js} +3 -3
- package/dist/{chunk-3LDE35U2.js.map → chunk-FEKBPTHK.js.map} +1 -1
- package/dist/{chunk-G44HHVK5.js → chunk-JXGCCDSV.js} +5 -4
- package/dist/chunk-JXGCCDSV.js.map +1 -0
- package/dist/{chunk-KI4MAI3N.js → chunk-K4MXTIY7.js} +3 -3
- package/dist/{chunk-KI4MAI3N.js.map → chunk-K4MXTIY7.js.map} +1 -1
- package/dist/{chunk-OVSPOZ5J.js → chunk-YCUSX3GG.js} +17 -5
- package/dist/chunk-YCUSX3GG.js.map +1 -0
- package/dist/classify.d.ts +9 -2
- package/dist/classify.js +1 -1
- package/dist/cyrillic.d.ts +60 -0
- package/dist/cyrillic.js +69 -0
- package/dist/cyrillic.js.map +1 -0
- package/dist/franc.d.ts +1 -1
- package/dist/franc.js +1 -1
- package/dist/fuse.d.ts +1 -1
- package/dist/fuse.js +3 -3
- package/dist/headers.d.ts +1 -1
- package/dist/headers.js +2 -2
- package/dist/html.d.ts +1 -1
- package/dist/html.js +2 -2
- package/dist/index.d.ts +58 -7
- package/dist/index.js +47 -13
- package/dist/index.js.map +1 -1
- package/dist/profiles.d.ts +5 -2
- package/dist/profiles.js +431 -2
- package/dist/profiles.js.map +1 -1
- package/dist/text.d.ts +1 -1
- package/dist/text.js +2 -2
- package/dist/{types-BIXrkuAr.d.ts → types-DfHZ5hkh.d.ts} +19 -3
- package/package.json +5 -1
- package/dist/chunk-G44HHVK5.js.map +0 -1
- package/dist/chunk-OVSPOZ5J.js.map +0 -1
- package/dist/chunk-U34Z3ZSV.js.map +0 -1
package/dist/fuse.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { W as Weights, L as LanguageProfile, N as NonDiscriminatingScript, a as LanguageEvidence, C as Classification } from './types-
|
|
1
|
+
import { W as Weights, L as LanguageProfile, N as NonDiscriminatingScript, a as LanguageEvidence, C as Classification } from './types-DfHZ5hkh.js';
|
|
2
2
|
|
|
3
3
|
interface FuseOptions {
|
|
4
4
|
weights?: Weights;
|
package/dist/fuse.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
export { fuse } from './chunk-
|
|
2
|
-
import './chunk-
|
|
3
|
-
import './chunk-
|
|
1
|
+
export { fuse } from './chunk-JXGCCDSV.js';
|
|
2
|
+
import './chunk-YCUSX3GG.js';
|
|
3
|
+
import './chunk-BL627TWI.js';
|
|
4
4
|
//# sourceMappingURL=fuse.js.map
|
|
5
5
|
//# sourceMappingURL=fuse.js.map
|
package/dist/headers.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { H as HeaderBag, a as LanguageEvidence } from './types-
|
|
1
|
+
import { H as HeaderBag, a as LanguageEvidence } from './types-DfHZ5hkh.js';
|
|
2
2
|
|
|
3
3
|
/** Producer: the HTTP `Content-Language` response header. */
|
|
4
4
|
declare function evidenceFromHeaders(headers: HeaderBag | undefined): LanguageEvidence[];
|
package/dist/headers.js
CHANGED
package/dist/html.d.ts
CHANGED
package/dist/html.js
CHANGED
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { E as EvidenceSource, D as DetectorConfig, c as DetectFn } from './types-
|
|
2
|
-
export { A as AsyncSource, C as Classification,
|
|
1
|
+
import { E as EvidenceSource, D as DetectorConfig, c as DetectFn, a as LanguageEvidence } from './types-DfHZ5hkh.js';
|
|
2
|
+
export { A as AsyncSource, d as AttrBag, C as Classification, e as DetectContext, f as DetectInput, g as EarlyExit, h as EvidenceKind, i as HasAsync, H as HeaderBag, b as LanguageCode, L as LanguageProfile, N as NonDiscriminatingScript, j as SourceInput, S as SyncSource, W as Weights } from './types-DfHZ5hkh.js';
|
|
3
3
|
export { FuseOptions, fuse } from './fuse.js';
|
|
4
4
|
export { evidenceFromText } from './text.js';
|
|
5
5
|
export { evidenceFromHtml } from './html.js';
|
|
@@ -14,6 +14,40 @@ import './classify.js';
|
|
|
14
14
|
*/
|
|
15
15
|
declare function compile<const E extends readonly EvidenceSource[] = []>(config?: DetectorConfig<E>): DetectFn<E>;
|
|
16
16
|
|
|
17
|
+
/** The pipeline's default declaration attributes: only the standardized
|
|
18
|
+
* global `lang`. Vendor labels (Google's `data-rl` and kin) are page-private
|
|
19
|
+
* `data-*` conventions and must be opted into per detector — see
|
|
20
|
+
* {@link DetectorConfig.nodeLangAttributes}. */
|
|
21
|
+
declare const DEFAULT_NODE_LANG_ATTRIBUTES: readonly string[];
|
|
22
|
+
/**
|
|
23
|
+
* Producer: a per-node declared language — the page's own label for one
|
|
24
|
+
* content node, read from an attribute. The one STANDARDIZED bearer is the
|
|
25
|
+
* global `lang` attribute (BCP-47, inheritable); everything else is a vendor
|
|
26
|
+
* convention the caller opts into — e.g. Google's `data-rl` response-language
|
|
27
|
+
* label on AI-generated answers. No attribute name is built in here: which
|
|
28
|
+
* attributes count as declarations is entirely the caller's choice.
|
|
29
|
+
*
|
|
30
|
+
* The document-scope siblings of this signal live in {@link evidenceFromHtml}
|
|
31
|
+
* (`<html lang>`, `content-language`, `og:locale`); this is the same
|
|
32
|
+
* declaration concept one scope down. Like them it is *context* evidence: the
|
|
33
|
+
* fuser weighs it against text reads, and its script guard keeps a
|
|
34
|
+
* declaration from overriding a confident text read — a mislabeled node loses
|
|
35
|
+
* to clear script evidence, while weak or absent text lets the declaration
|
|
36
|
+
* decide.
|
|
37
|
+
*
|
|
38
|
+
* DOM-free by design — this module never touches an Element. The caller
|
|
39
|
+
* passes the attribute values it extracted, keyed by attribute name
|
|
40
|
+
* (`{ lang: el.getAttribute("lang") }`, `{ "data-rl": … }`). Each recognized
|
|
41
|
+
* value becomes one evidence item whose `source` is the attribute name under
|
|
42
|
+
* the `node-lang:` namespace (`node-lang:data-rl`) — namespaced so an
|
|
43
|
+
* attribute that happens to share a name with an engine id or evidence kind
|
|
44
|
+
* can never collide in the {@link FuseOptions.weights} keyspace. That lets
|
|
45
|
+
* weights target one attribute — trust a curated vendor label above an
|
|
46
|
+
* often-stale inherited `lang`, or silence an attribute entirely — instead
|
|
47
|
+
* of the whole kind. Empty and unrecognized values are dropped.
|
|
48
|
+
*/
|
|
49
|
+
declare function evidenceFromNodeLang(attrs: Readonly<Record<string, string | null | undefined>>): LanguageEvidence[];
|
|
50
|
+
|
|
17
51
|
/**
|
|
18
52
|
* BCP-47 / language-code normalization.
|
|
19
53
|
*
|
|
@@ -35,17 +69,34 @@ declare function compile<const E extends readonly EvidenceSource[] = []>(config?
|
|
|
35
69
|
* directly, so exact lookup still finds them.
|
|
36
70
|
*/
|
|
37
71
|
declare function normalizeLanguageCode(input: string | undefined | null): string | null;
|
|
72
|
+
/** Options for {@link normalizeBCP47}. */
|
|
73
|
+
interface NormalizeBCP47Options {
|
|
74
|
+
/**
|
|
75
|
+
* What to return when the input's primary subtag is not in the alias table.
|
|
76
|
+
* - `"subtag"` (default) — pass the raw primary subtag through, so a code
|
|
77
|
+
* outside the table still resolves to its language (`pt-BR` → `pt`,
|
|
78
|
+
* `sv` → `sv`). Best for a permissive normalizer whose roster decides
|
|
79
|
+
* relevance downstream.
|
|
80
|
+
* - `"null"` — treat an unknown head as unsupported and return `null`. Best
|
|
81
|
+
* for callers that gate on a fixed alias set and read `null` as "not a
|
|
82
|
+
* language I handle".
|
|
83
|
+
*/
|
|
84
|
+
unknownHead?: "subtag" | "null";
|
|
85
|
+
}
|
|
38
86
|
/**
|
|
39
87
|
* BCP-47-aware normalization: try the full string first, then strip a
|
|
40
88
|
* region/script suffix (`en-US` → `en`, `zh_CN` → `zh`). Use ONLY for inputs
|
|
41
89
|
* documented to be BCP-47 — `hreflang`, `<html lang>`, `Content-Language`,
|
|
42
90
|
* `data-lang`/`data-locale` — never for free-text URL slugs.
|
|
43
91
|
*
|
|
44
|
-
*
|
|
45
|
-
* the alias table (e.g. `pt-BR` → `pt`) still resolves to its
|
|
46
|
-
* roster decides relevance downstream.
|
|
92
|
+
* By default, falls back to the raw primary subtag when no alias matches, so a
|
|
93
|
+
* code outside the alias table (e.g. `pt-BR` → `pt`) still resolves to its
|
|
94
|
+
* language; the roster decides relevance downstream. Pass
|
|
95
|
+
* `{ unknownHead: "null" }` to instead return `null` for any tag whose head
|
|
96
|
+
* isn't in the table — for callers that treat "not in my alias set" as
|
|
97
|
+
* unsupported. The default (`"subtag"`) is unchanged.
|
|
47
98
|
*/
|
|
48
|
-
declare function normalizeBCP47(input: string | undefined | null): string | null;
|
|
99
|
+
declare function normalizeBCP47(input: string | undefined | null, options?: NormalizeBCP47Options): string | null;
|
|
49
100
|
/**
|
|
50
101
|
* Extract the primary subtag from a BCP-47-ish value, lowercased, then resolve
|
|
51
102
|
* it through the alias table (`ua` → `uk`). Handles `Accept-Language`-style
|
|
@@ -57,4 +108,4 @@ declare function normalizeBCP47(input: string | undefined | null): string | null
|
|
|
57
108
|
*/
|
|
58
109
|
declare function primarySubtag(value: string | undefined | null): string | null;
|
|
59
110
|
|
|
60
|
-
export { DetectFn, DetectorConfig, EvidenceSource, compile, normalizeBCP47, normalizeLanguageCode, primarySubtag };
|
|
111
|
+
export { DEFAULT_NODE_LANG_ATTRIBUTES, DetectFn, DetectorConfig, EvidenceSource, LanguageEvidence, type NormalizeBCP47Options, compile, evidenceFromNodeLang, normalizeBCP47, normalizeLanguageCode, primarySubtag };
|
package/dist/index.js
CHANGED
|
@@ -1,16 +1,37 @@
|
|
|
1
|
-
import { evidenceFromText } from './chunk-
|
|
2
|
-
export { evidenceFromText } from './chunk-
|
|
3
|
-
import { evidenceFromHtml } from './chunk-
|
|
4
|
-
export { evidenceFromHtml } from './chunk-
|
|
5
|
-
import { evidenceFromHeaders } from './chunk-
|
|
6
|
-
export { evidenceFromHeaders } from './chunk-
|
|
7
|
-
import { fuse } from './chunk-
|
|
8
|
-
export { fuse } from './chunk-
|
|
9
|
-
|
|
10
|
-
|
|
1
|
+
import { evidenceFromText } from './chunk-7TDGJOIJ.js';
|
|
2
|
+
export { evidenceFromText } from './chunk-7TDGJOIJ.js';
|
|
3
|
+
import { evidenceFromHtml } from './chunk-K4MXTIY7.js';
|
|
4
|
+
export { evidenceFromHtml } from './chunk-K4MXTIY7.js';
|
|
5
|
+
import { evidenceFromHeaders } from './chunk-FEKBPTHK.js';
|
|
6
|
+
export { evidenceFromHeaders } from './chunk-FEKBPTHK.js';
|
|
7
|
+
import { fuse } from './chunk-JXGCCDSV.js';
|
|
8
|
+
export { fuse } from './chunk-JXGCCDSV.js';
|
|
9
|
+
import { normalizeBCP47 } from './chunk-YCUSX3GG.js';
|
|
10
|
+
export { normalizeBCP47, normalizeLanguageCode, primarySubtag } from './chunk-YCUSX3GG.js';
|
|
11
|
+
import './chunk-BL627TWI.js';
|
|
12
|
+
|
|
13
|
+
// src/node-lang.ts
|
|
14
|
+
var NODE_LANG_CONFIDENCE = 0.7;
|
|
15
|
+
var DEFAULT_NODE_LANG_ATTRIBUTES = ["lang"];
|
|
16
|
+
function evidenceFromNodeLang(attrs) {
|
|
17
|
+
const out = [];
|
|
18
|
+
for (const [attribute, raw] of Object.entries(attrs)) {
|
|
19
|
+
if (raw == null || raw === "") continue;
|
|
20
|
+
const language = normalizeBCP47(raw);
|
|
21
|
+
if (language === null) continue;
|
|
22
|
+
out.push({
|
|
23
|
+
kind: "node-lang",
|
|
24
|
+
language,
|
|
25
|
+
confidence: NODE_LANG_CONFIDENCE,
|
|
26
|
+
source: `node-lang:${attribute}`,
|
|
27
|
+
value: raw
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
return out;
|
|
31
|
+
}
|
|
11
32
|
|
|
12
33
|
// src/compile.ts
|
|
13
|
-
function builtIns(candidates) {
|
|
34
|
+
function builtIns(candidates, nodeLangAttributes) {
|
|
14
35
|
return [
|
|
15
36
|
{
|
|
16
37
|
id: "text",
|
|
@@ -24,6 +45,16 @@ function builtIns(candidates) {
|
|
|
24
45
|
sync: true,
|
|
25
46
|
inputs: ["headers"],
|
|
26
47
|
detect: (i) => evidenceFromHeaders(i.headers)
|
|
48
|
+
},
|
|
49
|
+
{
|
|
50
|
+
id: "node-lang",
|
|
51
|
+
sync: true,
|
|
52
|
+
inputs: ["attrs"],
|
|
53
|
+
detect: (i) => {
|
|
54
|
+
const declared = {};
|
|
55
|
+
for (const name of nodeLangAttributes) declared[name] = i.attrs?.[name];
|
|
56
|
+
return evidenceFromNodeLang(declared);
|
|
57
|
+
}
|
|
27
58
|
}
|
|
28
59
|
];
|
|
29
60
|
}
|
|
@@ -31,7 +62,10 @@ function applicable(source, input) {
|
|
|
31
62
|
return source.inputs.every((key) => input[key] !== void 0);
|
|
32
63
|
}
|
|
33
64
|
function compile(config = {}) {
|
|
34
|
-
const sources = [
|
|
65
|
+
const sources = [
|
|
66
|
+
...builtIns(config.candidates, config.nodeLangAttributes ?? DEFAULT_NODE_LANG_ATTRIBUTES),
|
|
67
|
+
...config.engines ?? []
|
|
68
|
+
];
|
|
35
69
|
const hasAsync = sources.some((source) => !source.sync);
|
|
36
70
|
const fuseOptions = {
|
|
37
71
|
weights: config.weights,
|
|
@@ -62,6 +96,6 @@ function compile(config = {}) {
|
|
|
62
96
|
return detect;
|
|
63
97
|
}
|
|
64
98
|
|
|
65
|
-
export { compile };
|
|
99
|
+
export { DEFAULT_NODE_LANG_ATTRIBUTES, compile, evidenceFromNodeLang };
|
|
66
100
|
//# sourceMappingURL=index.js.map
|
|
67
101
|
//# sourceMappingURL=index.js.map
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/compile.ts"],"names":["detect"],"mappings":"
|
|
1
|
+
{"version":3,"sources":["../src/node-lang.ts","../src/compile.ts"],"names":["detect"],"mappings":";;;;;;;;;;;;;AAOA,IAAM,oBAAA,GAAuB,GAAA;AAMtB,IAAM,4BAAA,GAAkD,CAAC,MAAM;AA6B/D,SAAS,qBACd,KAAA,EACoB;AACpB,EAAA,MAAM,MAA0B,EAAC;AACjC,EAAA,KAAA,MAAW,CAAC,SAAA,EAAW,GAAG,KAAK,MAAA,CAAO,OAAA,CAAQ,KAAK,CAAA,EAAG;AACpD,IAAA,IAAI,GAAA,IAAO,IAAA,IAAQ,GAAA,KAAQ,EAAA,EAAI;AAC/B,IAAA,MAAM,QAAA,GAAW,eAAe,GAAG,CAAA;AACnC,IAAA,IAAI,aAAa,IAAA,EAAM;AACvB,IAAA,GAAA,CAAI,IAAA,CAAK;AAAA,MACP,IAAA,EAAM,WAAA;AAAA,MACN,QAAA;AAAA,MACA,UAAA,EAAY,oBAAA;AAAA,MACZ,MAAA,EAAQ,aAAa,SAAS,CAAA,CAAA;AAAA,MAC9B,KAAA,EAAO;AAAA,KACR,CAAA;AAAA,EACH;AACA,EAAA,OAAO,GAAA;AACT;;;ACpCA,SAAS,QAAA,CACP,YACA,kBAAA,EACc;AACd,EAAA,OAAO;AAAA,IACL;AAAA,MACE,EAAA,EAAI,MAAA;AAAA,MACJ,IAAA,EAAM,IAAA;AAAA,MACN,MAAA,EAAQ,CAAC,MAAM,CAAA;AAAA,MACf,QAAQ,CAAC,CAAA,KAAM,gBAAA,CAAiB,CAAA,CAAE,MAAM,UAAU;AAAA,KACpD;AAAA,IACA,EAAE,EAAA,EAAI,MAAA,EAAQ,IAAA,EAAM,MAAM,MAAA,EAAQ,CAAC,MAAM,CAAA,EAAG,QAAQ,CAAC,CAAA,KAAM,gBAAA,CAAiB,CAAA,CAAE,IAAI,CAAA,EAAE;AAAA,IACpF;AAAA,MACE,EAAA,EAAI,SAAA;AAAA,MACJ,IAAA,EAAM,IAAA;AAAA,MACN,MAAA,EAAQ,CAAC,SAAS,CAAA;AAAA,MAClB,MAAA,EAAQ,CAAC,CAAA,KAAM,mBAAA,CAAoB,EAAE,OAAO;AAAA,KAC9C;AAAA,IACA;AAAA,MACE,EAAA,EAAI,WAAA;AAAA,MACJ,IAAA,EAAM,IAAA;AAAA,MACN,MAAA,EAAQ,CAAC,OAAO,CAAA;AAAA,MAChB,MAAA,EAAQ,CAAC,CAAA,KAAM;AACb,QAAA,MAAM,WAAsD,EAAC;AAC7D,QAAA,KAAA,MAAW,QAAQ,kBAAA,EAAoB,QAAA,CAAS,IAAI,CAAA,GAAI,CAAA,CAAE,QAAQ,IAAI,CAAA;AACtE,QAAA,OAAO,qBAAqB,QAAQ,CAAA;AAAA,MACtC;AAAA;AACF,GACF;AACF;AAGA,SAAS,UAAA,CAAW,QAAwB,KAAA,EAA6B;AACvE,EAAA,OAAO,MAAA,CAAO,OAAO,KAAA,CAAM,CAAC,QAAQ,KAAA,CAAM,GAAG,MAAM,MAAS,CAAA;AAC9D;AAQO,SAAS,OAAA,CACd,MAAA,GAA4B,EAAC,EAChB;AACb,EAAA,MAAM,OAAA,GAA4B;AAAA,IAChC,GAAG,QAAA,CAAS,MAAA,CAAO,UAAA,EAAY,MAAA,CAAO,sBAAsB,4BAA4B,CAAA;AAAA,IACxF,GAAI,MAAA,CAAO,OAAA,IAAW;AAAC,GACzB;AACA,EAAA,MAAM,WAAW,OAAA,CAAQ,IAAA,CAAK,CAAC,MAAA,KAAW,CAAC,OAAO,IAAI,CAAA;AACtD,EAAA,MAAM,WAAA,GAA2B;AAAA,IAC/B,SAAS,MAAA,CAAO,OAAA;AAAA,IAChB,YAAY,MAAA,CAAO,UAAA;AAAA,IACnB,yBAAyB,MAAA,CAAO;AAAA,GAClC;AAEA,EAAA,IAAI,CAAC,QAAA,EAAU;AACb,IAAA,MAAMA,OAAAA,GAAS,CAAC,KAAA,KAAuC;AACrD,MAAA,MAAM,WAA+B,EAAC;AACtC,MAAA,KAAA,MAAW,UAAU,OAAA,EAAS;AAC5B,QAAA,IAAI,MAAA,CAAO,IAAA,IAAQ,UAAA,CAAW,MAAA,EAAQ,KAAK,CAAA,EAAG,QAAA,CAAS,IAAA,CAAK,GAAG,MAAA,CAAO,MAAA,CAAO,KAAK,CAAC,CAAA;AAAA,MACrF;AACA,MAAA,OAAO,IAAA,CAAK,UAAU,WAAW,CAAA;AAAA,IACnC,CAAA;AACA,IAAA,OAAOA,OAAAA;AAAA,EACT;AAEA,EAAA,MAAM,MAAA,GAAS,OAAO,KAAA,EAAoB,GAAA,GAAqB,EAAC,KAA+B;AAC7F,IAAA,MAAM,WAA+B,EAAC;AACtC,IAAA,MAAM,UAAyC,EAAC;AAChD,IAAA,KAAA,MAAW,UAAU,OAAA,EAAS;AAC5B,MAAA,IAAI,CAAC,UAAA,CAAW,MAAA,EAAQ,KAAK,CAAA,EAAG;AAChC,MAAA,IAAI,MAAA,CAAO,MAAM,QAAA,CAAS,IAAA,CAAK,GAAG,MAAA,CAAO,MAAA,CAAO,KAAK,CAAC,CAAA;AAAA,WACjD,OAAA,CAAQ,IAAA,CAAK,OAAA,CAAQ,OAAA,CAAQ,OAAO,MAAA,CAAO,KAAA,EAAO,GAAG,CAAC,CAAA,CAAE,KAAA,CAAM,MAAM,EAAE,CAAC,CAAA;AAAA,IAC9E;AACA,IAAA,KAAA,MAAW,KAAA,IAAS,MAAM,OAAA,CAAQ,GAAA,CAAI,OAAO,CAAA,EAAG,QAAA,CAAS,IAAA,CAAK,GAAG,KAAK,CAAA;AACtE,IAAA,OAAO,IAAA,CAAK,UAAU,WAAW,CAAA;AAAA,EACnC,CAAA;AACA,EAAA,OAAO,MAAA;AACT","file":"index.js","sourcesContent":["import type { LanguageEvidence } from \"./types.js\";\nimport { normalizeBCP47 } from \"./internal/bcp47.js\";\n\n/** Confidence for a node-level declaration: above the page-scope tags (a label\n * on the node itself sits closer to the content than `<html lang>` does), but\n * still context evidence — {@link fuse}'s guard keeps it from flipping clear\n * script evidence. */\nconst NODE_LANG_CONFIDENCE = 0.7;\n\n/** The pipeline's default declaration attributes: only the standardized\n * global `lang`. Vendor labels (Google's `data-rl` and kin) are page-private\n * `data-*` conventions and must be opted into per detector — see\n * {@link DetectorConfig.nodeLangAttributes}. */\nexport const DEFAULT_NODE_LANG_ATTRIBUTES: readonly string[] = [\"lang\"];\n\n/**\n * Producer: a per-node declared language — the page's own label for one\n * content node, read from an attribute. The one STANDARDIZED bearer is the\n * global `lang` attribute (BCP-47, inheritable); everything else is a vendor\n * convention the caller opts into — e.g. Google's `data-rl` response-language\n * label on AI-generated answers. No attribute name is built in here: which\n * attributes count as declarations is entirely the caller's choice.\n *\n * The document-scope siblings of this signal live in {@link evidenceFromHtml}\n * (`<html lang>`, `content-language`, `og:locale`); this is the same\n * declaration concept one scope down. Like them it is *context* evidence: the\n * fuser weighs it against text reads, and its script guard keeps a\n * declaration from overriding a confident text read — a mislabeled node loses\n * to clear script evidence, while weak or absent text lets the declaration\n * decide.\n *\n * DOM-free by design — this module never touches an Element. The caller\n * passes the attribute values it extracted, keyed by attribute name\n * (`{ lang: el.getAttribute(\"lang\") }`, `{ \"data-rl\": … }`). Each recognized\n * value becomes one evidence item whose `source` is the attribute name under\n * the `node-lang:` namespace (`node-lang:data-rl`) — namespaced so an\n * attribute that happens to share a name with an engine id or evidence kind\n * can never collide in the {@link FuseOptions.weights} keyspace. That lets\n * weights target one attribute — trust a curated vendor label above an\n * often-stale inherited `lang`, or silence an attribute entirely — instead\n * of the whole kind. Empty and unrecognized values are dropped.\n */\nexport function evidenceFromNodeLang(\n attrs: Readonly<Record<string, string | null | undefined>>,\n): LanguageEvidence[] {\n const out: LanguageEvidence[] = [];\n for (const [attribute, raw] of Object.entries(attrs)) {\n if (raw == null || raw === \"\") continue;\n const language = normalizeBCP47(raw);\n if (language === null) continue;\n out.push({\n kind: \"node-lang\",\n language,\n confidence: NODE_LANG_CONFIDENCE,\n source: `node-lang:${attribute}`,\n value: raw,\n });\n }\n return out;\n}\n","import { evidenceFromHeaders } from \"./headers.js\";\nimport { evidenceFromHtml } from \"./html.js\";\nimport { DEFAULT_NODE_LANG_ATTRIBUTES, evidenceFromNodeLang } from \"./node-lang.js\";\nimport { evidenceFromText } from \"./text.js\";\nimport { fuse, type FuseOptions } from \"./fuse.js\";\nimport type {\n Classification,\n DetectContext,\n DetectFn,\n DetectInput,\n DetectorConfig,\n EvidenceSource,\n LanguageEvidence,\n LanguageProfile,\n SyncSource,\n} from \"./types.js\";\n\n/** The always-on, zero-dependency producers. The text producer is bound to the\n * configured candidate roster so its scoring is roster-relative (and so it\n * abstains when no roster was supplied — its signals need candidates). The\n * node-lang producer is bound to the configured declaration-attribute list\n * (standard `lang` only, unless the config opts into vendor attributes), and\n * reads them from `input.attrs` in configured order. */\nfunction builtIns(\n candidates: readonly LanguageProfile[] | undefined,\n nodeLangAttributes: readonly string[],\n): SyncSource[] {\n return [\n {\n id: \"text\",\n sync: true,\n inputs: [\"text\"],\n detect: (i) => evidenceFromText(i.text, candidates),\n },\n { id: \"html\", sync: true, inputs: [\"html\"], detect: (i) => evidenceFromHtml(i.html) },\n {\n id: \"headers\",\n sync: true,\n inputs: [\"headers\"],\n detect: (i) => evidenceFromHeaders(i.headers),\n },\n {\n id: \"node-lang\",\n sync: true,\n inputs: [\"attrs\"],\n detect: (i) => {\n const declared: Record<string, string | null | undefined> = {};\n for (const name of nodeLangAttributes) declared[name] = i.attrs?.[name];\n return evidenceFromNodeLang(declared);\n },\n },\n ];\n}\n\n/** Run a source only when every input it declares is present. */\nfunction applicable(source: EvidenceSource, input: DetectInput): boolean {\n return source.inputs.every((key) => input[key] !== undefined);\n}\n\n/**\n * Build a configured detector. Does the per-roster setup once and returns a\n * `detect` function whose sync/async shape is fixed by the registered engines\n * (see {@link DetectFn}). The built-in producers are always registered; opt-in\n * engines (franc, chrome-ai) are added via `config.engines`.\n */\nexport function compile<const E extends readonly EvidenceSource[] = []>(\n config: DetectorConfig<E> = {},\n): DetectFn<E> {\n const sources: EvidenceSource[] = [\n ...builtIns(config.candidates, config.nodeLangAttributes ?? DEFAULT_NODE_LANG_ATTRIBUTES),\n ...(config.engines ?? []),\n ];\n const hasAsync = sources.some((source) => !source.sync);\n const fuseOptions: FuseOptions = {\n weights: config.weights,\n candidates: config.candidates,\n nonDiscriminatingScript: config.nonDiscriminatingScript,\n };\n\n if (!hasAsync) {\n const detect = (input: DetectInput): Classification => {\n const evidence: LanguageEvidence[] = [];\n for (const source of sources) {\n if (source.sync && applicable(source, input)) evidence.push(...source.detect(input));\n }\n return fuse(evidence, fuseOptions);\n };\n return detect as DetectFn<E>;\n }\n\n const detect = async (input: DetectInput, ctx: DetectContext = {}): Promise<Classification> => {\n const evidence: LanguageEvidence[] = [];\n const pending: Promise<LanguageEvidence[]>[] = [];\n for (const source of sources) {\n if (!applicable(source, input)) continue;\n if (source.sync) evidence.push(...source.detect(input));\n else pending.push(Promise.resolve(source.detect(input, ctx)).catch(() => []));\n }\n for (const batch of await Promise.all(pending)) evidence.push(...batch);\n return fuse(evidence, fuseOptions);\n };\n return detect as DetectFn<E>;\n}\n"]}
|
package/dist/profiles.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { b as LanguageCode, L as LanguageProfile } from './types-
|
|
1
|
+
import { b as LanguageCode, L as LanguageProfile } from './types-DfHZ5hkh.js';
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
4
|
* `langtell/profiles` — ready-to-use {@link LanguageProfile} data.
|
|
@@ -30,6 +30,9 @@ declare const ru: LanguageProfile;
|
|
|
30
30
|
declare const be: LanguageProfile;
|
|
31
31
|
declare const bg: LanguageProfile;
|
|
32
32
|
declare const en: LanguageProfile;
|
|
33
|
+
declare const sr: LanguageProfile;
|
|
34
|
+
declare const mk: LanguageProfile;
|
|
35
|
+
declare const kk: LanguageProfile;
|
|
33
36
|
|
|
34
37
|
/** Registry of shipped profiles, keyed by BCP-47 code. */
|
|
35
38
|
declare const PROFILES: Readonly<Record<LanguageCode, LanguageProfile>>;
|
|
@@ -44,4 +47,4 @@ declare function hasProfile(code: LanguageCode): boolean;
|
|
|
44
47
|
/** Resolve profiles for the given codes, skipping any without a shipped profile. */
|
|
45
48
|
declare function getProfiles(codes: readonly LanguageCode[]): LanguageProfile[];
|
|
46
49
|
|
|
47
|
-
export { PROFILED_CODES, PROFILES, be, bg, en, getProfiles, hasProfile, ru, uk };
|
|
50
|
+
export { PROFILED_CODES, PROFILES, be, bg, en, getProfiles, hasProfile, kk, mk, ru, sr, uk };
|