disarm 0.15.0 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/binding.d.ts +72 -6
- package/binding.js +58 -52
- package/disarm.darwin-arm64.node +0 -0
- package/disarm.darwin-x64.node +0 -0
- package/disarm.linux-arm64-gnu.node +0 -0
- package/disarm.linux-x64-gnu.node +0 -0
- package/disarm.win32-x64-msvc.node +0 -0
- package/index.d.ts +87 -6
- package/index.js +88 -9
- package/package.json +1 -1
package/binding.d.ts
CHANGED
|
@@ -27,6 +27,17 @@ export declare class Lexicon {
|
|
|
27
27
|
export declare class Pipeline {
|
|
28
28
|
/** Run the named pipeline over `text`, returning the cleaned string. */
|
|
29
29
|
process(text: string): string
|
|
30
|
+
/**
|
|
31
|
+
* What the named profile this was built from is for, in one sentence (#860).
|
|
32
|
+
* `null` for a pipeline not built from a profile.
|
|
33
|
+
*/
|
|
34
|
+
get purpose(): string | null
|
|
35
|
+
/**
|
|
36
|
+
* A copy of this pipeline whose confusable passes fold under `digitPolicy` (#646).
|
|
37
|
+
* Throws when the profile has no confusables step and the policy is not the default:
|
|
38
|
+
* a setting that would never run is refused rather than kept.
|
|
39
|
+
*/
|
|
40
|
+
withDigitPolicy(digitPolicy: string): Pipeline
|
|
30
41
|
}
|
|
31
42
|
|
|
32
43
|
/**
|
|
@@ -71,7 +82,7 @@ export interface AutoLangInspection {
|
|
|
71
82
|
* WHEN the word also carries an ASCII letter, which is the gate that keeps ordinary
|
|
72
83
|
* non-Latin text from firing; a delimiter-only string is not reported.
|
|
73
84
|
*/
|
|
74
|
-
export declare function canonicalize(text: string): string
|
|
85
|
+
export declare function canonicalize(text: string, digitPolicy: string): string
|
|
75
86
|
|
|
76
87
|
/**
|
|
77
88
|
* Strip the non-interchange and invisible classes while KEEPING the script (#698).
|
|
@@ -88,17 +99,36 @@ export declare function canonicalize(text: string): string
|
|
|
88
99
|
* — the point of the preset.
|
|
89
100
|
* Canonicalize, but fail rather than silently normalize a structural difference away.
|
|
90
101
|
*/
|
|
91
|
-
export declare function canonicalizeStrict(text: string): string
|
|
102
|
+
export declare function canonicalizeStrict(text: string, digitPolicy: string): string
|
|
92
103
|
|
|
93
104
|
/**
|
|
94
105
|
* Library catalog deduplication key (like `searchKey` plus confusable folding).
|
|
95
106
|
* `lang` selects the transliteration table; `strict_iso9` picks the ISO 9:1995
|
|
96
107
|
* Cyrillic scheme.
|
|
97
108
|
*/
|
|
98
|
-
export declare function catalogKey(text: string, lang: string | undefined | null, strictIso9: boolean): string
|
|
109
|
+
export declare function catalogKey(text: string, lang: string | undefined | null, strictIso9: boolean, digitPolicy: string): string
|
|
99
110
|
|
|
100
111
|
export declare function collapseWhitespace(text: string): string
|
|
101
112
|
|
|
113
|
+
/**
|
|
114
|
+
* TR39 sources whose prototype is in `script`, and how many the bundled tables fold
|
|
115
|
+
* (#963). The denominator `unmappedConfusables` does not have: it measures one table
|
|
116
|
+
* against the whole source population, so a script disarm ships no table for reports a
|
|
117
|
+
* number determined by that absence. An unknown name raises a
|
|
118
|
+
* `DisarmInvalidArgument`-tagged error.
|
|
119
|
+
*/
|
|
120
|
+
export declare function confusableCoverage(script: string): ConfusableCoverage
|
|
121
|
+
|
|
122
|
+
/** Per-script confusable coverage — the fair denominator (#963). */
|
|
123
|
+
export interface ConfusableCoverage {
|
|
124
|
+
/** The script the figures are about, in disarm's spelling. */
|
|
125
|
+
script: string
|
|
126
|
+
/** TR39 sources whose prototype is in this script. */
|
|
127
|
+
sources: number
|
|
128
|
+
/** How many of those `sources` a bundled fold table reaches. */
|
|
129
|
+
folded: number
|
|
130
|
+
}
|
|
131
|
+
|
|
102
132
|
export declare function confusablesVersion(): string
|
|
103
133
|
|
|
104
134
|
/** Replace emoji with their plain names; `strip_modifiers` drops skin-tone marks. */
|
|
@@ -110,6 +140,9 @@ export declare function demojize(text: string, stripModifiers: boolean): string
|
|
|
110
140
|
*/
|
|
111
141
|
export declare function detectScripts(text: string): Array<string>
|
|
112
142
|
|
|
143
|
+
/** Levenshtein edit distance between `a` and `b`, in characters (#894). */
|
|
144
|
+
export declare function editDistance(a: string, b: string): number
|
|
145
|
+
|
|
113
146
|
/** One reason a token is anomalous (a single finding). */
|
|
114
147
|
export interface Finding {
|
|
115
148
|
/** Which branch fired: `"invisible"` | `"bidi"` | `"zalgo"` | `"mixed_script"` | `"bidi_mixed"` | `"leet"` | `"segmentation"`. */
|
|
@@ -256,6 +289,9 @@ export declare function inspectAnomalies(text: string, lexicon: Array<string> |
|
|
|
256
289
|
/** Explain how auto-language detection resolves `text`. */
|
|
257
290
|
export declare function inspectAutoLang(text: string): AutoLangInspection
|
|
258
291
|
|
|
292
|
+
/** Whether `text` is already its own canonical form under `preset` (#730). */
|
|
293
|
+
export declare function isCanonical(text: string, preset: string): boolean
|
|
294
|
+
|
|
259
295
|
/**
|
|
260
296
|
* Whether case folding and simple lowercasing agree, so the value is a stable
|
|
261
297
|
* identity key (#619).
|
|
@@ -336,12 +372,34 @@ export declare function listScripts(): Array<string>
|
|
|
336
372
|
*/
|
|
337
373
|
export declare function mlNormalize(text: string, lang: string | undefined | null, emojiStyle: string, foldCase: boolean): string
|
|
338
374
|
|
|
375
|
+
/**
|
|
376
|
+
* The candidate closest to `value`, with its distance, or `null` beyond `max_distance`
|
|
377
|
+
* (#894). An exact match is reported with distance 0; ties go to the first candidate at
|
|
378
|
+
* the lowest distance.
|
|
379
|
+
*/
|
|
380
|
+
export declare function nearestMatch(value: string, candidates: Array<string>, maxDistance: number): NearestMatch | null
|
|
381
|
+
|
|
382
|
+
/** A candidate and how far `nearestMatch` found it from the value asked about (#894). */
|
|
383
|
+
export interface NearestMatch {
|
|
384
|
+
/** The candidate, in the spelling the caller supplied. */
|
|
385
|
+
value: string
|
|
386
|
+
/** Its edit distance from the value asked about; `0` means the value *is* this candidate. */
|
|
387
|
+
distance: number
|
|
388
|
+
}
|
|
389
|
+
|
|
339
390
|
/** Apply a normalization form: `"NFC"` | `"NFD"` | `"NFKC"` | `"NFKD"`. */
|
|
340
391
|
export declare function normalize(text: string, form: string): string
|
|
341
392
|
|
|
342
393
|
/** Fold cross-script confusables toward `target` (`"latin"` | `"cyrillic"` | `"arabic"` | `"hebrew"`). */
|
|
343
394
|
export declare function normalizeConfusables(text: string, target: string, digitPolicy: string): string
|
|
344
395
|
|
|
396
|
+
/**
|
|
397
|
+
* Replace every emoji with `replacement`, verbatim (#972). The counterpart to
|
|
398
|
+
* `demojize`: that names an emoji from the CLDR table, which is wider than the emoji
|
|
399
|
+
* set, and this replaces what the UCD calls an emoji and nothing else.
|
|
400
|
+
*/
|
|
401
|
+
export declare function replaceEmoji(text: string, replacement: string): string
|
|
402
|
+
|
|
345
403
|
/** Reverse-transliterate Latin → native script. `lang` is `"el"` | `"ru"` | `"uk"`. */
|
|
346
404
|
export declare function reverseTransliterate(text: string, lang: string): string
|
|
347
405
|
|
|
@@ -379,7 +437,15 @@ export interface ScriptMeta {
|
|
|
379
437
|
* Case/accent/script-insensitive search lookup key. `lang` selects the
|
|
380
438
|
* transliteration table (omit for none).
|
|
381
439
|
*/
|
|
382
|
-
export declare function searchKey(text: string, lang
|
|
440
|
+
export declare function searchKey(text: string, lang: string | undefined | null, digitPolicy: string): string
|
|
441
|
+
|
|
442
|
+
/**
|
|
443
|
+
* The TR39 identifier skeleton plus the two prototype classes disarm's table keeps apart
|
|
444
|
+
* (#650). A spoof key: its only job is to make confusable identifiers collide, and its
|
|
445
|
+
* output is never for display. `digit_policy` is `"numeric"` (the letter half only),
|
|
446
|
+
* `"tr39"` (adds `1 ≡ l` and `0 ≡ O`) or `"preserve"`.
|
|
447
|
+
*/
|
|
448
|
+
export declare function skeletonKey(text: string, digitPolicy: string): string
|
|
383
449
|
|
|
384
450
|
/** Generate a URL-safe slug. */
|
|
385
451
|
export declare function slugify(text: string, opts: SlugOptions): string
|
|
@@ -404,7 +470,7 @@ export interface SlugOptions {
|
|
|
404
470
|
* Collation sort key (like `searchKey` but preserves base accented characters
|
|
405
471
|
* for correct ordering). `lang` selects the transliteration table.
|
|
406
472
|
*/
|
|
407
|
-
export declare function sortKey(text: string, lang
|
|
473
|
+
export declare function sortKey(text: string, lang: string | undefined | null, digitPolicy: string): string
|
|
408
474
|
|
|
409
475
|
export declare function stripAccents(text: string): string
|
|
410
476
|
|
|
@@ -417,7 +483,7 @@ export declare function stripFormat(text: string): string
|
|
|
417
483
|
/** Strip every Unicode noncharacter (#413). */
|
|
418
484
|
export declare function stripNoncharacters(text: string): string
|
|
419
485
|
|
|
420
|
-
export declare function stripObfuscation(text: string): string
|
|
486
|
+
export declare function stripObfuscation(text: string, digitPolicy: string): string
|
|
421
487
|
|
|
422
488
|
/** Strip every Private Use Area code point (#413). */
|
|
423
489
|
export declare function stripPua(text: string): string
|
package/binding.js
CHANGED
|
@@ -77,8 +77,8 @@ function requireNative() {
|
|
|
77
77
|
try {
|
|
78
78
|
const binding = require('disarm-android-arm64')
|
|
79
79
|
const bindingPackageVersion = require('disarm-android-arm64/package.json').version
|
|
80
|
-
if (bindingPackageVersion !== '0.
|
|
81
|
-
throw new Error(`Native binding package version mismatch, expected 0.
|
|
80
|
+
if (bindingPackageVersion !== '0.16.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
81
|
+
throw new Error(`Native binding package version mismatch, expected 0.16.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
82
82
|
}
|
|
83
83
|
return binding
|
|
84
84
|
} catch (e) {
|
|
@@ -93,8 +93,8 @@ function requireNative() {
|
|
|
93
93
|
try {
|
|
94
94
|
const binding = require('disarm-android-arm-eabi')
|
|
95
95
|
const bindingPackageVersion = require('disarm-android-arm-eabi/package.json').version
|
|
96
|
-
if (bindingPackageVersion !== '0.
|
|
97
|
-
throw new Error(`Native binding package version mismatch, expected 0.
|
|
96
|
+
if (bindingPackageVersion !== '0.16.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
97
|
+
throw new Error(`Native binding package version mismatch, expected 0.16.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
98
98
|
}
|
|
99
99
|
return binding
|
|
100
100
|
} catch (e) {
|
|
@@ -114,8 +114,8 @@ function requireNative() {
|
|
|
114
114
|
try {
|
|
115
115
|
const binding = require('disarm-win32-x64-gnu')
|
|
116
116
|
const bindingPackageVersion = require('disarm-win32-x64-gnu/package.json').version
|
|
117
|
-
if (bindingPackageVersion !== '0.
|
|
118
|
-
throw new Error(`Native binding package version mismatch, expected 0.
|
|
117
|
+
if (bindingPackageVersion !== '0.16.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
118
|
+
throw new Error(`Native binding package version mismatch, expected 0.16.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
119
119
|
}
|
|
120
120
|
return binding
|
|
121
121
|
} catch (e) {
|
|
@@ -130,8 +130,8 @@ function requireNative() {
|
|
|
130
130
|
try {
|
|
131
131
|
const binding = require('disarm-win32-x64-msvc')
|
|
132
132
|
const bindingPackageVersion = require('disarm-win32-x64-msvc/package.json').version
|
|
133
|
-
if (bindingPackageVersion !== '0.
|
|
134
|
-
throw new Error(`Native binding package version mismatch, expected 0.
|
|
133
|
+
if (bindingPackageVersion !== '0.16.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
134
|
+
throw new Error(`Native binding package version mismatch, expected 0.16.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
135
135
|
}
|
|
136
136
|
return binding
|
|
137
137
|
} catch (e) {
|
|
@@ -147,8 +147,8 @@ function requireNative() {
|
|
|
147
147
|
try {
|
|
148
148
|
const binding = require('disarm-win32-ia32-msvc')
|
|
149
149
|
const bindingPackageVersion = require('disarm-win32-ia32-msvc/package.json').version
|
|
150
|
-
if (bindingPackageVersion !== '0.
|
|
151
|
-
throw new Error(`Native binding package version mismatch, expected 0.
|
|
150
|
+
if (bindingPackageVersion !== '0.16.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
151
|
+
throw new Error(`Native binding package version mismatch, expected 0.16.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
152
152
|
}
|
|
153
153
|
return binding
|
|
154
154
|
} catch (e) {
|
|
@@ -163,8 +163,8 @@ function requireNative() {
|
|
|
163
163
|
try {
|
|
164
164
|
const binding = require('disarm-win32-arm64-msvc')
|
|
165
165
|
const bindingPackageVersion = require('disarm-win32-arm64-msvc/package.json').version
|
|
166
|
-
if (bindingPackageVersion !== '0.
|
|
167
|
-
throw new Error(`Native binding package version mismatch, expected 0.
|
|
166
|
+
if (bindingPackageVersion !== '0.16.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
167
|
+
throw new Error(`Native binding package version mismatch, expected 0.16.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
168
168
|
}
|
|
169
169
|
return binding
|
|
170
170
|
} catch (e) {
|
|
@@ -182,8 +182,8 @@ function requireNative() {
|
|
|
182
182
|
try {
|
|
183
183
|
const binding = require('disarm-darwin-universal')
|
|
184
184
|
const bindingPackageVersion = require('disarm-darwin-universal/package.json').version
|
|
185
|
-
if (bindingPackageVersion !== '0.
|
|
186
|
-
throw new Error(`Native binding package version mismatch, expected 0.
|
|
185
|
+
if (bindingPackageVersion !== '0.16.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
186
|
+
throw new Error(`Native binding package version mismatch, expected 0.16.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
187
187
|
}
|
|
188
188
|
return binding
|
|
189
189
|
} catch (e) {
|
|
@@ -198,8 +198,8 @@ function requireNative() {
|
|
|
198
198
|
try {
|
|
199
199
|
const binding = require('disarm-darwin-x64')
|
|
200
200
|
const bindingPackageVersion = require('disarm-darwin-x64/package.json').version
|
|
201
|
-
if (bindingPackageVersion !== '0.
|
|
202
|
-
throw new Error(`Native binding package version mismatch, expected 0.
|
|
201
|
+
if (bindingPackageVersion !== '0.16.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
202
|
+
throw new Error(`Native binding package version mismatch, expected 0.16.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
203
203
|
}
|
|
204
204
|
return binding
|
|
205
205
|
} catch (e) {
|
|
@@ -214,8 +214,8 @@ function requireNative() {
|
|
|
214
214
|
try {
|
|
215
215
|
const binding = require('disarm-darwin-arm64')
|
|
216
216
|
const bindingPackageVersion = require('disarm-darwin-arm64/package.json').version
|
|
217
|
-
if (bindingPackageVersion !== '0.
|
|
218
|
-
throw new Error(`Native binding package version mismatch, expected 0.
|
|
217
|
+
if (bindingPackageVersion !== '0.16.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
218
|
+
throw new Error(`Native binding package version mismatch, expected 0.16.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
219
219
|
}
|
|
220
220
|
return binding
|
|
221
221
|
} catch (e) {
|
|
@@ -234,8 +234,8 @@ function requireNative() {
|
|
|
234
234
|
try {
|
|
235
235
|
const binding = require('disarm-freebsd-x64')
|
|
236
236
|
const bindingPackageVersion = require('disarm-freebsd-x64/package.json').version
|
|
237
|
-
if (bindingPackageVersion !== '0.
|
|
238
|
-
throw new Error(`Native binding package version mismatch, expected 0.
|
|
237
|
+
if (bindingPackageVersion !== '0.16.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
238
|
+
throw new Error(`Native binding package version mismatch, expected 0.16.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
239
239
|
}
|
|
240
240
|
return binding
|
|
241
241
|
} catch (e) {
|
|
@@ -250,8 +250,8 @@ function requireNative() {
|
|
|
250
250
|
try {
|
|
251
251
|
const binding = require('disarm-freebsd-arm64')
|
|
252
252
|
const bindingPackageVersion = require('disarm-freebsd-arm64/package.json').version
|
|
253
|
-
if (bindingPackageVersion !== '0.
|
|
254
|
-
throw new Error(`Native binding package version mismatch, expected 0.
|
|
253
|
+
if (bindingPackageVersion !== '0.16.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
254
|
+
throw new Error(`Native binding package version mismatch, expected 0.16.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
255
255
|
}
|
|
256
256
|
return binding
|
|
257
257
|
} catch (e) {
|
|
@@ -271,8 +271,8 @@ function requireNative() {
|
|
|
271
271
|
try {
|
|
272
272
|
const binding = require('disarm-linux-x64-musl')
|
|
273
273
|
const bindingPackageVersion = require('disarm-linux-x64-musl/package.json').version
|
|
274
|
-
if (bindingPackageVersion !== '0.
|
|
275
|
-
throw new Error(`Native binding package version mismatch, expected 0.
|
|
274
|
+
if (bindingPackageVersion !== '0.16.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
275
|
+
throw new Error(`Native binding package version mismatch, expected 0.16.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
276
276
|
}
|
|
277
277
|
return binding
|
|
278
278
|
} catch (e) {
|
|
@@ -287,8 +287,8 @@ function requireNative() {
|
|
|
287
287
|
try {
|
|
288
288
|
const binding = require('disarm-linux-x64-gnu')
|
|
289
289
|
const bindingPackageVersion = require('disarm-linux-x64-gnu/package.json').version
|
|
290
|
-
if (bindingPackageVersion !== '0.
|
|
291
|
-
throw new Error(`Native binding package version mismatch, expected 0.
|
|
290
|
+
if (bindingPackageVersion !== '0.16.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
291
|
+
throw new Error(`Native binding package version mismatch, expected 0.16.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
292
292
|
}
|
|
293
293
|
return binding
|
|
294
294
|
} catch (e) {
|
|
@@ -305,8 +305,8 @@ function requireNative() {
|
|
|
305
305
|
try {
|
|
306
306
|
const binding = require('disarm-linux-arm64-musl')
|
|
307
307
|
const bindingPackageVersion = require('disarm-linux-arm64-musl/package.json').version
|
|
308
|
-
if (bindingPackageVersion !== '0.
|
|
309
|
-
throw new Error(`Native binding package version mismatch, expected 0.
|
|
308
|
+
if (bindingPackageVersion !== '0.16.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
309
|
+
throw new Error(`Native binding package version mismatch, expected 0.16.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
310
310
|
}
|
|
311
311
|
return binding
|
|
312
312
|
} catch (e) {
|
|
@@ -321,8 +321,8 @@ function requireNative() {
|
|
|
321
321
|
try {
|
|
322
322
|
const binding = require('disarm-linux-arm64-gnu')
|
|
323
323
|
const bindingPackageVersion = require('disarm-linux-arm64-gnu/package.json').version
|
|
324
|
-
if (bindingPackageVersion !== '0.
|
|
325
|
-
throw new Error(`Native binding package version mismatch, expected 0.
|
|
324
|
+
if (bindingPackageVersion !== '0.16.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
325
|
+
throw new Error(`Native binding package version mismatch, expected 0.16.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
326
326
|
}
|
|
327
327
|
return binding
|
|
328
328
|
} catch (e) {
|
|
@@ -339,8 +339,8 @@ function requireNative() {
|
|
|
339
339
|
try {
|
|
340
340
|
const binding = require('disarm-linux-arm-musleabihf')
|
|
341
341
|
const bindingPackageVersion = require('disarm-linux-arm-musleabihf/package.json').version
|
|
342
|
-
if (bindingPackageVersion !== '0.
|
|
343
|
-
throw new Error(`Native binding package version mismatch, expected 0.
|
|
342
|
+
if (bindingPackageVersion !== '0.16.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
343
|
+
throw new Error(`Native binding package version mismatch, expected 0.16.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
344
344
|
}
|
|
345
345
|
return binding
|
|
346
346
|
} catch (e) {
|
|
@@ -355,8 +355,8 @@ function requireNative() {
|
|
|
355
355
|
try {
|
|
356
356
|
const binding = require('disarm-linux-arm-gnueabihf')
|
|
357
357
|
const bindingPackageVersion = require('disarm-linux-arm-gnueabihf/package.json').version
|
|
358
|
-
if (bindingPackageVersion !== '0.
|
|
359
|
-
throw new Error(`Native binding package version mismatch, expected 0.
|
|
358
|
+
if (bindingPackageVersion !== '0.16.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
359
|
+
throw new Error(`Native binding package version mismatch, expected 0.16.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
360
360
|
}
|
|
361
361
|
return binding
|
|
362
362
|
} catch (e) {
|
|
@@ -373,8 +373,8 @@ function requireNative() {
|
|
|
373
373
|
try {
|
|
374
374
|
const binding = require('disarm-linux-loong64-musl')
|
|
375
375
|
const bindingPackageVersion = require('disarm-linux-loong64-musl/package.json').version
|
|
376
|
-
if (bindingPackageVersion !== '0.
|
|
377
|
-
throw new Error(`Native binding package version mismatch, expected 0.
|
|
376
|
+
if (bindingPackageVersion !== '0.16.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
377
|
+
throw new Error(`Native binding package version mismatch, expected 0.16.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
378
378
|
}
|
|
379
379
|
return binding
|
|
380
380
|
} catch (e) {
|
|
@@ -389,8 +389,8 @@ function requireNative() {
|
|
|
389
389
|
try {
|
|
390
390
|
const binding = require('disarm-linux-loong64-gnu')
|
|
391
391
|
const bindingPackageVersion = require('disarm-linux-loong64-gnu/package.json').version
|
|
392
|
-
if (bindingPackageVersion !== '0.
|
|
393
|
-
throw new Error(`Native binding package version mismatch, expected 0.
|
|
392
|
+
if (bindingPackageVersion !== '0.16.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
393
|
+
throw new Error(`Native binding package version mismatch, expected 0.16.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
394
394
|
}
|
|
395
395
|
return binding
|
|
396
396
|
} catch (e) {
|
|
@@ -407,8 +407,8 @@ function requireNative() {
|
|
|
407
407
|
try {
|
|
408
408
|
const binding = require('disarm-linux-riscv64-musl')
|
|
409
409
|
const bindingPackageVersion = require('disarm-linux-riscv64-musl/package.json').version
|
|
410
|
-
if (bindingPackageVersion !== '0.
|
|
411
|
-
throw new Error(`Native binding package version mismatch, expected 0.
|
|
410
|
+
if (bindingPackageVersion !== '0.16.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
411
|
+
throw new Error(`Native binding package version mismatch, expected 0.16.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
412
412
|
}
|
|
413
413
|
return binding
|
|
414
414
|
} catch (e) {
|
|
@@ -423,8 +423,8 @@ function requireNative() {
|
|
|
423
423
|
try {
|
|
424
424
|
const binding = require('disarm-linux-riscv64-gnu')
|
|
425
425
|
const bindingPackageVersion = require('disarm-linux-riscv64-gnu/package.json').version
|
|
426
|
-
if (bindingPackageVersion !== '0.
|
|
427
|
-
throw new Error(`Native binding package version mismatch, expected 0.
|
|
426
|
+
if (bindingPackageVersion !== '0.16.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
427
|
+
throw new Error(`Native binding package version mismatch, expected 0.16.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
428
428
|
}
|
|
429
429
|
return binding
|
|
430
430
|
} catch (e) {
|
|
@@ -440,8 +440,8 @@ function requireNative() {
|
|
|
440
440
|
try {
|
|
441
441
|
const binding = require('disarm-linux-ppc64-gnu')
|
|
442
442
|
const bindingPackageVersion = require('disarm-linux-ppc64-gnu/package.json').version
|
|
443
|
-
if (bindingPackageVersion !== '0.
|
|
444
|
-
throw new Error(`Native binding package version mismatch, expected 0.
|
|
443
|
+
if (bindingPackageVersion !== '0.16.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
444
|
+
throw new Error(`Native binding package version mismatch, expected 0.16.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
445
445
|
}
|
|
446
446
|
return binding
|
|
447
447
|
} catch (e) {
|
|
@@ -456,8 +456,8 @@ function requireNative() {
|
|
|
456
456
|
try {
|
|
457
457
|
const binding = require('disarm-linux-s390x-gnu')
|
|
458
458
|
const bindingPackageVersion = require('disarm-linux-s390x-gnu/package.json').version
|
|
459
|
-
if (bindingPackageVersion !== '0.
|
|
460
|
-
throw new Error(`Native binding package version mismatch, expected 0.
|
|
459
|
+
if (bindingPackageVersion !== '0.16.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
460
|
+
throw new Error(`Native binding package version mismatch, expected 0.16.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
461
461
|
}
|
|
462
462
|
return binding
|
|
463
463
|
} catch (e) {
|
|
@@ -476,8 +476,8 @@ function requireNative() {
|
|
|
476
476
|
try {
|
|
477
477
|
const binding = require('disarm-openharmony-arm64')
|
|
478
478
|
const bindingPackageVersion = require('disarm-openharmony-arm64/package.json').version
|
|
479
|
-
if (bindingPackageVersion !== '0.
|
|
480
|
-
throw new Error(`Native binding package version mismatch, expected 0.
|
|
479
|
+
if (bindingPackageVersion !== '0.16.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
480
|
+
throw new Error(`Native binding package version mismatch, expected 0.16.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
481
481
|
}
|
|
482
482
|
return binding
|
|
483
483
|
} catch (e) {
|
|
@@ -492,8 +492,8 @@ function requireNative() {
|
|
|
492
492
|
try {
|
|
493
493
|
const binding = require('disarm-openharmony-x64')
|
|
494
494
|
const bindingPackageVersion = require('disarm-openharmony-x64/package.json').version
|
|
495
|
-
if (bindingPackageVersion !== '0.
|
|
496
|
-
throw new Error(`Native binding package version mismatch, expected 0.
|
|
495
|
+
if (bindingPackageVersion !== '0.16.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
496
|
+
throw new Error(`Native binding package version mismatch, expected 0.16.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
497
497
|
}
|
|
498
498
|
return binding
|
|
499
499
|
} catch (e) {
|
|
@@ -508,8 +508,8 @@ function requireNative() {
|
|
|
508
508
|
try {
|
|
509
509
|
const binding = require('disarm-openharmony-arm')
|
|
510
510
|
const bindingPackageVersion = require('disarm-openharmony-arm/package.json').version
|
|
511
|
-
if (bindingPackageVersion !== '0.
|
|
512
|
-
throw new Error(`Native binding package version mismatch, expected 0.
|
|
511
|
+
if (bindingPackageVersion !== '0.16.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
|
|
512
|
+
throw new Error(`Native binding package version mismatch, expected 0.16.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
|
|
513
513
|
}
|
|
514
514
|
return binding
|
|
515
515
|
} catch (e) {
|
|
@@ -594,9 +594,11 @@ module.exports.canonicalize = nativeBinding.canonicalize
|
|
|
594
594
|
module.exports.canonicalizeStrict = nativeBinding.canonicalizeStrict
|
|
595
595
|
module.exports.catalogKey = nativeBinding.catalogKey
|
|
596
596
|
module.exports.collapseWhitespace = nativeBinding.collapseWhitespace
|
|
597
|
+
module.exports.confusableCoverage = nativeBinding.confusableCoverage
|
|
597
598
|
module.exports.confusablesVersion = nativeBinding.confusablesVersion
|
|
598
599
|
module.exports.demojize = nativeBinding.demojize
|
|
599
600
|
module.exports.detectScripts = nativeBinding.detectScripts
|
|
601
|
+
module.exports.editDistance = nativeBinding.editDistance
|
|
600
602
|
module.exports.findKeyCollisions = nativeBinding.findKeyCollisions
|
|
601
603
|
module.exports.findUnmappedConfusables = nativeBinding.findUnmappedConfusables
|
|
602
604
|
module.exports.findUntranslatable = nativeBinding.findUntranslatable
|
|
@@ -611,6 +613,7 @@ module.exports.hasBidiConflict = nativeBinding.hasBidiConflict
|
|
|
611
613
|
module.exports.hasBidiControl = nativeBinding.hasBidiControl
|
|
612
614
|
module.exports.inspectAnomalies = nativeBinding.inspectAnomalies
|
|
613
615
|
module.exports.inspectAutoLang = nativeBinding.inspectAutoLang
|
|
616
|
+
module.exports.isCanonical = nativeBinding.isCanonical
|
|
614
617
|
module.exports.isCaseFoldStable = nativeBinding.isCaseFoldStable
|
|
615
618
|
module.exports.isConfusable = nativeBinding.isConfusable
|
|
616
619
|
module.exports.isMixedScript = nativeBinding.isMixedScript
|
|
@@ -622,12 +625,15 @@ module.exports.langInfo = nativeBinding.langInfo
|
|
|
622
625
|
module.exports.listContextLangs = nativeBinding.listContextLangs
|
|
623
626
|
module.exports.listScripts = nativeBinding.listScripts
|
|
624
627
|
module.exports.mlNormalize = nativeBinding.mlNormalize
|
|
628
|
+
module.exports.nearestMatch = nativeBinding.nearestMatch
|
|
625
629
|
module.exports.normalize = nativeBinding.normalize
|
|
626
630
|
module.exports.normalizeConfusables = nativeBinding.normalizeConfusables
|
|
631
|
+
module.exports.replaceEmoji = nativeBinding.replaceEmoji
|
|
627
632
|
module.exports.reverseTransliterate = nativeBinding.reverseTransliterate
|
|
628
633
|
module.exports.sanitizeFilename = nativeBinding.sanitizeFilename
|
|
629
634
|
module.exports.scriptInfo = nativeBinding.scriptInfo
|
|
630
635
|
module.exports.searchKey = nativeBinding.searchKey
|
|
636
|
+
module.exports.skeletonKey = nativeBinding.skeletonKey
|
|
631
637
|
module.exports.slugify = nativeBinding.slugify
|
|
632
638
|
module.exports.sortKey = nativeBinding.sortKey
|
|
633
639
|
module.exports.stripAccents = nativeBinding.stripAccents
|
package/disarm.darwin-arm64.node
CHANGED
|
Binary file
|
package/disarm.darwin-x64.node
CHANGED
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
package/index.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { Lexicon, Pipeline } from './binding';
|
|
2
|
-
import type { Untranslatable, UnmappedConfusable, AutoLangInspection, KeyCollision, LangMeta, ScriptMeta, Finding as NativeFinding, AnomalyReport as NativeAnomalyReport, HostnameAnalysis as NativeHostnameAnalysis } from './binding';
|
|
3
|
-
export type { Untranslatable, UnmappedConfusable, AutoLangInspection, KeyCollision, LangMeta, ScriptMeta, };
|
|
2
|
+
import type { Untranslatable, UnmappedConfusable, AutoLangInspection, KeyCollision, LangMeta, ScriptMeta, ConfusableCoverage, Finding as NativeFinding, AnomalyReport as NativeAnomalyReport, HostnameAnalysis as NativeHostnameAnalysis } from './binding';
|
|
3
|
+
export type { Untranslatable, UnmappedConfusable, AutoLangInspection, KeyCollision, LangMeta, ScriptMeta, ConfusableCoverage, };
|
|
4
4
|
/** Findings from {@link analyzeHostname}. `suspicious` is a maximally
|
|
5
5
|
* conservative screen, not a precise verdict (#549). */
|
|
6
6
|
export type HostnameAnalysis = NativeHostnameAnalysis;
|
|
@@ -25,7 +25,7 @@ export { Lexicon };
|
|
|
25
25
|
*/
|
|
26
26
|
export { Pipeline };
|
|
27
27
|
/** The anomaly branch that fired for a finding. */
|
|
28
|
-
export type AnomalyKind = 'invisible' | 'bidi' | 'bidi_mixed' | 'zalgo' | 'mixed_script' | 'leet' | 'segmentation' | 'control' | 'compat_fold' | 'confusable' | 'enclosing_mark' | 'mixed_numbers' | 'duplicate_mark';
|
|
28
|
+
export type AnomalyKind = 'invisible' | 'bidi' | 'bidi_mixed' | 'zalgo' | 'mixed_script' | 'leet' | 'segmentation' | 'control' | 'compat_fold' | 'confusable' | 'enclosing_mark' | 'mixed_numbers' | 'duplicate_mark' | 'deletion' | 'smuggled';
|
|
29
29
|
/**
|
|
30
30
|
* One reason a token is anomalous. Re-typed over the generated {@link NativeFinding}
|
|
31
31
|
* so `kind` is the {@link AnomalyKind} string-union rather than a bare `string`.
|
|
@@ -93,6 +93,20 @@ export declare function normalizeConfusables(text: string, options?: {
|
|
|
93
93
|
target?: TargetScript;
|
|
94
94
|
digitPolicy?: DigitPolicy;
|
|
95
95
|
}): string;
|
|
96
|
+
/**
|
|
97
|
+
* Whether `text` is already its own canonical form under `preset` (#730).
|
|
98
|
+
*
|
|
99
|
+
* The verification-path counterpart to the presets: text in, normalized text out is the
|
|
100
|
+
* generation path, and this is the question a caller asks about bytes that arrive already
|
|
101
|
+
* bound. `hasAnomalies` is not this predicate — 142,760 assigned code points are reported
|
|
102
|
+
* clean by the detector and are not their own canonical form.
|
|
103
|
+
*
|
|
104
|
+
* `preset` is any name in the preset registry or any profile, defaulting to
|
|
105
|
+
* `'canonicalize'`. An unknown one throws {@link DisarmInvalidArgument}.
|
|
106
|
+
*/
|
|
107
|
+
export declare function isCanonical(text: string, options?: {
|
|
108
|
+
preset?: string;
|
|
109
|
+
}): boolean;
|
|
96
110
|
/** Whether `text` contains a character confusable with `target` (default `'latin'`). */
|
|
97
111
|
export declare function isConfusable(text: string, options?: {
|
|
98
112
|
target?: TargetScript;
|
|
@@ -173,6 +187,19 @@ export declare function findKeyCollisions(values: string[], key: CollisionKey, o
|
|
|
173
187
|
export declare function demojize(text: string, options?: {
|
|
174
188
|
stripModifiers?: boolean;
|
|
175
189
|
}): string;
|
|
190
|
+
/** Replace every emoji with `replacement`, verbatim (#972).
|
|
191
|
+
*
|
|
192
|
+
* The counterpart to {@link demojize}, and a different question of a different table.
|
|
193
|
+
* `demojize` asks *what does CLDR call this?*, so its domain is the CLDR name table,
|
|
194
|
+
* which is wider than the emoji: `demojize('x™y')` is `'x trade mark y'`. This asks *is
|
|
195
|
+
* this an emoji by the UCD's properties?* — `Emoji_Presentation=Yes`, an `Emoji=Yes` base
|
|
196
|
+
* carrying `U+FE0F`, and the ZWJ, modifier, keycap and flag sequences on those. Nothing
|
|
197
|
+
* else moves.
|
|
198
|
+
*
|
|
199
|
+
* `replacement` is inserted exactly as given, with no padding and no whitespace collapse:
|
|
200
|
+
* `''` closes an intra-word split (`aa🔥bb` → `aabb`) and `' '` keeps two words apart
|
|
201
|
+
* (`stop🛑now` → `stop now`), and no rule serves both. */
|
|
202
|
+
export declare function replaceEmoji(text: string, replacement?: string): string;
|
|
176
203
|
/** Apply a Unicode normalization `form` (default `'NFC'`). */
|
|
177
204
|
export declare function normalize(text: string, options?: {
|
|
178
205
|
form?: NormalizationForm;
|
|
@@ -219,7 +246,9 @@ export declare function isZalgo(text: string, options?: {
|
|
|
219
246
|
* the half of the pair that lets a caller reject input instead of comparing a value the
|
|
220
247
|
* sender never wrote.
|
|
221
248
|
*/
|
|
222
|
-
export declare function canonicalizeStrict(text: string
|
|
249
|
+
export declare function canonicalizeStrict(text: string, options?: {
|
|
250
|
+
digitPolicy?: DigitPolicy;
|
|
251
|
+
}): string;
|
|
223
252
|
/**
|
|
224
253
|
* Strip the non-interchange and invisible classes while KEEPING the script.
|
|
225
254
|
*
|
|
@@ -231,7 +260,9 @@ export declare function canonicalizeStrict(text: string): string;
|
|
|
231
260
|
*/
|
|
232
261
|
export declare function stripFormat(text: string): string;
|
|
233
262
|
/** Remove obfuscation (zero-width, bidi, combining-mark abuse, homoglyphs) while keeping legible content. */
|
|
234
|
-
export declare function stripObfuscation(text: string
|
|
263
|
+
export declare function stripObfuscation(text: string, options?: {
|
|
264
|
+
digitPolicy?: DigitPolicy;
|
|
265
|
+
}): string;
|
|
235
266
|
/**
|
|
236
267
|
* Canonicalize text for security-sensitive comparison: NFKC → strip bidi/format
|
|
237
268
|
* → strip invisible classes (#413) → strip control → strip zero-width → collapse
|
|
@@ -241,7 +272,9 @@ export declare function stripObfuscation(text: string): string;
|
|
|
241
272
|
* The name describes the mechanism (Unicode canonicalization for matching), not
|
|
242
273
|
* a safety guarantee — this is not an output sanitizer; encode at the sink.
|
|
243
274
|
*/
|
|
244
|
-
export declare function canonicalize(text: string
|
|
275
|
+
export declare function canonicalize(text: string, options?: {
|
|
276
|
+
digitPolicy?: DigitPolicy;
|
|
277
|
+
}): string;
|
|
245
278
|
/**
|
|
246
279
|
* @deprecated Renamed to {@link canonicalize} in 0.11 (the `*Clean` name
|
|
247
280
|
* overpromised safety); removed in 1.0.
|
|
@@ -272,6 +305,7 @@ export declare function sanitizeFilename(text: string, options?: SanitizeFilenam
|
|
|
272
305
|
*/
|
|
273
306
|
export declare function searchKey(text: string, options?: {
|
|
274
307
|
lang?: string;
|
|
308
|
+
digitPolicy?: DigitPolicy;
|
|
275
309
|
}): string;
|
|
276
310
|
/**
|
|
277
311
|
* Collation sort key — like {@link searchKey} but preserves base accented
|
|
@@ -279,6 +313,7 @@ export declare function searchKey(text: string, options?: {
|
|
|
279
313
|
*/
|
|
280
314
|
export declare function sortKey(text: string, options?: {
|
|
281
315
|
lang?: string;
|
|
316
|
+
digitPolicy?: DigitPolicy;
|
|
282
317
|
}): string;
|
|
283
318
|
/**
|
|
284
319
|
* Library catalog deduplication key — like {@link searchKey} plus confusable
|
|
@@ -288,7 +323,38 @@ export declare function sortKey(text: string, options?: {
|
|
|
288
323
|
export declare function catalogKey(text: string, options?: {
|
|
289
324
|
lang?: string;
|
|
290
325
|
strictIso9?: boolean;
|
|
326
|
+
digitPolicy?: DigitPolicy;
|
|
291
327
|
}): string;
|
|
328
|
+
/**
|
|
329
|
+
* The TR39 identifier skeleton plus the two prototype classes disarm's table keeps
|
|
330
|
+
* apart (#650). A spoof key: its only job is to make confusable identifiers collide,
|
|
331
|
+
* and its output is never for display. `digitPolicy` `'numeric'` (default) applies the
|
|
332
|
+
* letter half only; `'tr39'` adds `1 ≡ l` and `0 ≡ O`; `'preserve'` keeps a non-Latin
|
|
333
|
+
* numeral in its script.
|
|
334
|
+
*/
|
|
335
|
+
export declare function skeletonKey(text: string, options?: {
|
|
336
|
+
digitPolicy?: DigitPolicy;
|
|
337
|
+
}): string;
|
|
338
|
+
/**
|
|
339
|
+
* Levenshtein edit distance between `a` and `b`, in characters (#894) — the one class of
|
|
340
|
+
* registry spoofing the confusable tables deliberately do not model (`paypa1`, `adm1n`).
|
|
341
|
+
* Canonicalize both sides first when composed and decomposed spellings should compare
|
|
342
|
+
* equal.
|
|
343
|
+
*/
|
|
344
|
+
export declare function editDistance(a: string, b: string): number;
|
|
345
|
+
/** A candidate and how far {@link nearestMatch} found it from the value asked about. */
|
|
346
|
+
export interface NearestMatch {
|
|
347
|
+
value: string;
|
|
348
|
+
distance: number;
|
|
349
|
+
}
|
|
350
|
+
/**
|
|
351
|
+
* The candidate closest to `value`, with its distance, or `null` beyond `maxDistance`
|
|
352
|
+
* (default 1). Reports; it does not decide. An exact match is reported with distance 0,
|
|
353
|
+
* and ties go to the first candidate at the lowest distance (#894).
|
|
354
|
+
*/
|
|
355
|
+
export declare function nearestMatch(value: string, candidates: string[], options?: {
|
|
356
|
+
maxDistance?: number;
|
|
357
|
+
}): NearestMatch | null;
|
|
292
358
|
/** Options for {@link mlNormalize}. */
|
|
293
359
|
export interface MlNormalizeOptions {
|
|
294
360
|
/** Language code selecting the transliteration table; omit for none. */
|
|
@@ -370,6 +436,21 @@ export declare function langInfo(code: string): LangMeta;
|
|
|
370
436
|
* unknown name throws {@link DisarmInvalidArgument}.
|
|
371
437
|
*/
|
|
372
438
|
export declare function scriptInfo(name: string): ScriptMeta;
|
|
439
|
+
/** TR39 sources whose prototype is in `script`, and how many of those disarm's bundled
|
|
440
|
+
* tables fold (#963).
|
|
441
|
+
*
|
|
442
|
+
* The denominator {@link unmappedConfusables} does not have. That function measures one
|
|
443
|
+
* bundled table against the whole 6,565-source population, which is the right question
|
|
444
|
+
* for a target disarm ships and a misleading one for a script it does not: Greek reports
|
|
445
|
+
* almost the entire population unmapped, and the number means only "there is no Greek
|
|
446
|
+
* table".
|
|
447
|
+
*
|
|
448
|
+
* `folded` counts sources any bundled table reaches, not sources folded *toward* this
|
|
449
|
+
* script — Greek is 71 of 159 because the Latin table folds Greek letters that look
|
|
450
|
+
* Latin. The grouping uses the UCD's script property, so `"Yi"` and 18 other scripts
|
|
451
|
+
* disarm's own enum does not name are addressable here. A script disarm knows that TR39
|
|
452
|
+
* never uses as a prototype returns 0 of 0. */
|
|
453
|
+
export declare function confusableCoverage(script: string): ConfusableCoverage;
|
|
373
454
|
/**
|
|
374
455
|
* The UCD release disarm's normalizer implements. Not a library-wide Unicode version —
|
|
375
456
|
* the bundled tables track different releases. This is the one integrators ask about,
|
package/index.js
CHANGED
|
@@ -38,6 +38,7 @@ exports.transliterate = transliterate;
|
|
|
38
38
|
exports.reverseTransliterate = reverseTransliterate;
|
|
39
39
|
exports.findUntranslatable = findUntranslatable;
|
|
40
40
|
exports.normalizeConfusables = normalizeConfusables;
|
|
41
|
+
exports.isCanonical = isCanonical;
|
|
41
42
|
exports.isConfusable = isConfusable;
|
|
42
43
|
exports.unmappedConfusables = unmappedConfusables;
|
|
43
44
|
exports.findUnmappedConfusables = findUnmappedConfusables;
|
|
@@ -47,6 +48,7 @@ exports.foldCase = foldCase;
|
|
|
47
48
|
exports.isCaseFoldStable = isCaseFoldStable;
|
|
48
49
|
exports.findKeyCollisions = findKeyCollisions;
|
|
49
50
|
exports.demojize = demojize;
|
|
51
|
+
exports.replaceEmoji = replaceEmoji;
|
|
50
52
|
exports.normalize = normalize;
|
|
51
53
|
exports.isNormalized = isNormalized;
|
|
52
54
|
exports.collapseWhitespace = collapseWhitespace;
|
|
@@ -69,6 +71,9 @@ exports.sanitizeFilename = sanitizeFilename;
|
|
|
69
71
|
exports.searchKey = searchKey;
|
|
70
72
|
exports.sortKey = sortKey;
|
|
71
73
|
exports.catalogKey = catalogKey;
|
|
74
|
+
exports.skeletonKey = skeletonKey;
|
|
75
|
+
exports.editDistance = editDistance;
|
|
76
|
+
exports.nearestMatch = nearestMatch;
|
|
72
77
|
exports.mlNormalize = mlNormalize;
|
|
73
78
|
exports.graphemeLen = graphemeLen;
|
|
74
79
|
exports.graphemeSplit = graphemeSplit;
|
|
@@ -84,6 +89,7 @@ exports.hasBidiConflict = hasBidiConflict;
|
|
|
84
89
|
exports.inspectAutoLang = inspectAutoLang;
|
|
85
90
|
exports.langInfo = langInfo;
|
|
86
91
|
exports.scriptInfo = scriptInfo;
|
|
92
|
+
exports.confusableCoverage = confusableCoverage;
|
|
87
93
|
exports.unicodeVersion = unicodeVersion;
|
|
88
94
|
exports.keySchemaVersion = keySchemaVersion;
|
|
89
95
|
exports.confusablesVersion = confusablesVersion;
|
|
@@ -167,6 +173,20 @@ function findUntranslatable(text, options = {}) {
|
|
|
167
173
|
function normalizeConfusables(text, options = {}) {
|
|
168
174
|
return call(() => native.normalizeConfusables(text, options.target ?? 'latin', options.digitPolicy ?? 'numeric'));
|
|
169
175
|
}
|
|
176
|
+
/**
|
|
177
|
+
* Whether `text` is already its own canonical form under `preset` (#730).
|
|
178
|
+
*
|
|
179
|
+
* The verification-path counterpart to the presets: text in, normalized text out is the
|
|
180
|
+
* generation path, and this is the question a caller asks about bytes that arrive already
|
|
181
|
+
* bound. `hasAnomalies` is not this predicate — 142,760 assigned code points are reported
|
|
182
|
+
* clean by the detector and are not their own canonical form.
|
|
183
|
+
*
|
|
184
|
+
* `preset` is any name in the preset registry or any profile, defaulting to
|
|
185
|
+
* `'canonicalize'`. An unknown one throws {@link DisarmInvalidArgument}.
|
|
186
|
+
*/
|
|
187
|
+
function isCanonical(text, options = {}) {
|
|
188
|
+
return call(() => native.isCanonical(text, options.preset ?? 'canonicalize'));
|
|
189
|
+
}
|
|
170
190
|
/** Whether `text` contains a character confusable with `target` (default `'latin'`). */
|
|
171
191
|
function isConfusable(text, options = {}) {
|
|
172
192
|
return call(() => native.isConfusable(text, options.target ?? 'latin'));
|
|
@@ -252,6 +272,21 @@ function findKeyCollisions(values, key, options = {}) {
|
|
|
252
272
|
function demojize(text, options = {}) {
|
|
253
273
|
return native.demojize(text, options.stripModifiers ?? false);
|
|
254
274
|
}
|
|
275
|
+
/** Replace every emoji with `replacement`, verbatim (#972).
|
|
276
|
+
*
|
|
277
|
+
* The counterpart to {@link demojize}, and a different question of a different table.
|
|
278
|
+
* `demojize` asks *what does CLDR call this?*, so its domain is the CLDR name table,
|
|
279
|
+
* which is wider than the emoji: `demojize('x™y')` is `'x trade mark y'`. This asks *is
|
|
280
|
+
* this an emoji by the UCD's properties?* — `Emoji_Presentation=Yes`, an `Emoji=Yes` base
|
|
281
|
+
* carrying `U+FE0F`, and the ZWJ, modifier, keycap and flag sequences on those. Nothing
|
|
282
|
+
* else moves.
|
|
283
|
+
*
|
|
284
|
+
* `replacement` is inserted exactly as given, with no padding and no whitespace collapse:
|
|
285
|
+
* `''` closes an intra-word split (`aa🔥bb` → `aabb`) and `' '` keeps two words apart
|
|
286
|
+
* (`stop🛑now` → `stop now`), and no rule serves both. */
|
|
287
|
+
function replaceEmoji(text, replacement = '') {
|
|
288
|
+
return native.replaceEmoji(text, replacement);
|
|
289
|
+
}
|
|
255
290
|
// ── Normalization ───────────────────────────────────────────────────────────
|
|
256
291
|
/** Apply a Unicode normalization `form` (default `'NFC'`). */
|
|
257
292
|
function normalize(text, options = {}) {
|
|
@@ -317,8 +352,8 @@ function isZalgo(text, options = {}) {
|
|
|
317
352
|
* the half of the pair that lets a caller reject input instead of comparing a value the
|
|
318
353
|
* sender never wrote.
|
|
319
354
|
*/
|
|
320
|
-
function canonicalizeStrict(text) {
|
|
321
|
-
return call(() => native.canonicalizeStrict(text));
|
|
355
|
+
function canonicalizeStrict(text, options = {}) {
|
|
356
|
+
return call(() => native.canonicalizeStrict(text, options.digitPolicy ?? 'numeric'));
|
|
322
357
|
}
|
|
323
358
|
/**
|
|
324
359
|
* Strip the non-interchange and invisible classes while KEEPING the script.
|
|
@@ -333,8 +368,8 @@ function stripFormat(text) {
|
|
|
333
368
|
return native.stripFormat(text);
|
|
334
369
|
}
|
|
335
370
|
/** Remove obfuscation (zero-width, bidi, combining-mark abuse, homoglyphs) while keeping legible content. */
|
|
336
|
-
function stripObfuscation(text) {
|
|
337
|
-
return call(() => native.stripObfuscation(text));
|
|
371
|
+
function stripObfuscation(text, options = {}) {
|
|
372
|
+
return call(() => native.stripObfuscation(text, options.digitPolicy ?? 'numeric'));
|
|
338
373
|
}
|
|
339
374
|
/**
|
|
340
375
|
* Canonicalize text for security-sensitive comparison: NFKC → strip bidi/format
|
|
@@ -345,8 +380,8 @@ function stripObfuscation(text) {
|
|
|
345
380
|
* The name describes the mechanism (Unicode canonicalization for matching), not
|
|
346
381
|
* a safety guarantee — this is not an output sanitizer; encode at the sink.
|
|
347
382
|
*/
|
|
348
|
-
function canonicalize(text) {
|
|
349
|
-
return call(() => native.canonicalize(text));
|
|
383
|
+
function canonicalize(text, options = {}) {
|
|
384
|
+
return call(() => native.canonicalize(text, options.digitPolicy ?? 'numeric'));
|
|
350
385
|
}
|
|
351
386
|
/**
|
|
352
387
|
* @deprecated Renamed to {@link canonicalize} in 0.11 (the `*Clean` name
|
|
@@ -377,14 +412,14 @@ function sanitizeFilename(text, options = {}) {
|
|
|
377
412
|
* without confusable folding). `lang` selects the transliteration table.
|
|
378
413
|
*/
|
|
379
414
|
function searchKey(text, options = {}) {
|
|
380
|
-
return call(() => native.searchKey(text, options.lang ?? undefined));
|
|
415
|
+
return call(() => native.searchKey(text, options.lang ?? undefined, options.digitPolicy ?? 'numeric'));
|
|
381
416
|
}
|
|
382
417
|
/**
|
|
383
418
|
* Collation sort key — like {@link searchKey} but preserves base accented
|
|
384
419
|
* characters for correct ordering. `lang` selects the transliteration table.
|
|
385
420
|
*/
|
|
386
421
|
function sortKey(text, options = {}) {
|
|
387
|
-
return call(() => native.sortKey(text, options.lang ?? undefined));
|
|
422
|
+
return call(() => native.sortKey(text, options.lang ?? undefined, options.digitPolicy ?? 'numeric'));
|
|
388
423
|
}
|
|
389
424
|
/**
|
|
390
425
|
* Library catalog deduplication key — like {@link searchKey} plus confusable
|
|
@@ -392,7 +427,34 @@ function sortKey(text, options = {}) {
|
|
|
392
427
|
* `false`) picks the ISO 9:1995 Cyrillic scheme.
|
|
393
428
|
*/
|
|
394
429
|
function catalogKey(text, options = {}) {
|
|
395
|
-
return call(() => native.catalogKey(text, options.lang ?? undefined, options.strictIso9 ?? false));
|
|
430
|
+
return call(() => native.catalogKey(text, options.lang ?? undefined, options.strictIso9 ?? false, options.digitPolicy ?? 'numeric'));
|
|
431
|
+
}
|
|
432
|
+
/**
|
|
433
|
+
* The TR39 identifier skeleton plus the two prototype classes disarm's table keeps
|
|
434
|
+
* apart (#650). A spoof key: its only job is to make confusable identifiers collide,
|
|
435
|
+
* and its output is never for display. `digitPolicy` `'numeric'` (default) applies the
|
|
436
|
+
* letter half only; `'tr39'` adds `1 ≡ l` and `0 ≡ O`; `'preserve'` keeps a non-Latin
|
|
437
|
+
* numeral in its script.
|
|
438
|
+
*/
|
|
439
|
+
function skeletonKey(text, options = {}) {
|
|
440
|
+
return call(() => native.skeletonKey(text, options.digitPolicy ?? 'numeric'));
|
|
441
|
+
}
|
|
442
|
+
/**
|
|
443
|
+
* Levenshtein edit distance between `a` and `b`, in characters (#894) — the one class of
|
|
444
|
+
* registry spoofing the confusable tables deliberately do not model (`paypa1`, `adm1n`).
|
|
445
|
+
* Canonicalize both sides first when composed and decomposed spellings should compare
|
|
446
|
+
* equal.
|
|
447
|
+
*/
|
|
448
|
+
function editDistance(a, b) {
|
|
449
|
+
return native.editDistance(a, b);
|
|
450
|
+
}
|
|
451
|
+
/**
|
|
452
|
+
* The candidate closest to `value`, with its distance, or `null` beyond `maxDistance`
|
|
453
|
+
* (default 1). Reports; it does not decide. An exact match is reported with distance 0,
|
|
454
|
+
* and ties go to the first candidate at the lowest distance (#894).
|
|
455
|
+
*/
|
|
456
|
+
function nearestMatch(value, candidates, options = {}) {
|
|
457
|
+
return call(() => native.nearestMatch(value, candidates, options.maxDistance ?? 1)) ?? null;
|
|
396
458
|
}
|
|
397
459
|
/**
|
|
398
460
|
* ML/NLP normalization: NFKC → emoji→text → transliterate → strip accents →
|
|
@@ -487,6 +549,23 @@ function langInfo(code) {
|
|
|
487
549
|
function scriptInfo(name) {
|
|
488
550
|
return call(() => native.scriptInfo(name));
|
|
489
551
|
}
|
|
552
|
+
/** TR39 sources whose prototype is in `script`, and how many of those disarm's bundled
|
|
553
|
+
* tables fold (#963).
|
|
554
|
+
*
|
|
555
|
+
* The denominator {@link unmappedConfusables} does not have. That function measures one
|
|
556
|
+
* bundled table against the whole 6,565-source population, which is the right question
|
|
557
|
+
* for a target disarm ships and a misleading one for a script it does not: Greek reports
|
|
558
|
+
* almost the entire population unmapped, and the number means only "there is no Greek
|
|
559
|
+
* table".
|
|
560
|
+
*
|
|
561
|
+
* `folded` counts sources any bundled table reaches, not sources folded *toward* this
|
|
562
|
+
* script — Greek is 71 of 159 because the Latin table folds Greek letters that look
|
|
563
|
+
* Latin. The grouping uses the UCD's script property, so `"Yi"` and 18 other scripts
|
|
564
|
+
* disarm's own enum does not name are addressable here. A script disarm knows that TR39
|
|
565
|
+
* never uses as a prototype returns 0 of 0. */
|
|
566
|
+
function confusableCoverage(script) {
|
|
567
|
+
return call(() => native.confusableCoverage(script));
|
|
568
|
+
}
|
|
490
569
|
/**
|
|
491
570
|
* The UCD release disarm's normalizer implements. Not a library-wide Unicode version —
|
|
492
571
|
* the bundled tables track different releases. This is the one integrators ask about,
|