disarm 0.15.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/binding.d.ts +106 -14
- package/binding.js +300 -103
- package/disarm.darwin-arm64.node +0 -0
- package/disarm.darwin-x64.node +0 -0
- package/disarm.linux-arm64-gnu.node +0 -0
- package/disarm.linux-x64-gnu.node +2 -2
- package/disarm.win32-x64-msvc.node +0 -0
- package/index.d.ts +142 -24
- package/index.js +263 -74
- package/package.json +4 -4
package/index.js
CHANGED
|
@@ -38,6 +38,7 @@ exports.transliterate = transliterate;
|
|
|
38
38
|
exports.reverseTransliterate = reverseTransliterate;
|
|
39
39
|
exports.findUntranslatable = findUntranslatable;
|
|
40
40
|
exports.normalizeConfusables = normalizeConfusables;
|
|
41
|
+
exports.isCanonical = isCanonical;
|
|
41
42
|
exports.isConfusable = isConfusable;
|
|
42
43
|
exports.unmappedConfusables = unmappedConfusables;
|
|
43
44
|
exports.findUnmappedConfusables = findUnmappedConfusables;
|
|
@@ -47,6 +48,7 @@ exports.foldCase = foldCase;
|
|
|
47
48
|
exports.isCaseFoldStable = isCaseFoldStable;
|
|
48
49
|
exports.findKeyCollisions = findKeyCollisions;
|
|
49
50
|
exports.demojize = demojize;
|
|
51
|
+
exports.replaceEmoji = replaceEmoji;
|
|
50
52
|
exports.normalize = normalize;
|
|
51
53
|
exports.isNormalized = isNormalized;
|
|
52
54
|
exports.collapseWhitespace = collapseWhitespace;
|
|
@@ -69,6 +71,9 @@ exports.sanitizeFilename = sanitizeFilename;
|
|
|
69
71
|
exports.searchKey = searchKey;
|
|
70
72
|
exports.sortKey = sortKey;
|
|
71
73
|
exports.catalogKey = catalogKey;
|
|
74
|
+
exports.skeletonKey = skeletonKey;
|
|
75
|
+
exports.editDistance = editDistance;
|
|
76
|
+
exports.nearestMatch = nearestMatch;
|
|
72
77
|
exports.mlNormalize = mlNormalize;
|
|
73
78
|
exports.graphemeLen = graphemeLen;
|
|
74
79
|
exports.graphemeSplit = graphemeSplit;
|
|
@@ -84,6 +89,7 @@ exports.hasBidiConflict = hasBidiConflict;
|
|
|
84
89
|
exports.inspectAutoLang = inspectAutoLang;
|
|
85
90
|
exports.langInfo = langInfo;
|
|
86
91
|
exports.scriptInfo = scriptInfo;
|
|
92
|
+
exports.confusableCoverage = confusableCoverage;
|
|
87
93
|
exports.unicodeVersion = unicodeVersion;
|
|
88
94
|
exports.keySchemaVersion = keySchemaVersion;
|
|
89
95
|
exports.confusablesVersion = confusablesVersion;
|
|
@@ -102,8 +108,40 @@ exports.inspectAnomalies = inspectAnomalies;
|
|
|
102
108
|
*/
|
|
103
109
|
const native = __importStar(require("./binding"));
|
|
104
110
|
const binding_1 = require("./binding");
|
|
105
|
-
Object.defineProperty(exports, "Lexicon", { enumerable: true, get: function () { return binding_1.Lexicon; } });
|
|
106
111
|
Object.defineProperty(exports, "Pipeline", { enumerable: true, get: function () { return binding_1.Pipeline; } });
|
|
112
|
+
/**
|
|
113
|
+
* A reusable, opaque lexicon handle (HAI-SDLC 6.1). `hasAnomalies` /
|
|
114
|
+
* `inspectAnomalies` rebuild an internal set from the caller's word array on
|
|
115
|
+
* every call; constructing a `Lexicon` once (`new Lexicon([...])`) and passing
|
|
116
|
+
* it instead builds that set a single time and reuses it across calls.
|
|
117
|
+
*
|
|
118
|
+
* A thin subclass of the native handle, so a bad argument to the constructor throws a
|
|
119
|
+
* {@link DisarmError} like every other entry point (`formal/bindings`, N2).
|
|
120
|
+
*/
|
|
121
|
+
class Lexicon extends binding_1.Lexicon {
|
|
122
|
+
constructor(words) {
|
|
123
|
+
try {
|
|
124
|
+
super(words);
|
|
125
|
+
}
|
|
126
|
+
catch (e) {
|
|
127
|
+
throw translate(e);
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
exports.Lexicon = Lexicon;
|
|
132
|
+
// The handle's methods are native; route them through `call()` as every function is, so a
|
|
133
|
+
// wrong argument or a refused digit policy throws a `DisarmError` rather than a bare
|
|
134
|
+
// `Error` whose message still carries the shim's tag (`formal/bindings`, N2). Patched on
|
|
135
|
+
// the prototype, so `getPipeline`'s native instances and `instanceof Pipeline` are
|
|
136
|
+
// unchanged.
|
|
137
|
+
const nativeProcess = binding_1.Pipeline.prototype.process;
|
|
138
|
+
const nativeWithDigitPolicy = binding_1.Pipeline.prototype.withDigitPolicy;
|
|
139
|
+
binding_1.Pipeline.prototype.process = function process(text) {
|
|
140
|
+
return call(() => nativeProcess.call(this, text));
|
|
141
|
+
};
|
|
142
|
+
binding_1.Pipeline.prototype.withDigitPolicy = function withDigitPolicy(digitPolicy) {
|
|
143
|
+
return call(() => nativeWithDigitPolicy.call(this, digitPolicy));
|
|
144
|
+
};
|
|
107
145
|
// ── Errors ──────────────────────────────────────────────────────────────────
|
|
108
146
|
/** Base class for every error disarm raises, so callers can `catch (e) { if (e instanceof DisarmError) … }`. */
|
|
109
147
|
class DisarmError extends Error {
|
|
@@ -123,33 +161,83 @@ class DisarmInvalidArgument extends DisarmError {
|
|
|
123
161
|
exports.DisarmInvalidArgument = DisarmInvalidArgument;
|
|
124
162
|
const INVALID_ARG_TAG = 'DisarmInvalidArgument: ';
|
|
125
163
|
const ERROR_TAG = 'DisarmError: ';
|
|
164
|
+
/**
|
|
165
|
+
* The napi statuses an argument conversion fails with: a value of the wrong JS type
|
|
166
|
+
* (`transliterate(123)`, `stripAccents(undefined)`). They are invalid arguments, as a
|
|
167
|
+
* `TypeError` is in the Ruby binding.
|
|
168
|
+
*/
|
|
169
|
+
const NAPI_ARGUMENT_STATUSES = new Set([
|
|
170
|
+
'InvalidArg',
|
|
171
|
+
'ObjectExpected',
|
|
172
|
+
'StringExpected',
|
|
173
|
+
'NameExpected',
|
|
174
|
+
'FunctionExpected',
|
|
175
|
+
'NumberExpected',
|
|
176
|
+
'BooleanExpected',
|
|
177
|
+
'ArrayExpected',
|
|
178
|
+
'BigintExpected',
|
|
179
|
+
]);
|
|
126
180
|
/**
|
|
127
181
|
* Run a native call, re-raising its tagged napi error as the matching
|
|
128
182
|
* `DisarmError` subclass. The native shim prefixes fallible messages with
|
|
129
183
|
* `"DisarmInvalidArgument: "` or `"DisarmError: "`; we strip the matched tag
|
|
130
|
-
* cleanly.
|
|
131
|
-
*
|
|
184
|
+
* cleanly. An argument napi could not convert is a `DisarmInvalidArgument`. Any
|
|
185
|
+
* other throw — an untagged `Error`, or a non-`Error` value — is still wrapped as a
|
|
186
|
+
* `DisarmError` so nothing leaks out unwrapped.
|
|
187
|
+
*
|
|
188
|
+
* Every export goes through here, the infallible ones included: they throw on an
|
|
189
|
+
* argument of the wrong type, and "everything disarm throws is a `DisarmError`" was
|
|
190
|
+
* false for them while they called the native function directly (`formal/bindings`,
|
|
191
|
+
* N2).
|
|
132
192
|
*/
|
|
133
193
|
function call(fn) {
|
|
134
194
|
try {
|
|
135
195
|
return fn();
|
|
136
196
|
}
|
|
137
197
|
catch (e) {
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
throw new DisarmError(msg);
|
|
198
|
+
throw translate(e);
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
/** The {@link DisarmError} a thrown value becomes; see {@link call}. */
|
|
202
|
+
function translate(e) {
|
|
203
|
+
if (e instanceof DisarmError) {
|
|
204
|
+
return e;
|
|
146
205
|
}
|
|
206
|
+
const msg = e instanceof Error ? e.message : String(e);
|
|
207
|
+
if (msg.startsWith(INVALID_ARG_TAG)) {
|
|
208
|
+
return new DisarmInvalidArgument(msg.slice(INVALID_ARG_TAG.length));
|
|
209
|
+
}
|
|
210
|
+
if (msg.startsWith(ERROR_TAG)) {
|
|
211
|
+
return new DisarmError(msg.slice(ERROR_TAG.length));
|
|
212
|
+
}
|
|
213
|
+
const code = e?.code;
|
|
214
|
+
if (typeof code === 'string' && NAPI_ARGUMENT_STATUSES.has(code)) {
|
|
215
|
+
return new DisarmInvalidArgument(msg);
|
|
216
|
+
}
|
|
217
|
+
return new DisarmError(msg);
|
|
218
|
+
}
|
|
219
|
+
/**
|
|
220
|
+
* Check a size or threshold option before it reaches napi, which converts any JS
|
|
221
|
+
* number to an integer: `NaN` and `0.9` became 0, `2 ** 64` saturated, so
|
|
222
|
+
* `stripZalgo('caf\u00e9', { maxMarks: NaN })` stripped the accent (`formal/bindings`, N1).
|
|
223
|
+
* A size is a non-negative safe integer; anything else throws
|
|
224
|
+
* {@link DisarmInvalidArgument}.
|
|
225
|
+
*/
|
|
226
|
+
function size(name, value) {
|
|
227
|
+
if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 0) {
|
|
228
|
+
throw new DisarmInvalidArgument(`${name} must be a non-negative integer (got ${String(value)})`);
|
|
229
|
+
}
|
|
230
|
+
return value;
|
|
231
|
+
}
|
|
232
|
+
/** {@link size} for an optional option: `undefined` or `null` stays absent. */
|
|
233
|
+
function optionalSize(name, value) {
|
|
234
|
+
return value == null ? undefined : size(name, value);
|
|
147
235
|
}
|
|
148
|
-
/** Romanize Unicode text to ASCII. */
|
|
236
|
+
/** Romanize Unicode text to ASCII. An unknown `lang` throws {@link DisarmInvalidArgument}. */
|
|
149
237
|
function transliterate(text, options = {}) {
|
|
150
238
|
const { scheme = 'default', lang } = options;
|
|
151
239
|
if (scheme === 'default' && lang == null) {
|
|
152
|
-
return native.transliterate(text);
|
|
240
|
+
return call(() => native.transliterate(text));
|
|
153
241
|
}
|
|
154
242
|
return call(() => native.transliterateOpts(text, scheme, lang ?? undefined));
|
|
155
243
|
}
|
|
@@ -157,7 +245,10 @@ function transliterate(text, options = {}) {
|
|
|
157
245
|
function reverseTransliterate(text, options) {
|
|
158
246
|
return call(() => native.reverseTransliterate(text, options.lang));
|
|
159
247
|
}
|
|
160
|
-
/**
|
|
248
|
+
/**
|
|
249
|
+
* Every character in `text` with no romanization, as `{ char, offset }` (byte offset), in
|
|
250
|
+
* order. An unknown `lang` throws {@link DisarmInvalidArgument}.
|
|
251
|
+
*/
|
|
161
252
|
function findUntranslatable(text, options = {}) {
|
|
162
253
|
const { scheme = 'default', lang } = options;
|
|
163
254
|
return call(() => native.findUntranslatable(text, scheme, lang ?? undefined));
|
|
@@ -167,6 +258,20 @@ function findUntranslatable(text, options = {}) {
|
|
|
167
258
|
function normalizeConfusables(text, options = {}) {
|
|
168
259
|
return call(() => native.normalizeConfusables(text, options.target ?? 'latin', options.digitPolicy ?? 'numeric'));
|
|
169
260
|
}
|
|
261
|
+
/**
|
|
262
|
+
* Whether `text` is already its own canonical form under `preset` (#730).
|
|
263
|
+
*
|
|
264
|
+
* The verification-path counterpart to the presets: text in, normalized text out is the
|
|
265
|
+
* generation path, and this is the question a caller asks about bytes that arrive already
|
|
266
|
+
* bound. `hasAnomalies` is not this predicate — 142,760 assigned code points are reported
|
|
267
|
+
* clean by the detector and are not their own canonical form.
|
|
268
|
+
*
|
|
269
|
+
* `preset` is any name in the preset registry or any profile, defaulting to
|
|
270
|
+
* `'canonicalize'`. An unknown one throws {@link DisarmInvalidArgument}.
|
|
271
|
+
*/
|
|
272
|
+
function isCanonical(text, options = {}) {
|
|
273
|
+
return call(() => native.isCanonical(text, options.preset ?? 'canonicalize'));
|
|
274
|
+
}
|
|
170
275
|
/** Whether `text` contains a character confusable with `target` (default `'latin'`). */
|
|
171
276
|
function isConfusable(text, options = {}) {
|
|
172
277
|
return call(() => native.isConfusable(text, options.target ?? 'latin'));
|
|
@@ -190,12 +295,15 @@ function unmappedConfusables(options = {}) {
|
|
|
190
295
|
function findUnmappedConfusables(text, options = {}) {
|
|
191
296
|
return call(() => native.findUnmappedConfusables(text, options.target ?? 'latin'));
|
|
192
297
|
}
|
|
193
|
-
/**
|
|
298
|
+
/**
|
|
299
|
+
* Generate a URL-safe slug. Mirrors the core's `SlugConfig` defaults. An unknown `lang`
|
|
300
|
+
* throws {@link DisarmInvalidArgument}.
|
|
301
|
+
*/
|
|
194
302
|
function slugify(text, options = {}) {
|
|
195
303
|
return call(() => native.slugify(text, {
|
|
196
304
|
separator: options.separator ?? '-',
|
|
197
305
|
lowercase: options.lowercase ?? true,
|
|
198
|
-
maxLength: options.maxLength ?? 0,
|
|
306
|
+
maxLength: optionalSize('maxLength', options.maxLength) ?? 0,
|
|
199
307
|
wordBoundary: options.wordBoundary ?? false,
|
|
200
308
|
saveOrder: options.saveOrder ?? false,
|
|
201
309
|
stopwords: options.stopwords ?? [],
|
|
@@ -210,11 +318,11 @@ function slugify(text, options = {}) {
|
|
|
210
318
|
// ── Canonicalization primitives ─────────────────────────────────────────────
|
|
211
319
|
/** Strip diacritics (`"café"` → `"cafe"`). */
|
|
212
320
|
function stripAccents(text) {
|
|
213
|
-
return native.stripAccents(text);
|
|
321
|
+
return call(() => native.stripAccents(text));
|
|
214
322
|
}
|
|
215
323
|
/** Full Unicode case fold — more aggressive than `String.toLowerCase()`. */
|
|
216
324
|
function foldCase(text) {
|
|
217
|
-
return native.foldCase(text);
|
|
325
|
+
return call(() => native.foldCase(text));
|
|
218
326
|
}
|
|
219
327
|
/**
|
|
220
328
|
* Whether `text` is a stable identity key under case folding — that is, whether
|
|
@@ -227,7 +335,7 @@ function foldCase(text) {
|
|
|
227
335
|
* folded into {@link hasAnomalies}.
|
|
228
336
|
*/
|
|
229
337
|
function isCaseFoldStable(text) {
|
|
230
|
-
return native.isCaseFoldStable(text);
|
|
338
|
+
return call(() => native.isCaseFoldStable(text));
|
|
231
339
|
}
|
|
232
340
|
/**
|
|
233
341
|
* Which of `values` are the same name under `key` (#620).
|
|
@@ -248,9 +356,30 @@ function isCaseFoldStable(text) {
|
|
|
248
356
|
function findKeyCollisions(values, key, options = {}) {
|
|
249
357
|
return call(() => native.findKeyCollisions(values, key, options.lang));
|
|
250
358
|
}
|
|
251
|
-
/**
|
|
359
|
+
/**
|
|
360
|
+
* Replace emoji with their plain names. `stripModifiers` drops skin-tone/variation marks.
|
|
361
|
+
* An emoji CLDR cannot name — a regional indicator or a Plane 14 tag character standing
|
|
362
|
+
* alone — becomes `'[?]'`, the sentinel {@link transliterate} writes, and the default in
|
|
363
|
+
* every binding.
|
|
364
|
+
*/
|
|
252
365
|
function demojize(text, options = {}) {
|
|
253
|
-
return native.demojize(text, options.stripModifiers ?? false);
|
|
366
|
+
return call(() => native.demojize(text, options.stripModifiers ?? false));
|
|
367
|
+
}
|
|
368
|
+
/** Replace every emoji with `replacement`, verbatim (#972).
|
|
369
|
+
*
|
|
370
|
+
* The counterpart to {@link demojize}, and a different question of a different table.
|
|
371
|
+
* `demojize` asks *what does CLDR call this?*, so its domain is the CLDR name table,
|
|
372
|
+
* which is wider than the emoji: `demojize('x™y')` is `'x trade mark y'`. This asks *is
|
|
373
|
+
* this an emoji by the UCD's properties?* — `Emoji_Presentation=Yes`, an `Emoji=Yes`
|
|
374
|
+
* base carrying `U+FE0F` (not an `Extended_Pictographic` one, so `★` + `U+FE0F` stays),
|
|
375
|
+
* and the ZWJ, modifier, keycap and flag sequences on those. Nothing
|
|
376
|
+
* else moves.
|
|
377
|
+
*
|
|
378
|
+
* `replacement` is inserted exactly as given, with no padding and no whitespace collapse:
|
|
379
|
+
* `''` closes an intra-word split (`aa🔥bb` → `aabb`) and `' '` keeps two words apart
|
|
380
|
+
* (`stop🛑now` → `stop now`), and no rule serves both. */
|
|
381
|
+
function replaceEmoji(text, replacement = '') {
|
|
382
|
+
return call(() => native.replaceEmoji(text, replacement));
|
|
254
383
|
}
|
|
255
384
|
// ── Normalization ───────────────────────────────────────────────────────────
|
|
256
385
|
/** Apply a Unicode normalization `form` (default `'NFC'`). */
|
|
@@ -273,43 +402,52 @@ function isNormalized(text, options = {}) {
|
|
|
273
402
|
* deleting them) means `a\rb` → `a b`, never `ab`.
|
|
274
403
|
*/
|
|
275
404
|
function collapseWhitespace(text) {
|
|
276
|
-
return native.collapseWhitespace(text);
|
|
405
|
+
return call(() => native.collapseWhitespace(text));
|
|
277
406
|
}
|
|
278
407
|
/** Remove C0/C1 control characters (except tab/newline). */
|
|
279
408
|
function stripControlChars(text) {
|
|
280
|
-
return native.stripControlChars(text);
|
|
409
|
+
return call(() => native.stripControlChars(text));
|
|
281
410
|
}
|
|
282
411
|
/** Remove zero-width characters (ZWSP/ZWNJ/ZWJ/word-joiner). */
|
|
283
412
|
function stripZeroWidthChars(text) {
|
|
284
|
-
return native.stripZeroWidthChars(text);
|
|
413
|
+
return call(() => native.stripZeroWidthChars(text));
|
|
285
414
|
}
|
|
286
415
|
/** Remove Unicode bidirectional control characters. */
|
|
287
416
|
function stripBidi(text) {
|
|
288
|
-
return native.stripBidi(text);
|
|
417
|
+
return call(() => native.stripBidi(text));
|
|
289
418
|
}
|
|
290
419
|
/** Strip the Unicode Tags block (U+E0000–U+E007F), preserving valid emoji flag sequences (#413). */
|
|
291
420
|
function stripTags(text) {
|
|
292
|
-
return native.stripTags(text);
|
|
421
|
+
return call(() => native.stripTags(text));
|
|
293
422
|
}
|
|
294
423
|
/** Strip every variation selector (VS1–VS256) (#413). */
|
|
295
424
|
function stripVariationSelectors(text) {
|
|
296
|
-
return native.stripVariationSelectors(text);
|
|
425
|
+
return call(() => native.stripVariationSelectors(text));
|
|
297
426
|
}
|
|
298
427
|
/** Strip every Unicode noncharacter (#413). */
|
|
299
428
|
function stripNoncharacters(text) {
|
|
300
|
-
return native.stripNoncharacters(text);
|
|
429
|
+
return call(() => native.stripNoncharacters(text));
|
|
301
430
|
}
|
|
302
431
|
/** Strip every Private Use Area code point (#413). */
|
|
303
432
|
function stripPua(text) {
|
|
304
|
-
return native.stripPua(text);
|
|
433
|
+
return call(() => native.stripPua(text));
|
|
305
434
|
}
|
|
306
|
-
/**
|
|
435
|
+
/**
|
|
436
|
+
* Cap the marks of each combining class on one base character at `maxMarks`, a
|
|
437
|
+
* non-negative integer. The default is the core's, 3 — equal to {@link isZalgo}'s
|
|
438
|
+
* threshold (#788), so this never strips from text `isZalgo` declines to flag. It is read
|
|
439
|
+
* from the core rather than restated here: this layer had kept the old cap of 2
|
|
440
|
+
* (`formal/bindings`, B1).
|
|
441
|
+
*/
|
|
307
442
|
function stripZalgo(text, options = {}) {
|
|
308
|
-
return call(() => native.stripZalgo(text, options.maxMarks
|
|
443
|
+
return call(() => native.stripZalgo(text, optionalSize('maxMarks', options.maxMarks)));
|
|
309
444
|
}
|
|
310
|
-
/**
|
|
445
|
+
/**
|
|
446
|
+
* Whether any base character carries more than `threshold` marks of one combining class.
|
|
447
|
+
* The default is the core's, 3.
|
|
448
|
+
*/
|
|
311
449
|
function isZalgo(text, options = {}) {
|
|
312
|
-
return call(() => native.isZalgo(text, options.threshold
|
|
450
|
+
return call(() => native.isZalgo(text, optionalSize('threshold', options.threshold)));
|
|
313
451
|
}
|
|
314
452
|
// ── Deobfuscation & security presets ────────────────────────────────────────
|
|
315
453
|
/**
|
|
@@ -317,8 +455,8 @@ function isZalgo(text, options = {}) {
|
|
|
317
455
|
* the half of the pair that lets a caller reject input instead of comparing a value the
|
|
318
456
|
* sender never wrote.
|
|
319
457
|
*/
|
|
320
|
-
function canonicalizeStrict(text) {
|
|
321
|
-
return call(() => native.canonicalizeStrict(text));
|
|
458
|
+
function canonicalizeStrict(text, options = {}) {
|
|
459
|
+
return call(() => native.canonicalizeStrict(text, options.digitPolicy ?? 'numeric'));
|
|
322
460
|
}
|
|
323
461
|
/**
|
|
324
462
|
* Strip the non-interchange and invisible classes while KEEPING the script.
|
|
@@ -330,23 +468,24 @@ function canonicalizeStrict(text) {
|
|
|
330
468
|
* it collapses TAB/LF to a space, which the primitives leave alone. Infallible.
|
|
331
469
|
*/
|
|
332
470
|
function stripFormat(text) {
|
|
333
|
-
return native.stripFormat(text);
|
|
471
|
+
return call(() => native.stripFormat(text));
|
|
334
472
|
}
|
|
335
473
|
/** Remove obfuscation (zero-width, bidi, combining-mark abuse, homoglyphs) while keeping legible content. */
|
|
336
|
-
function stripObfuscation(text) {
|
|
337
|
-
return call(() => native.stripObfuscation(text));
|
|
474
|
+
function stripObfuscation(text, options = {}) {
|
|
475
|
+
return call(() => native.stripObfuscation(text, options.digitPolicy ?? 'numeric'));
|
|
338
476
|
}
|
|
339
477
|
/**
|
|
340
|
-
* Canonicalize text for security-sensitive comparison:
|
|
341
|
-
* → strip invisible classes (#413) → strip control → strip
|
|
342
|
-
*
|
|
343
|
-
* (confusables
|
|
478
|
+
* Canonicalize text for security-sensitive comparison: resolve deletions → NFKC →
|
|
479
|
+
* strip bidi/format → strip invisible classes (#413) → strip control → strip
|
|
480
|
+
* zero-width → collapse whitespace → drop repeated marks → cap combining marks
|
|
481
|
+
* (anti-zalgo) → NFC → confusables and NFC to a fixed point → drop repeated marks
|
|
482
|
+
* → cap combining marks again (the fold is iterated with NFC for idempotency).
|
|
344
483
|
*
|
|
345
484
|
* The name describes the mechanism (Unicode canonicalization for matching), not
|
|
346
485
|
* a safety guarantee — this is not an output sanitizer; encode at the sink.
|
|
347
486
|
*/
|
|
348
|
-
function canonicalize(text) {
|
|
349
|
-
return call(() => native.canonicalize(text));
|
|
487
|
+
function canonicalize(text, options = {}) {
|
|
488
|
+
return call(() => native.canonicalize(text, options.digitPolicy ?? 'numeric'));
|
|
350
489
|
}
|
|
351
490
|
/**
|
|
352
491
|
* @deprecated Renamed to {@link canonicalize} in 0.11 (the `*Clean` name
|
|
@@ -367,9 +506,15 @@ function securityClean(text) {
|
|
|
367
506
|
function getPipeline(profile) {
|
|
368
507
|
return call(() => native.getPipeline(profile));
|
|
369
508
|
}
|
|
370
|
-
/**
|
|
509
|
+
/**
|
|
510
|
+
* Turn arbitrary text into a filesystem-safe filename.
|
|
511
|
+
*
|
|
512
|
+
* `separator` must be printable, non-space ASCII with no character illegal on
|
|
513
|
+
* `platform` and no path separator (`/`, `\`); anything else throws
|
|
514
|
+
* {@link DisarmInvalidArgument}. `''` is allowed.
|
|
515
|
+
*/
|
|
371
516
|
function sanitizeFilename(text, options = {}) {
|
|
372
|
-
return call(() => native.sanitizeFilename(text, options.separator ?? '_', options.maxLength ?? 255, options.platform ?? 'universal', options.lang ?? undefined, options.preserveExtension ?? true));
|
|
517
|
+
return call(() => native.sanitizeFilename(text, options.separator ?? '_', optionalSize('maxLength', options.maxLength) ?? 255, options.platform ?? 'universal', options.lang ?? undefined, options.preserveExtension ?? true));
|
|
373
518
|
}
|
|
374
519
|
// ── Key-derivation presets ──────────────────────────────────────────────────
|
|
375
520
|
/**
|
|
@@ -377,14 +522,14 @@ function sanitizeFilename(text, options = {}) {
|
|
|
377
522
|
* without confusable folding). `lang` selects the transliteration table.
|
|
378
523
|
*/
|
|
379
524
|
function searchKey(text, options = {}) {
|
|
380
|
-
return call(() => native.searchKey(text, options.lang ?? undefined));
|
|
525
|
+
return call(() => native.searchKey(text, options.lang ?? undefined, options.digitPolicy ?? 'numeric'));
|
|
381
526
|
}
|
|
382
527
|
/**
|
|
383
528
|
* Collation sort key — like {@link searchKey} but preserves base accented
|
|
384
529
|
* characters for correct ordering. `lang` selects the transliteration table.
|
|
385
530
|
*/
|
|
386
531
|
function sortKey(text, options = {}) {
|
|
387
|
-
return call(() => native.sortKey(text, options.lang ?? undefined));
|
|
532
|
+
return call(() => native.sortKey(text, options.lang ?? undefined, options.digitPolicy ?? 'numeric'));
|
|
388
533
|
}
|
|
389
534
|
/**
|
|
390
535
|
* Library catalog deduplication key — like {@link searchKey} plus confusable
|
|
@@ -392,11 +537,39 @@ function sortKey(text, options = {}) {
|
|
|
392
537
|
* `false`) picks the ISO 9:1995 Cyrillic scheme.
|
|
393
538
|
*/
|
|
394
539
|
function catalogKey(text, options = {}) {
|
|
395
|
-
return call(() => native.catalogKey(text, options.lang ?? undefined, options.strictIso9 ?? false));
|
|
540
|
+
return call(() => native.catalogKey(text, options.lang ?? undefined, options.strictIso9 ?? false, options.digitPolicy ?? 'numeric'));
|
|
541
|
+
}
|
|
542
|
+
/**
|
|
543
|
+
* The TR39 identifier skeleton plus the two prototype classes disarm's table keeps
|
|
544
|
+
* apart (#650). A spoof key: its only job is to make confusable identifiers collide,
|
|
545
|
+
* and its output is never for display. `digitPolicy` `'numeric'` (default) applies the
|
|
546
|
+
* letter half only; `'tr39'` adds `1 ≡ l` and `0 ≡ O`; `'preserve'` keeps a non-Latin
|
|
547
|
+
* numeral in its script.
|
|
548
|
+
*/
|
|
549
|
+
function skeletonKey(text, options = {}) {
|
|
550
|
+
return call(() => native.skeletonKey(text, options.digitPolicy ?? 'numeric'));
|
|
551
|
+
}
|
|
552
|
+
/**
|
|
553
|
+
* Levenshtein edit distance between `a` and `b`, in characters (#894) — the one class of
|
|
554
|
+
* registry spoofing the confusable tables deliberately do not model (`paypa1`, `adm1n`).
|
|
555
|
+
* Canonicalize both sides first when composed and decomposed spellings should compare
|
|
556
|
+
* equal.
|
|
557
|
+
*/
|
|
558
|
+
function editDistance(a, b) {
|
|
559
|
+
return call(() => native.editDistance(a, b));
|
|
560
|
+
}
|
|
561
|
+
/**
|
|
562
|
+
* The candidate closest to `value`, with its distance, or `null` beyond `maxDistance`
|
|
563
|
+
* (default 1). Reports; it does not decide. An exact match is reported with distance 0,
|
|
564
|
+
* and ties go to the first candidate at the lowest distance (#894).
|
|
565
|
+
*/
|
|
566
|
+
function nearestMatch(value, candidates, options = {}) {
|
|
567
|
+
return (call(() => native.nearestMatch(value, candidates, optionalSize('maxDistance', options.maxDistance) ?? 1)) ?? null);
|
|
396
568
|
}
|
|
397
569
|
/**
|
|
398
|
-
* ML/NLP normalization: NFKC → emoji→text → transliterate →
|
|
399
|
-
* [case fold] → strip control → strip zero-width →
|
|
570
|
+
* ML/NLP normalization: resolve deletions → NFKC → emoji→text → transliterate →
|
|
571
|
+
* strip accents → emoji→text → [case fold] → strip control → strip zero-width →
|
|
572
|
+
* collapse whitespace → NFC.
|
|
400
573
|
*
|
|
401
574
|
* Note this folds no confusables — it is not a homoglyph defence at any setting. Put
|
|
402
575
|
* {@link normalizeConfusables} in front of it when a model needs both.
|
|
@@ -407,28 +580,28 @@ function mlNormalize(text, options = {}) {
|
|
|
407
580
|
// ── Grapheme clusters ───────────────────────────────────────────────────────
|
|
408
581
|
/** Number of grapheme clusters (user-perceived characters). */
|
|
409
582
|
function graphemeLen(text) {
|
|
410
|
-
return native.graphemeLen(text);
|
|
583
|
+
return call(() => native.graphemeLen(text));
|
|
411
584
|
}
|
|
412
585
|
/** Split `text` into grapheme-cluster strings. */
|
|
413
586
|
function graphemeSplit(text) {
|
|
414
|
-
return native.graphemeSplit(text);
|
|
587
|
+
return call(() => native.graphemeSplit(text));
|
|
415
588
|
}
|
|
416
589
|
/** Truncate to at most `maxGraphemes` clusters, never cutting through one. */
|
|
417
590
|
function graphemeTruncate(text, maxGraphemes) {
|
|
418
|
-
return call(() => native.graphemeTruncate(text, maxGraphemes));
|
|
591
|
+
return call(() => native.graphemeTruncate(text, size('maxGraphemes', maxGraphemes)));
|
|
419
592
|
}
|
|
420
593
|
/** Display width (terminal columns) of a single grapheme `cluster` by East Asian Width. */
|
|
421
594
|
function graphemeWidth(cluster, options = {}) {
|
|
422
|
-
return native.graphemeWidth(cluster, options.ambiguousWide ?? false);
|
|
595
|
+
return call(() => native.graphemeWidth(cluster, options.ambiguousWide ?? false));
|
|
423
596
|
}
|
|
424
597
|
/** Total display width (terminal columns) of `text`. */
|
|
425
598
|
function terminalWidth(text, options = {}) {
|
|
426
|
-
return native.terminalWidth(text, options.ambiguousWide ?? false);
|
|
599
|
+
return call(() => native.terminalWidth(text, options.ambiguousWide ?? false));
|
|
427
600
|
}
|
|
428
601
|
// ── Hostname / script analysis ──────────────────────────────────────────────
|
|
429
602
|
/** Whether the hostname looks like a mixed-script / confusable IDN spoof (a `false` is not a safety guarantee). */
|
|
430
603
|
function isSuspiciousHostname(host) {
|
|
431
|
-
return native.isSuspiciousHostname(host);
|
|
604
|
+
return call(() => native.isSuspiciousHostname(host));
|
|
432
605
|
}
|
|
433
606
|
/**
|
|
434
607
|
* Analyze a hostname for Unicode homoglyph spoofing, returning the full
|
|
@@ -440,11 +613,11 @@ function analyzeHostname(host, options = {}) {
|
|
|
440
613
|
}
|
|
441
614
|
/** The Unicode scripts present, in first-appearance order (Common/Inherited excluded). */
|
|
442
615
|
function detectScripts(text) {
|
|
443
|
-
return native.detectScripts(text);
|
|
616
|
+
return call(() => native.detectScripts(text));
|
|
444
617
|
}
|
|
445
618
|
/** Whether `text` mixes characters from more than one script. */
|
|
446
619
|
function isMixedScript(text) {
|
|
447
|
-
return native.isMixedScript(text);
|
|
620
|
+
return call(() => native.isMixedScript(text));
|
|
448
621
|
}
|
|
449
622
|
/**
|
|
450
623
|
* All twelve UAX #9 explicit formatting characters, uncontexted.
|
|
@@ -455,7 +628,7 @@ function isMixedScript(text) {
|
|
|
455
628
|
* right-to-left text.
|
|
456
629
|
*/
|
|
457
630
|
function hasBidiControl(text) {
|
|
458
|
-
return native.hasBidiControl(text);
|
|
631
|
+
return call(() => native.hasBidiControl(text));
|
|
459
632
|
}
|
|
460
633
|
/**
|
|
461
634
|
* Whether `text` mixes strong left-to-right and strong right-to-left characters
|
|
@@ -464,11 +637,11 @@ function hasBidiControl(text) {
|
|
|
464
637
|
* guarantee.
|
|
465
638
|
*/
|
|
466
639
|
function hasBidiConflict(text) {
|
|
467
|
-
return native.hasBidiConflict(text);
|
|
640
|
+
return call(() => native.hasBidiConflict(text));
|
|
468
641
|
}
|
|
469
642
|
/** Explain how `lang: 'auto'` detection resolves `text`. */
|
|
470
643
|
function inspectAutoLang(text) {
|
|
471
|
-
return native.inspectAutoLang(text);
|
|
644
|
+
return call(() => native.inspectAutoLang(text));
|
|
472
645
|
}
|
|
473
646
|
// ── Metadata introspection (#404) ───────────────────────────────────────────
|
|
474
647
|
/**
|
|
@@ -487,13 +660,30 @@ function langInfo(code) {
|
|
|
487
660
|
function scriptInfo(name) {
|
|
488
661
|
return call(() => native.scriptInfo(name));
|
|
489
662
|
}
|
|
663
|
+
/** TR39 sources whose prototype is in `script`, and how many of those disarm's bundled
|
|
664
|
+
* tables fold (#963).
|
|
665
|
+
*
|
|
666
|
+
* The denominator {@link unmappedConfusables} does not have. That function measures one
|
|
667
|
+
* bundled table against the whole 6,565-source population, which is the right question
|
|
668
|
+
* for a target disarm ships and a misleading one for a script it does not: Greek reports
|
|
669
|
+
* almost the entire population unmapped, and the number means only "there is no Greek
|
|
670
|
+
* table".
|
|
671
|
+
*
|
|
672
|
+
* `folded` counts sources any bundled table reaches, not sources folded *toward* this
|
|
673
|
+
* script — Greek is 71 of 159 because the Latin table folds Greek letters that look
|
|
674
|
+
* Latin. The grouping uses the UCD's script property, so `"Yi"` and 18 other scripts
|
|
675
|
+
* disarm's own enum does not name are addressable here. A script disarm knows that TR39
|
|
676
|
+
* never uses as a prototype returns 0 of 0. */
|
|
677
|
+
function confusableCoverage(script) {
|
|
678
|
+
return call(() => native.confusableCoverage(script));
|
|
679
|
+
}
|
|
490
680
|
/**
|
|
491
681
|
* The UCD release disarm's normalizer implements. Not a library-wide Unicode version —
|
|
492
682
|
* the bundled tables track different releases. This is the one integrators ask about,
|
|
493
683
|
* because it decides whether disarm's normalization agrees with the host platform's.
|
|
494
684
|
*/
|
|
495
685
|
function unicodeVersion() {
|
|
496
|
-
return native.unicodeVersion();
|
|
686
|
+
return call(() => native.unicodeVersion());
|
|
497
687
|
}
|
|
498
688
|
/**
|
|
499
689
|
* Whether a key stored under an earlier release still compares equal. A monotonic
|
|
@@ -501,7 +691,7 @@ function unicodeVersion() {
|
|
|
501
691
|
* for the same input. Meaningless in isolation, by design.
|
|
502
692
|
*/
|
|
503
693
|
function keySchemaVersion() {
|
|
504
|
-
return native.keySchemaVersion();
|
|
694
|
+
return call(() => native.keySchemaVersion());
|
|
505
695
|
}
|
|
506
696
|
/**
|
|
507
697
|
* The Unicode `confusables.txt` release the bundled confusable tables were folded
|
|
@@ -512,15 +702,15 @@ function keySchemaVersion() {
|
|
|
512
702
|
* confusables fold stale?" without inferring it from behaviour.
|
|
513
703
|
*/
|
|
514
704
|
function confusablesVersion() {
|
|
515
|
-
return native.confusablesVersion();
|
|
705
|
+
return call(() => native.confusablesVersion());
|
|
516
706
|
}
|
|
517
707
|
/** Every Unicode script name known to the transliteration tables. */
|
|
518
708
|
function listScripts() {
|
|
519
|
-
return native.listScripts();
|
|
709
|
+
return call(() => native.listScripts());
|
|
520
710
|
}
|
|
521
711
|
/** Every language code that has a context-aware transliteration profile. */
|
|
522
712
|
function listContextLangs() {
|
|
523
|
-
return native.listContextLangs();
|
|
713
|
+
return call(() => native.listContextLangs());
|
|
524
714
|
}
|
|
525
715
|
// ── Anomaly detection ───────────────────────────────────────────────────────
|
|
526
716
|
/**
|
|
@@ -532,10 +722,10 @@ function listContextLangs() {
|
|
|
532
722
|
* internal set on every call — used only by the leet and segmentation branches.
|
|
533
723
|
*/
|
|
534
724
|
function hasAnomalies(text, lexicon = []) {
|
|
535
|
-
if (lexicon instanceof
|
|
536
|
-
return native.hasAnomalies(text, lexicon);
|
|
725
|
+
if (lexicon instanceof Lexicon) {
|
|
726
|
+
return call(() => native.hasAnomalies(text, lexicon));
|
|
537
727
|
}
|
|
538
|
-
return native.hasAnomalies(text, Array.isArray(lexicon) ? lexicon : [...lexicon]);
|
|
728
|
+
return call(() => native.hasAnomalies(text, Array.isArray(lexicon) ? lexicon : [...lexicon]));
|
|
539
729
|
}
|
|
540
730
|
/**
|
|
541
731
|
* Full anomaly analysis: an `AnomalyReport` with `anomalous`, `kinds` (in
|
|
@@ -544,9 +734,8 @@ function hasAnomalies(text, lexicon = []) {
|
|
|
544
734
|
* `lexicon` may be a `Set`/array of words or a prebuilt {@link Lexicon} handle.
|
|
545
735
|
*/
|
|
546
736
|
function inspectAnomalies(text, lexicon = []) {
|
|
547
|
-
if (lexicon instanceof
|
|
548
|
-
return native.inspectAnomalies(text, lexicon);
|
|
737
|
+
if (lexicon instanceof Lexicon) {
|
|
738
|
+
return call(() => native.inspectAnomalies(text, lexicon));
|
|
549
739
|
}
|
|
550
|
-
|
|
551
|
-
return native.inspectAnomalies(text, words);
|
|
740
|
+
return call(() => native.inspectAnomalies(text, Array.isArray(lexicon) ? lexicon : [...lexicon]));
|
|
552
741
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "disarm",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.17.0",
|
|
4
4
|
"description": "Unicode confusable/text-security building blocks, powered by Rust",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"types": "index.d.ts",
|
|
@@ -22,7 +22,7 @@
|
|
|
22
22
|
"napi-rs"
|
|
23
23
|
],
|
|
24
24
|
"engines": {
|
|
25
|
-
"node": ">=
|
|
25
|
+
"node": ">= 22"
|
|
26
26
|
},
|
|
27
27
|
"napi": {
|
|
28
28
|
"binaryName": "disarm",
|
|
@@ -50,7 +50,7 @@
|
|
|
50
50
|
"devDependencies": {
|
|
51
51
|
"@biomejs/biome": "^2.5.0",
|
|
52
52
|
"@napi-rs/cli": "^3.7.2",
|
|
53
|
-
"typescript": "^
|
|
54
|
-
"vitest": "^
|
|
53
|
+
"typescript": "^7.0.2",
|
|
54
|
+
"vitest": "^5.0.0"
|
|
55
55
|
}
|
|
56
56
|
}
|