disarm 0.16.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/binding.d.ts +34 -8
- package/binding.js +294 -103
- package/disarm.darwin-arm64.node +0 -0
- package/disarm.darwin-x64.node +0 -0
- package/disarm.linux-arm64-gnu.node +0 -0
- package/disarm.linux-x64-gnu.node +2 -2
- package/disarm.win32-x64-msvc.node +0 -0
- package/index.d.ts +57 -20
- package/index.js +180 -70
- package/package.json +4 -4
package/index.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { Lexicon, Pipeline } from './binding';
|
|
1
|
+
import { Lexicon as NativeLexicon, Pipeline } from './binding';
|
|
2
2
|
import type { Untranslatable, UnmappedConfusable, AutoLangInspection, KeyCollision, LangMeta, ScriptMeta, ConfusableCoverage, Finding as NativeFinding, AnomalyReport as NativeAnomalyReport, HostnameAnalysis as NativeHostnameAnalysis } from './binding';
|
|
3
3
|
export type { Untranslatable, UnmappedConfusable, AutoLangInspection, KeyCollision, LangMeta, ScriptMeta, ConfusableCoverage, };
|
|
4
4
|
/** Findings from {@link analyzeHostname}. `suspicious` is a maximally
|
|
@@ -9,8 +9,13 @@ export type HostnameAnalysis = NativeHostnameAnalysis;
|
|
|
9
9
|
* `inspectAnomalies` rebuild an internal set from the caller's word array on
|
|
10
10
|
* every call; constructing a `Lexicon` once (`new Lexicon([...])`) and passing
|
|
11
11
|
* it instead builds that set a single time and reuses it across calls.
|
|
12
|
+
*
|
|
13
|
+
* A thin subclass of the native handle, so a bad argument to the constructor throws a
|
|
14
|
+
* {@link DisarmError} like every other entry point (`formal/bindings`, N2).
|
|
12
15
|
*/
|
|
13
|
-
export
|
|
16
|
+
export declare class Lexicon extends NativeLexicon {
|
|
17
|
+
constructor(words: string[]);
|
|
18
|
+
}
|
|
14
19
|
/**
|
|
15
20
|
* A reusable, opaque named-policy-profile pipeline handle (#404). Build it once
|
|
16
21
|
* with {@link getPipeline} for a named profile, then apply it to any number of
|
|
@@ -47,14 +52,17 @@ export declare class DisarmInvalidArgument extends DisarmError {
|
|
|
47
52
|
}
|
|
48
53
|
/** Transliteration scheme: the general-purpose default, ISO 9-style ASCII, or GOST R 7.0.34. */
|
|
49
54
|
export type Scheme = 'default' | 'strict_iso9' | 'gost7034';
|
|
50
|
-
/**
|
|
51
|
-
|
|
55
|
+
/**
|
|
56
|
+
* Confusable-folding target script. `'arabic'` and `'hebrew'` fold toward those scripts
|
|
57
|
+
* (#792); the runtime accepted them before this type admitted them.
|
|
58
|
+
*/
|
|
59
|
+
export type TargetScript = 'latin' | 'cyrillic' | 'arabic' | 'hebrew';
|
|
52
60
|
/**
|
|
53
61
|
* How the fold treats non-Latin digits.
|
|
54
62
|
*
|
|
55
63
|
* `'numeric'` (default) sends them to the ASCII digit — `०` becomes `0` — which is right
|
|
56
64
|
* for prose. `'tr39'` uses upstream's targets, which send most of them to a Latin letter
|
|
57
|
-
* (`०` → `o`), and that is what an identifier *skeleton* wants. The two differ on
|
|
65
|
+
* (`०` → `o`), and that is what an identifier *skeleton* wants. The two differ on 47 rows.
|
|
58
66
|
*
|
|
59
67
|
* Three of those rows do not land on a letter: `٠` and `۰` fold to `.`, and `𑣣` folds to
|
|
60
68
|
* the two characters `rn`. A skeleton feeding a label- or path-shaped key has to allow
|
|
@@ -80,13 +88,16 @@ export interface TransliterateOptions {
|
|
|
80
88
|
/** A language profile applied on top of the scheme (e.g. `'uk'`, `'de'`, or `'auto'`). */
|
|
81
89
|
lang?: string;
|
|
82
90
|
}
|
|
83
|
-
/** Romanize Unicode text to ASCII. */
|
|
91
|
+
/** Romanize Unicode text to ASCII. An unknown `lang` throws {@link DisarmInvalidArgument}. */
|
|
84
92
|
export declare function transliterate(text: string, options?: TransliterateOptions): string;
|
|
85
93
|
/** Reverse-transliterate Latin back to a native script (`'el'`, `'ru'`, or `'uk'`). */
|
|
86
94
|
export declare function reverseTransliterate(text: string, options: {
|
|
87
95
|
lang: ReverseLang;
|
|
88
96
|
}): string;
|
|
89
|
-
/**
|
|
97
|
+
/**
|
|
98
|
+
* Every character in `text` with no romanization, as `{ char, offset }` (byte offset), in
|
|
99
|
+
* order. An unknown `lang` throws {@link DisarmInvalidArgument}.
|
|
100
|
+
*/
|
|
90
101
|
export declare function findUntranslatable(text: string, options?: TransliterateOptions): Untranslatable[];
|
|
91
102
|
/** Fold cross-script confusables toward `target` (default `'latin'`). */
|
|
92
103
|
export declare function normalizeConfusables(text: string, options?: {
|
|
@@ -144,7 +155,10 @@ export interface SlugifyOptions {
|
|
|
144
155
|
hexadecimal?: boolean;
|
|
145
156
|
safeChars?: string;
|
|
146
157
|
}
|
|
147
|
-
/**
|
|
158
|
+
/**
|
|
159
|
+
* Generate a URL-safe slug. Mirrors the core's `SlugConfig` defaults. An unknown `lang`
|
|
160
|
+
* throws {@link DisarmInvalidArgument}.
|
|
161
|
+
*/
|
|
148
162
|
export declare function slugify(text: string, options?: SlugifyOptions): string;
|
|
149
163
|
/** Strip diacritics (`"café"` → `"cafe"`). */
|
|
150
164
|
export declare function stripAccents(text: string): string;
|
|
@@ -183,7 +197,12 @@ export type CollisionKey = 'fold_case' | 'search_key' | 'catalog_key' | 'canonic
|
|
|
183
197
|
export declare function findKeyCollisions(values: string[], key: CollisionKey, options?: {
|
|
184
198
|
lang?: string;
|
|
185
199
|
}): KeyCollision[];
|
|
186
|
-
/**
|
|
200
|
+
/**
|
|
201
|
+
* Replace emoji with their plain names. `stripModifiers` drops skin-tone/variation marks.
|
|
202
|
+
* An emoji CLDR cannot name — a regional indicator or a Plane 14 tag character standing
|
|
203
|
+
* alone — becomes `'[?]'`, the sentinel {@link transliterate} writes, and the default in
|
|
204
|
+
* every binding.
|
|
205
|
+
*/
|
|
187
206
|
export declare function demojize(text: string, options?: {
|
|
188
207
|
stripModifiers?: boolean;
|
|
189
208
|
}): string;
|
|
@@ -192,8 +211,9 @@ export declare function demojize(text: string, options?: {
|
|
|
192
211
|
* The counterpart to {@link demojize}, and a different question of a different table.
|
|
193
212
|
* `demojize` asks *what does CLDR call this?*, so its domain is the CLDR name table,
|
|
194
213
|
* which is wider than the emoji: `demojize('x™y')` is `'x trade mark y'`. This asks *is
|
|
195
|
-
* this an emoji by the UCD's properties?* — `Emoji_Presentation=Yes`, an `Emoji=Yes`
|
|
196
|
-
* carrying `U+FE0F
|
|
214
|
+
* this an emoji by the UCD's properties?* — `Emoji_Presentation=Yes`, an `Emoji=Yes`
|
|
215
|
+
* base carrying `U+FE0F` (not an `Extended_Pictographic` one, so `★` + `U+FE0F` stays),
|
|
216
|
+
* and the ZWJ, modifier, keycap and flag sequences on those. Nothing
|
|
197
217
|
* else moves.
|
|
198
218
|
*
|
|
199
219
|
* `replacement` is inserted exactly as given, with no padding and no whitespace collapse:
|
|
@@ -233,11 +253,20 @@ export declare function stripVariationSelectors(text: string): string;
|
|
|
233
253
|
export declare function stripNoncharacters(text: string): string;
|
|
234
254
|
/** Strip every Private Use Area code point (#413). */
|
|
235
255
|
export declare function stripPua(text: string): string;
|
|
236
|
-
/**
|
|
256
|
+
/**
|
|
257
|
+
* Cap the marks of each combining class on one base character at `maxMarks`, a
|
|
258
|
+
* non-negative integer. The default is the core's, 3 — equal to {@link isZalgo}'s
|
|
259
|
+
* threshold (#788), so this never strips from text `isZalgo` declines to flag. It is read
|
|
260
|
+
* from the core rather than restated here: this layer had kept the old cap of 2
|
|
261
|
+
* (`formal/bindings`, B1).
|
|
262
|
+
*/
|
|
237
263
|
export declare function stripZalgo(text: string, options?: {
|
|
238
264
|
maxMarks?: number;
|
|
239
265
|
}): string;
|
|
240
|
-
/**
|
|
266
|
+
/**
|
|
267
|
+
* Whether any base character carries more than `threshold` marks of one combining class.
|
|
268
|
+
* The default is the core's, 3.
|
|
269
|
+
*/
|
|
241
270
|
export declare function isZalgo(text: string, options?: {
|
|
242
271
|
threshold?: number;
|
|
243
272
|
}): boolean;
|
|
@@ -264,10 +293,11 @@ export declare function stripObfuscation(text: string, options?: {
|
|
|
264
293
|
digitPolicy?: DigitPolicy;
|
|
265
294
|
}): string;
|
|
266
295
|
/**
|
|
267
|
-
* Canonicalize text for security-sensitive comparison:
|
|
268
|
-
* → strip invisible classes (#413) → strip control → strip
|
|
269
|
-
*
|
|
270
|
-
* (confusables
|
|
296
|
+
* Canonicalize text for security-sensitive comparison: resolve deletions → NFKC →
|
|
297
|
+
* strip bidi/format → strip invisible classes (#413) → strip control → strip
|
|
298
|
+
* zero-width → collapse whitespace → drop repeated marks → cap combining marks
|
|
299
|
+
* (anti-zalgo) → NFC → confusables and NFC to a fixed point → drop repeated marks
|
|
300
|
+
* → cap combining marks again (the fold is iterated with NFC for idempotency).
|
|
271
301
|
*
|
|
272
302
|
* The name describes the mechanism (Unicode canonicalization for matching), not
|
|
273
303
|
* a safety guarantee — this is not an output sanitizer; encode at the sink.
|
|
@@ -297,7 +327,13 @@ export interface SanitizeFilenameOptions {
|
|
|
297
327
|
lang?: string;
|
|
298
328
|
preserveExtension?: boolean;
|
|
299
329
|
}
|
|
300
|
-
/**
|
|
330
|
+
/**
|
|
331
|
+
* Turn arbitrary text into a filesystem-safe filename.
|
|
332
|
+
*
|
|
333
|
+
* `separator` must be printable, non-space ASCII with no character illegal on
|
|
334
|
+
* `platform` and no path separator (`/`, `\`); anything else throws
|
|
335
|
+
* {@link DisarmInvalidArgument}. `''` is allowed.
|
|
336
|
+
*/
|
|
301
337
|
export declare function sanitizeFilename(text: string, options?: SanitizeFilenameOptions): string;
|
|
302
338
|
/**
|
|
303
339
|
* Case/accent/script-insensitive search lookup key (like {@link catalogKey}
|
|
@@ -371,8 +407,9 @@ export interface MlNormalizeOptions {
|
|
|
371
407
|
foldCase?: boolean;
|
|
372
408
|
}
|
|
373
409
|
/**
|
|
374
|
-
* ML/NLP normalization: NFKC → emoji→text → transliterate →
|
|
375
|
-
* [case fold] → strip control → strip zero-width →
|
|
410
|
+
* ML/NLP normalization: resolve deletions → NFKC → emoji→text → transliterate →
|
|
411
|
+
* strip accents → emoji→text → [case fold] → strip control → strip zero-width →
|
|
412
|
+
* collapse whitespace → NFC.
|
|
376
413
|
*
|
|
377
414
|
* Note this folds no confusables — it is not a homoglyph defence at any setting. Put
|
|
378
415
|
* {@link normalizeConfusables} in front of it when a model needs both.
|
package/index.js
CHANGED
|
@@ -108,8 +108,40 @@ exports.inspectAnomalies = inspectAnomalies;
|
|
|
108
108
|
*/
|
|
109
109
|
const native = __importStar(require("./binding"));
|
|
110
110
|
const binding_1 = require("./binding");
|
|
111
|
-
Object.defineProperty(exports, "Lexicon", { enumerable: true, get: function () { return binding_1.Lexicon; } });
|
|
112
111
|
Object.defineProperty(exports, "Pipeline", { enumerable: true, get: function () { return binding_1.Pipeline; } });
|
|
112
|
+
/**
|
|
113
|
+
* A reusable, opaque lexicon handle (HAI-SDLC 6.1). `hasAnomalies` /
|
|
114
|
+
* `inspectAnomalies` rebuild an internal set from the caller's word array on
|
|
115
|
+
* every call; constructing a `Lexicon` once (`new Lexicon([...])`) and passing
|
|
116
|
+
* it instead builds that set a single time and reuses it across calls.
|
|
117
|
+
*
|
|
118
|
+
* A thin subclass of the native handle, so a bad argument to the constructor throws a
|
|
119
|
+
* {@link DisarmError} like every other entry point (`formal/bindings`, N2).
|
|
120
|
+
*/
|
|
121
|
+
class Lexicon extends binding_1.Lexicon {
|
|
122
|
+
constructor(words) {
|
|
123
|
+
try {
|
|
124
|
+
super(words);
|
|
125
|
+
}
|
|
126
|
+
catch (e) {
|
|
127
|
+
throw translate(e);
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
exports.Lexicon = Lexicon;
|
|
132
|
+
// The handle's methods are native; route them through `call()` as every function is, so a
|
|
133
|
+
// wrong argument or a refused digit policy throws a `DisarmError` rather than a bare
|
|
134
|
+
// `Error` whose message still carries the shim's tag (`formal/bindings`, N2). Patched on
|
|
135
|
+
// the prototype, so `getPipeline`'s native instances and `instanceof Pipeline` are
|
|
136
|
+
// unchanged.
|
|
137
|
+
const nativeProcess = binding_1.Pipeline.prototype.process;
|
|
138
|
+
const nativeWithDigitPolicy = binding_1.Pipeline.prototype.withDigitPolicy;
|
|
139
|
+
binding_1.Pipeline.prototype.process = function process(text) {
|
|
140
|
+
return call(() => nativeProcess.call(this, text));
|
|
141
|
+
};
|
|
142
|
+
binding_1.Pipeline.prototype.withDigitPolicy = function withDigitPolicy(digitPolicy) {
|
|
143
|
+
return call(() => nativeWithDigitPolicy.call(this, digitPolicy));
|
|
144
|
+
};
|
|
113
145
|
// ── Errors ──────────────────────────────────────────────────────────────────
|
|
114
146
|
/** Base class for every error disarm raises, so callers can `catch (e) { if (e instanceof DisarmError) … }`. */
|
|
115
147
|
class DisarmError extends Error {
|
|
@@ -129,33 +161,83 @@ class DisarmInvalidArgument extends DisarmError {
|
|
|
129
161
|
exports.DisarmInvalidArgument = DisarmInvalidArgument;
|
|
130
162
|
const INVALID_ARG_TAG = 'DisarmInvalidArgument: ';
|
|
131
163
|
const ERROR_TAG = 'DisarmError: ';
|
|
164
|
+
/**
|
|
165
|
+
* The napi statuses an argument conversion fails with: a value of the wrong JS type
|
|
166
|
+
* (`transliterate(123)`, `stripAccents(undefined)`). They are invalid arguments, as a
|
|
167
|
+
* `TypeError` is in the Ruby binding.
|
|
168
|
+
*/
|
|
169
|
+
const NAPI_ARGUMENT_STATUSES = new Set([
|
|
170
|
+
'InvalidArg',
|
|
171
|
+
'ObjectExpected',
|
|
172
|
+
'StringExpected',
|
|
173
|
+
'NameExpected',
|
|
174
|
+
'FunctionExpected',
|
|
175
|
+
'NumberExpected',
|
|
176
|
+
'BooleanExpected',
|
|
177
|
+
'ArrayExpected',
|
|
178
|
+
'BigintExpected',
|
|
179
|
+
]);
|
|
132
180
|
/**
|
|
133
181
|
* Run a native call, re-raising its tagged napi error as the matching
|
|
134
182
|
* `DisarmError` subclass. The native shim prefixes fallible messages with
|
|
135
183
|
* `"DisarmInvalidArgument: "` or `"DisarmError: "`; we strip the matched tag
|
|
136
|
-
* cleanly.
|
|
137
|
-
*
|
|
184
|
+
* cleanly. An argument napi could not convert is a `DisarmInvalidArgument`. Any
|
|
185
|
+
* other throw — an untagged `Error`, or a non-`Error` value — is still wrapped as a
|
|
186
|
+
* `DisarmError` so nothing leaks out unwrapped.
|
|
187
|
+
*
|
|
188
|
+
* Every export goes through here, the infallible ones included: they throw on an
|
|
189
|
+
* argument of the wrong type, and "everything disarm throws is a `DisarmError`" was
|
|
190
|
+
* false for them while they called the native function directly (`formal/bindings`,
|
|
191
|
+
* N2).
|
|
138
192
|
*/
|
|
139
193
|
function call(fn) {
|
|
140
194
|
try {
|
|
141
195
|
return fn();
|
|
142
196
|
}
|
|
143
197
|
catch (e) {
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
198
|
+
throw translate(e);
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
/** The {@link DisarmError} a thrown value becomes; see {@link call}. */
|
|
202
|
+
function translate(e) {
|
|
203
|
+
if (e instanceof DisarmError) {
|
|
204
|
+
return e;
|
|
205
|
+
}
|
|
206
|
+
const msg = e instanceof Error ? e.message : String(e);
|
|
207
|
+
if (msg.startsWith(INVALID_ARG_TAG)) {
|
|
208
|
+
return new DisarmInvalidArgument(msg.slice(INVALID_ARG_TAG.length));
|
|
209
|
+
}
|
|
210
|
+
if (msg.startsWith(ERROR_TAG)) {
|
|
211
|
+
return new DisarmError(msg.slice(ERROR_TAG.length));
|
|
212
|
+
}
|
|
213
|
+
const code = e?.code;
|
|
214
|
+
if (typeof code === 'string' && NAPI_ARGUMENT_STATUSES.has(code)) {
|
|
215
|
+
return new DisarmInvalidArgument(msg);
|
|
216
|
+
}
|
|
217
|
+
return new DisarmError(msg);
|
|
218
|
+
}
|
|
219
|
+
/**
|
|
220
|
+
* Check a size or threshold option before it reaches napi, which converts any JS
|
|
221
|
+
* number to an integer: `NaN` and `0.9` became 0, `2 ** 64` saturated, so
|
|
222
|
+
* `stripZalgo('caf\u00e9', { maxMarks: NaN })` stripped the accent (`formal/bindings`, N1).
|
|
223
|
+
* A size is a non-negative safe integer; anything else throws
|
|
224
|
+
* {@link DisarmInvalidArgument}.
|
|
225
|
+
*/
|
|
226
|
+
function size(name, value) {
|
|
227
|
+
if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 0) {
|
|
228
|
+
throw new DisarmInvalidArgument(`${name} must be a non-negative integer (got ${String(value)})`);
|
|
152
229
|
}
|
|
230
|
+
return value;
|
|
231
|
+
}
|
|
232
|
+
/** {@link size} for an optional option: `undefined` or `null` stays absent. */
|
|
233
|
+
function optionalSize(name, value) {
|
|
234
|
+
return value == null ? undefined : size(name, value);
|
|
153
235
|
}
|
|
154
|
-
/** Romanize Unicode text to ASCII. */
|
|
236
|
+
/** Romanize Unicode text to ASCII. An unknown `lang` throws {@link DisarmInvalidArgument}. */
|
|
155
237
|
function transliterate(text, options = {}) {
|
|
156
238
|
const { scheme = 'default', lang } = options;
|
|
157
239
|
if (scheme === 'default' && lang == null) {
|
|
158
|
-
return native.transliterate(text);
|
|
240
|
+
return call(() => native.transliterate(text));
|
|
159
241
|
}
|
|
160
242
|
return call(() => native.transliterateOpts(text, scheme, lang ?? undefined));
|
|
161
243
|
}
|
|
@@ -163,7 +245,10 @@ function transliterate(text, options = {}) {
|
|
|
163
245
|
function reverseTransliterate(text, options) {
|
|
164
246
|
return call(() => native.reverseTransliterate(text, options.lang));
|
|
165
247
|
}
|
|
166
|
-
/**
|
|
248
|
+
/**
|
|
249
|
+
* Every character in `text` with no romanization, as `{ char, offset }` (byte offset), in
|
|
250
|
+
* order. An unknown `lang` throws {@link DisarmInvalidArgument}.
|
|
251
|
+
*/
|
|
167
252
|
function findUntranslatable(text, options = {}) {
|
|
168
253
|
const { scheme = 'default', lang } = options;
|
|
169
254
|
return call(() => native.findUntranslatable(text, scheme, lang ?? undefined));
|
|
@@ -210,12 +295,15 @@ function unmappedConfusables(options = {}) {
|
|
|
210
295
|
function findUnmappedConfusables(text, options = {}) {
|
|
211
296
|
return call(() => native.findUnmappedConfusables(text, options.target ?? 'latin'));
|
|
212
297
|
}
|
|
213
|
-
/**
|
|
298
|
+
/**
|
|
299
|
+
* Generate a URL-safe slug. Mirrors the core's `SlugConfig` defaults. An unknown `lang`
|
|
300
|
+
* throws {@link DisarmInvalidArgument}.
|
|
301
|
+
*/
|
|
214
302
|
function slugify(text, options = {}) {
|
|
215
303
|
return call(() => native.slugify(text, {
|
|
216
304
|
separator: options.separator ?? '-',
|
|
217
305
|
lowercase: options.lowercase ?? true,
|
|
218
|
-
maxLength: options.maxLength ?? 0,
|
|
306
|
+
maxLength: optionalSize('maxLength', options.maxLength) ?? 0,
|
|
219
307
|
wordBoundary: options.wordBoundary ?? false,
|
|
220
308
|
saveOrder: options.saveOrder ?? false,
|
|
221
309
|
stopwords: options.stopwords ?? [],
|
|
@@ -230,11 +318,11 @@ function slugify(text, options = {}) {
|
|
|
230
318
|
// ── Canonicalization primitives ─────────────────────────────────────────────
|
|
231
319
|
/** Strip diacritics (`"café"` → `"cafe"`). */
|
|
232
320
|
function stripAccents(text) {
|
|
233
|
-
return native.stripAccents(text);
|
|
321
|
+
return call(() => native.stripAccents(text));
|
|
234
322
|
}
|
|
235
323
|
/** Full Unicode case fold — more aggressive than `String.toLowerCase()`. */
|
|
236
324
|
function foldCase(text) {
|
|
237
|
-
return native.foldCase(text);
|
|
325
|
+
return call(() => native.foldCase(text));
|
|
238
326
|
}
|
|
239
327
|
/**
|
|
240
328
|
* Whether `text` is a stable identity key under case folding — that is, whether
|
|
@@ -247,7 +335,7 @@ function foldCase(text) {
|
|
|
247
335
|
* folded into {@link hasAnomalies}.
|
|
248
336
|
*/
|
|
249
337
|
function isCaseFoldStable(text) {
|
|
250
|
-
return native.isCaseFoldStable(text);
|
|
338
|
+
return call(() => native.isCaseFoldStable(text));
|
|
251
339
|
}
|
|
252
340
|
/**
|
|
253
341
|
* Which of `values` are the same name under `key` (#620).
|
|
@@ -268,24 +356,30 @@ function isCaseFoldStable(text) {
|
|
|
268
356
|
function findKeyCollisions(values, key, options = {}) {
|
|
269
357
|
return call(() => native.findKeyCollisions(values, key, options.lang));
|
|
270
358
|
}
|
|
271
|
-
/**
|
|
359
|
+
/**
|
|
360
|
+
* Replace emoji with their plain names. `stripModifiers` drops skin-tone/variation marks.
|
|
361
|
+
* An emoji CLDR cannot name — a regional indicator or a Plane 14 tag character standing
|
|
362
|
+
* alone — becomes `'[?]'`, the sentinel {@link transliterate} writes, and the default in
|
|
363
|
+
* every binding.
|
|
364
|
+
*/
|
|
272
365
|
function demojize(text, options = {}) {
|
|
273
|
-
return native.demojize(text, options.stripModifiers ?? false);
|
|
366
|
+
return call(() => native.demojize(text, options.stripModifiers ?? false));
|
|
274
367
|
}
|
|
275
368
|
/** Replace every emoji with `replacement`, verbatim (#972).
|
|
276
369
|
*
|
|
277
370
|
* The counterpart to {@link demojize}, and a different question of a different table.
|
|
278
371
|
* `demojize` asks *what does CLDR call this?*, so its domain is the CLDR name table,
|
|
279
372
|
* which is wider than the emoji: `demojize('x™y')` is `'x trade mark y'`. This asks *is
|
|
280
|
-
* this an emoji by the UCD's properties?* — `Emoji_Presentation=Yes`, an `Emoji=Yes`
|
|
281
|
-
* carrying `U+FE0F
|
|
373
|
+
* this an emoji by the UCD's properties?* — `Emoji_Presentation=Yes`, an `Emoji=Yes`
|
|
374
|
+
* base carrying `U+FE0F` (not an `Extended_Pictographic` one, so `★` + `U+FE0F` stays),
|
|
375
|
+
* and the ZWJ, modifier, keycap and flag sequences on those. Nothing
|
|
282
376
|
* else moves.
|
|
283
377
|
*
|
|
284
378
|
* `replacement` is inserted exactly as given, with no padding and no whitespace collapse:
|
|
285
379
|
* `''` closes an intra-word split (`aa🔥bb` → `aabb`) and `' '` keeps two words apart
|
|
286
380
|
* (`stop🛑now` → `stop now`), and no rule serves both. */
|
|
287
381
|
function replaceEmoji(text, replacement = '') {
|
|
288
|
-
return native.replaceEmoji(text, replacement);
|
|
382
|
+
return call(() => native.replaceEmoji(text, replacement));
|
|
289
383
|
}
|
|
290
384
|
// ── Normalization ───────────────────────────────────────────────────────────
|
|
291
385
|
/** Apply a Unicode normalization `form` (default `'NFC'`). */
|
|
@@ -308,43 +402,52 @@ function isNormalized(text, options = {}) {
|
|
|
308
402
|
* deleting them) means `a\rb` → `a b`, never `ab`.
|
|
309
403
|
*/
|
|
310
404
|
function collapseWhitespace(text) {
|
|
311
|
-
return native.collapseWhitespace(text);
|
|
405
|
+
return call(() => native.collapseWhitespace(text));
|
|
312
406
|
}
|
|
313
407
|
/** Remove C0/C1 control characters (except tab/newline). */
|
|
314
408
|
function stripControlChars(text) {
|
|
315
|
-
return native.stripControlChars(text);
|
|
409
|
+
return call(() => native.stripControlChars(text));
|
|
316
410
|
}
|
|
317
411
|
/** Remove zero-width characters (ZWSP/ZWNJ/ZWJ/word-joiner). */
|
|
318
412
|
function stripZeroWidthChars(text) {
|
|
319
|
-
return native.stripZeroWidthChars(text);
|
|
413
|
+
return call(() => native.stripZeroWidthChars(text));
|
|
320
414
|
}
|
|
321
415
|
/** Remove Unicode bidirectional control characters. */
|
|
322
416
|
function stripBidi(text) {
|
|
323
|
-
return native.stripBidi(text);
|
|
417
|
+
return call(() => native.stripBidi(text));
|
|
324
418
|
}
|
|
325
419
|
/** Strip the Unicode Tags block (U+E0000–U+E007F), preserving valid emoji flag sequences (#413). */
|
|
326
420
|
function stripTags(text) {
|
|
327
|
-
return native.stripTags(text);
|
|
421
|
+
return call(() => native.stripTags(text));
|
|
328
422
|
}
|
|
329
423
|
/** Strip every variation selector (VS1–VS256) (#413). */
|
|
330
424
|
function stripVariationSelectors(text) {
|
|
331
|
-
return native.stripVariationSelectors(text);
|
|
425
|
+
return call(() => native.stripVariationSelectors(text));
|
|
332
426
|
}
|
|
333
427
|
/** Strip every Unicode noncharacter (#413). */
|
|
334
428
|
function stripNoncharacters(text) {
|
|
335
|
-
return native.stripNoncharacters(text);
|
|
429
|
+
return call(() => native.stripNoncharacters(text));
|
|
336
430
|
}
|
|
337
431
|
/** Strip every Private Use Area code point (#413). */
|
|
338
432
|
function stripPua(text) {
|
|
339
|
-
return native.stripPua(text);
|
|
433
|
+
return call(() => native.stripPua(text));
|
|
340
434
|
}
|
|
341
|
-
/**
|
|
435
|
+
/**
|
|
436
|
+
* Cap the marks of each combining class on one base character at `maxMarks`, a
|
|
437
|
+
* non-negative integer. The default is the core's, 3 — equal to {@link isZalgo}'s
|
|
438
|
+
* threshold (#788), so this never strips from text `isZalgo` declines to flag. It is read
|
|
439
|
+
* from the core rather than restated here: this layer had kept the old cap of 2
|
|
440
|
+
* (`formal/bindings`, B1).
|
|
441
|
+
*/
|
|
342
442
|
function stripZalgo(text, options = {}) {
|
|
343
|
-
return call(() => native.stripZalgo(text, options.maxMarks
|
|
443
|
+
return call(() => native.stripZalgo(text, optionalSize('maxMarks', options.maxMarks)));
|
|
344
444
|
}
|
|
345
|
-
/**
|
|
445
|
+
/**
|
|
446
|
+
* Whether any base character carries more than `threshold` marks of one combining class.
|
|
447
|
+
* The default is the core's, 3.
|
|
448
|
+
*/
|
|
346
449
|
function isZalgo(text, options = {}) {
|
|
347
|
-
return call(() => native.isZalgo(text, options.threshold
|
|
450
|
+
return call(() => native.isZalgo(text, optionalSize('threshold', options.threshold)));
|
|
348
451
|
}
|
|
349
452
|
// ── Deobfuscation & security presets ────────────────────────────────────────
|
|
350
453
|
/**
|
|
@@ -365,17 +468,18 @@ function canonicalizeStrict(text, options = {}) {
|
|
|
365
468
|
* it collapses TAB/LF to a space, which the primitives leave alone. Infallible.
|
|
366
469
|
*/
|
|
367
470
|
function stripFormat(text) {
|
|
368
|
-
return native.stripFormat(text);
|
|
471
|
+
return call(() => native.stripFormat(text));
|
|
369
472
|
}
|
|
370
473
|
/** Remove obfuscation (zero-width, bidi, combining-mark abuse, homoglyphs) while keeping legible content. */
|
|
371
474
|
function stripObfuscation(text, options = {}) {
|
|
372
475
|
return call(() => native.stripObfuscation(text, options.digitPolicy ?? 'numeric'));
|
|
373
476
|
}
|
|
374
477
|
/**
|
|
375
|
-
* Canonicalize text for security-sensitive comparison:
|
|
376
|
-
* → strip invisible classes (#413) → strip control → strip
|
|
377
|
-
*
|
|
378
|
-
* (confusables
|
|
478
|
+
* Canonicalize text for security-sensitive comparison: resolve deletions → NFKC →
|
|
479
|
+
* strip bidi/format → strip invisible classes (#413) → strip control → strip
|
|
480
|
+
* zero-width → collapse whitespace → drop repeated marks → cap combining marks
|
|
481
|
+
* (anti-zalgo) → NFC → confusables and NFC to a fixed point → drop repeated marks
|
|
482
|
+
* → cap combining marks again (the fold is iterated with NFC for idempotency).
|
|
379
483
|
*
|
|
380
484
|
* The name describes the mechanism (Unicode canonicalization for matching), not
|
|
381
485
|
* a safety guarantee — this is not an output sanitizer; encode at the sink.
|
|
@@ -402,9 +506,15 @@ function securityClean(text) {
|
|
|
402
506
|
function getPipeline(profile) {
|
|
403
507
|
return call(() => native.getPipeline(profile));
|
|
404
508
|
}
|
|
405
|
-
/**
|
|
509
|
+
/**
|
|
510
|
+
* Turn arbitrary text into a filesystem-safe filename.
|
|
511
|
+
*
|
|
512
|
+
* `separator` must be printable, non-space ASCII with no character illegal on
|
|
513
|
+
* `platform` and no path separator (`/`, `\`); anything else throws
|
|
514
|
+
* {@link DisarmInvalidArgument}. `''` is allowed.
|
|
515
|
+
*/
|
|
406
516
|
function sanitizeFilename(text, options = {}) {
|
|
407
|
-
return call(() => native.sanitizeFilename(text, options.separator ?? '_', options.maxLength ?? 255, options.platform ?? 'universal', options.lang ?? undefined, options.preserveExtension ?? true));
|
|
517
|
+
return call(() => native.sanitizeFilename(text, options.separator ?? '_', optionalSize('maxLength', options.maxLength) ?? 255, options.platform ?? 'universal', options.lang ?? undefined, options.preserveExtension ?? true));
|
|
408
518
|
}
|
|
409
519
|
// ── Key-derivation presets ──────────────────────────────────────────────────
|
|
410
520
|
/**
|
|
@@ -446,7 +556,7 @@ function skeletonKey(text, options = {}) {
|
|
|
446
556
|
* equal.
|
|
447
557
|
*/
|
|
448
558
|
function editDistance(a, b) {
|
|
449
|
-
return native.editDistance(a, b);
|
|
559
|
+
return call(() => native.editDistance(a, b));
|
|
450
560
|
}
|
|
451
561
|
/**
|
|
452
562
|
* The candidate closest to `value`, with its distance, or `null` beyond `maxDistance`
|
|
@@ -454,11 +564,12 @@ function editDistance(a, b) {
|
|
|
454
564
|
* and ties go to the first candidate at the lowest distance (#894).
|
|
455
565
|
*/
|
|
456
566
|
function nearestMatch(value, candidates, options = {}) {
|
|
457
|
-
return call(() => native.nearestMatch(value, candidates, options.maxDistance ?? 1)) ?? null;
|
|
567
|
+
return (call(() => native.nearestMatch(value, candidates, optionalSize('maxDistance', options.maxDistance) ?? 1)) ?? null);
|
|
458
568
|
}
|
|
459
569
|
/**
|
|
460
|
-
* ML/NLP normalization: NFKC → emoji→text → transliterate →
|
|
461
|
-
* [case fold] → strip control → strip zero-width →
|
|
570
|
+
* ML/NLP normalization: resolve deletions → NFKC → emoji→text → transliterate →
|
|
571
|
+
* strip accents → emoji→text → [case fold] → strip control → strip zero-width →
|
|
572
|
+
* collapse whitespace → NFC.
|
|
462
573
|
*
|
|
463
574
|
* Note this folds no confusables — it is not a homoglyph defence at any setting. Put
|
|
464
575
|
* {@link normalizeConfusables} in front of it when a model needs both.
|
|
@@ -469,28 +580,28 @@ function mlNormalize(text, options = {}) {
|
|
|
469
580
|
// ── Grapheme clusters ───────────────────────────────────────────────────────
|
|
470
581
|
/** Number of grapheme clusters (user-perceived characters). */
|
|
471
582
|
function graphemeLen(text) {
|
|
472
|
-
return native.graphemeLen(text);
|
|
583
|
+
return call(() => native.graphemeLen(text));
|
|
473
584
|
}
|
|
474
585
|
/** Split `text` into grapheme-cluster strings. */
|
|
475
586
|
function graphemeSplit(text) {
|
|
476
|
-
return native.graphemeSplit(text);
|
|
587
|
+
return call(() => native.graphemeSplit(text));
|
|
477
588
|
}
|
|
478
589
|
/** Truncate to at most `maxGraphemes` clusters, never cutting through one. */
|
|
479
590
|
function graphemeTruncate(text, maxGraphemes) {
|
|
480
|
-
return call(() => native.graphemeTruncate(text, maxGraphemes));
|
|
591
|
+
return call(() => native.graphemeTruncate(text, size('maxGraphemes', maxGraphemes)));
|
|
481
592
|
}
|
|
482
593
|
/** Display width (terminal columns) of a single grapheme `cluster` by East Asian Width. */
|
|
483
594
|
function graphemeWidth(cluster, options = {}) {
|
|
484
|
-
return native.graphemeWidth(cluster, options.ambiguousWide ?? false);
|
|
595
|
+
return call(() => native.graphemeWidth(cluster, options.ambiguousWide ?? false));
|
|
485
596
|
}
|
|
486
597
|
/** Total display width (terminal columns) of `text`. */
|
|
487
598
|
function terminalWidth(text, options = {}) {
|
|
488
|
-
return native.terminalWidth(text, options.ambiguousWide ?? false);
|
|
599
|
+
return call(() => native.terminalWidth(text, options.ambiguousWide ?? false));
|
|
489
600
|
}
|
|
490
601
|
// ── Hostname / script analysis ──────────────────────────────────────────────
|
|
491
602
|
/** Whether the hostname looks like a mixed-script / confusable IDN spoof (a `false` is not a safety guarantee). */
|
|
492
603
|
function isSuspiciousHostname(host) {
|
|
493
|
-
return native.isSuspiciousHostname(host);
|
|
604
|
+
return call(() => native.isSuspiciousHostname(host));
|
|
494
605
|
}
|
|
495
606
|
/**
|
|
496
607
|
* Analyze a hostname for Unicode homoglyph spoofing, returning the full
|
|
@@ -502,11 +613,11 @@ function analyzeHostname(host, options = {}) {
|
|
|
502
613
|
}
|
|
503
614
|
/** The Unicode scripts present, in first-appearance order (Common/Inherited excluded). */
|
|
504
615
|
function detectScripts(text) {
|
|
505
|
-
return native.detectScripts(text);
|
|
616
|
+
return call(() => native.detectScripts(text));
|
|
506
617
|
}
|
|
507
618
|
/** Whether `text` mixes characters from more than one script. */
|
|
508
619
|
function isMixedScript(text) {
|
|
509
|
-
return native.isMixedScript(text);
|
|
620
|
+
return call(() => native.isMixedScript(text));
|
|
510
621
|
}
|
|
511
622
|
/**
|
|
512
623
|
* All twelve UAX #9 explicit formatting characters, uncontexted.
|
|
@@ -517,7 +628,7 @@ function isMixedScript(text) {
|
|
|
517
628
|
* right-to-left text.
|
|
518
629
|
*/
|
|
519
630
|
function hasBidiControl(text) {
|
|
520
|
-
return native.hasBidiControl(text);
|
|
631
|
+
return call(() => native.hasBidiControl(text));
|
|
521
632
|
}
|
|
522
633
|
/**
|
|
523
634
|
* Whether `text` mixes strong left-to-right and strong right-to-left characters
|
|
@@ -526,11 +637,11 @@ function hasBidiControl(text) {
|
|
|
526
637
|
* guarantee.
|
|
527
638
|
*/
|
|
528
639
|
function hasBidiConflict(text) {
|
|
529
|
-
return native.hasBidiConflict(text);
|
|
640
|
+
return call(() => native.hasBidiConflict(text));
|
|
530
641
|
}
|
|
531
642
|
/** Explain how `lang: 'auto'` detection resolves `text`. */
|
|
532
643
|
function inspectAutoLang(text) {
|
|
533
|
-
return native.inspectAutoLang(text);
|
|
644
|
+
return call(() => native.inspectAutoLang(text));
|
|
534
645
|
}
|
|
535
646
|
// ── Metadata introspection (#404) ───────────────────────────────────────────
|
|
536
647
|
/**
|
|
@@ -572,7 +683,7 @@ function confusableCoverage(script) {
|
|
|
572
683
|
* because it decides whether disarm's normalization agrees with the host platform's.
|
|
573
684
|
*/
|
|
574
685
|
function unicodeVersion() {
|
|
575
|
-
return native.unicodeVersion();
|
|
686
|
+
return call(() => native.unicodeVersion());
|
|
576
687
|
}
|
|
577
688
|
/**
|
|
578
689
|
* Whether a key stored under an earlier release still compares equal. A monotonic
|
|
@@ -580,7 +691,7 @@ function unicodeVersion() {
|
|
|
580
691
|
* for the same input. Meaningless in isolation, by design.
|
|
581
692
|
*/
|
|
582
693
|
function keySchemaVersion() {
|
|
583
|
-
return native.keySchemaVersion();
|
|
694
|
+
return call(() => native.keySchemaVersion());
|
|
584
695
|
}
|
|
585
696
|
/**
|
|
586
697
|
* The Unicode `confusables.txt` release the bundled confusable tables were folded
|
|
@@ -591,15 +702,15 @@ function keySchemaVersion() {
|
|
|
591
702
|
* confusables fold stale?" without inferring it from behaviour.
|
|
592
703
|
*/
|
|
593
704
|
function confusablesVersion() {
|
|
594
|
-
return native.confusablesVersion();
|
|
705
|
+
return call(() => native.confusablesVersion());
|
|
595
706
|
}
|
|
596
707
|
/** Every Unicode script name known to the transliteration tables. */
|
|
597
708
|
function listScripts() {
|
|
598
|
-
return native.listScripts();
|
|
709
|
+
return call(() => native.listScripts());
|
|
599
710
|
}
|
|
600
711
|
/** Every language code that has a context-aware transliteration profile. */
|
|
601
712
|
function listContextLangs() {
|
|
602
|
-
return native.listContextLangs();
|
|
713
|
+
return call(() => native.listContextLangs());
|
|
603
714
|
}
|
|
604
715
|
// ── Anomaly detection ───────────────────────────────────────────────────────
|
|
605
716
|
/**
|
|
@@ -611,10 +722,10 @@ function listContextLangs() {
|
|
|
611
722
|
* internal set on every call — used only by the leet and segmentation branches.
|
|
612
723
|
*/
|
|
613
724
|
function hasAnomalies(text, lexicon = []) {
|
|
614
|
-
if (lexicon instanceof
|
|
615
|
-
return native.hasAnomalies(text, lexicon);
|
|
725
|
+
if (lexicon instanceof Lexicon) {
|
|
726
|
+
return call(() => native.hasAnomalies(text, lexicon));
|
|
616
727
|
}
|
|
617
|
-
return native.hasAnomalies(text, Array.isArray(lexicon) ? lexicon : [...lexicon]);
|
|
728
|
+
return call(() => native.hasAnomalies(text, Array.isArray(lexicon) ? lexicon : [...lexicon]));
|
|
618
729
|
}
|
|
619
730
|
/**
|
|
620
731
|
* Full anomaly analysis: an `AnomalyReport` with `anomalous`, `kinds` (in
|
|
@@ -623,9 +734,8 @@ function hasAnomalies(text, lexicon = []) {
|
|
|
623
734
|
* `lexicon` may be a `Set`/array of words or a prebuilt {@link Lexicon} handle.
|
|
624
735
|
*/
|
|
625
736
|
function inspectAnomalies(text, lexicon = []) {
|
|
626
|
-
if (lexicon instanceof
|
|
627
|
-
return native.inspectAnomalies(text, lexicon);
|
|
737
|
+
if (lexicon instanceof Lexicon) {
|
|
738
|
+
return call(() => native.inspectAnomalies(text, lexicon));
|
|
628
739
|
}
|
|
629
|
-
|
|
630
|
-
return native.inspectAnomalies(text, words);
|
|
740
|
+
return call(() => native.inspectAnomalies(text, Array.isArray(lexicon) ? lexicon : [...lexicon]));
|
|
631
741
|
}
|