iraqi-dialect-bad-words 1.3.0 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +74 -8
- package/dist/index.d.mts +95 -6
- package/dist/index.d.ts +95 -6
- package/dist/index.js +372 -24
- package/dist/index.mjs +372 -24
- package/package.json +6 -3
package/README.md
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
|
|
7
7
|
A profanity filter for the Iraqi Arabic dialect. Detects and masks bad words in user-generated text.
|
|
8
8
|
|
|
9
|
-
Works in **React Native / Expo**, **React** (Vite, Next.js, CRA, …) and **Node.js**, with TypeScript types included. No dependencies.
|
|
9
|
+
Works in **React Native / Expo**, **React** (Vite, Next.js, CRA, …), **Vue** (Vite, Nuxt) and **Node.js**, with TypeScript types included. No dependencies.
|
|
10
10
|
|
|
11
11
|
## Install
|
|
12
12
|
|
|
@@ -55,6 +55,37 @@ export function CommentInput() {
|
|
|
55
55
|
}
|
|
56
56
|
```
|
|
57
57
|
|
|
58
|
+
### Vue example
|
|
59
|
+
|
|
60
|
+
Create the filter once in its own module and import it where you need it. Keep it out of `ref()` and `reactive()`: the filter never changes, and wrapping it in reactive proxies only slows every check down.
|
|
61
|
+
|
|
62
|
+
```ts
|
|
63
|
+
// src/profanity.ts
|
|
64
|
+
import IraqiBadWordsFilter from 'iraqi-dialect-bad-words';
|
|
65
|
+
|
|
66
|
+
export const filter = new IraqiBadWordsFilter();
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
```vue
|
|
70
|
+
<script setup lang="ts">
|
|
71
|
+
import { ref, computed } from 'vue';
|
|
72
|
+
import { filter } from '@/profanity';
|
|
73
|
+
|
|
74
|
+
const text = ref('');
|
|
75
|
+
const isProfane = computed(() => filter.isProfane(text.value));
|
|
76
|
+
</script>
|
|
77
|
+
|
|
78
|
+
<template>
|
|
79
|
+
<textarea v-model="text" dir="auto" />
|
|
80
|
+
<p v-if="isProfane">Please keep it respectful.</p>
|
|
81
|
+
<button :disabled="isProfane">Send</button>
|
|
82
|
+
</template>
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
To highlight bad words, render the text in pieces split at the `start` and `end` of each match from [`inspect()`](#inspecttext-string-options-profanitymatch) rather than with `v-html`, so user text can't inject HTML.
|
|
86
|
+
|
|
87
|
+
In any framework, check the text on your server too (a Nuxt server route or any Node backend can use the same package): a check in the browser can be skipped by calling your API directly.
|
|
88
|
+
|
|
58
89
|
## API
|
|
59
90
|
|
|
60
91
|
### `new IraqiBadWordsFilter(options?)`
|
|
@@ -64,6 +95,7 @@ Creates a filter with the built-in word list.
|
|
|
64
95
|
```ts
|
|
65
96
|
new IraqiBadWordsFilter(); // profanity only
|
|
66
97
|
new IraqiBadWordsFilter({ includeInsults: true }); // also milder insults
|
|
98
|
+
new IraqiBadWordsFilter({ includePhrases: true }); // also vulgar phrases
|
|
67
99
|
new IraqiBadWordsFilter({ stripSuffixes: true }); // also unlisted suffix forms
|
|
68
100
|
new IraqiBadWordsFilter({ dialects: ['mosul'] }); // also Mosul ر → غ forms
|
|
69
101
|
new IraqiBadWordsFilter({ customWords: ['word1', 'word2'] });
|
|
@@ -76,6 +108,8 @@ new IraqiBadWordsFilter(['word1', 'word2']); // same as customWords
|
|
|
76
108
|
| `customWords` | `[]` | Extra words to filter, on top of the built-in list. Can come from your own JSON file (see below). |
|
|
77
109
|
| `safeWords` | `[]` | Extra safe stems, on top of the built-in [`safe-stems.json`](src/safe-stems.json): clean words that would otherwise match once a prefix or suffix is stripped, like the built-in `بعير` ("camel" = `ب` + `عير`). Use it for clean words your app sees flagged, especially ones your own `customWords` cause. Their prefixed and suffixed forms are safe too (`وبعيرك`). Can come from your own JSON file (see below). |
|
|
78
110
|
| `includeInsults` | `false` | Also filter milder insults like كلب, حمار, قندرة, نعال. Off by default because many of them are everyday words, so turning it on flags sentences like `عندي كلب` ("I have a dog"). |
|
|
111
|
+
| `includePhrases` | `false` | Also filter the phrases in [`phrases.json`](src/phrases.json): clean words that are vulgar together, like `تعال العب بيه`. Off by default because the same words can be meant literally ("come play with it"). |
|
|
112
|
+
| `customPhrases` | `[]` | Extra phrases to filter, whether or not `includePhrases` is on. Can come from your own JSON file. |
|
|
79
113
|
| `stripSuffixes` | `false` | Also match words with a pronoun suffix (ي ك ج ه ها نا كم هم) that the list doesn't spell out, e.g. `طيزهم` → `طيز`. Off by default because it widens matching: a clean word that ends like a bad word plus a suffix gets flagged too. |
|
|
80
114
|
| `dialects` | `[]` | Regional pronunciations to also match. `'mosul'`: ر said as غ, so every entry with ر also matches its غ form (`شرموطة` → `شغموطة`); the ر forms keep matching. Safe stems get their غ form too. |
|
|
81
115
|
|
|
@@ -128,7 +162,7 @@ filter.inspect('روح للكواد!');
|
|
|
128
162
|
| `start`, `end` | Offsets in the **original** text, so `text.slice(start, end) === token`. Counted in JavaScript string units (UTF-16); convert them if another platform counts characters differently. |
|
|
129
163
|
| `normalized` | The word after normalization (Arabizi is folded too, so `Qa7ba` shows as `gahba`). For a word with گ, the reading that matched (ق or ك). |
|
|
130
164
|
| `entry` | The list entry it matched, spelled as in the list. For a Mosul form, the listed ر spelling. |
|
|
131
|
-
| `source` | `'words'`, `'insults'` or `'custom'`. A word on more than one list keeps the first: built-in, then insults, then custom. |
|
|
165
|
+
| `source` | `'words'`, `'insults'`, `'phrases'` or `'custom'`. A word on more than one list keeps the first: built-in, then insults, then custom. |
|
|
132
166
|
| `prefix` | The attached prefix stripped before matching (after normalization, so `لل` shows as `ل`), or `''` for a direct match. |
|
|
133
167
|
| `suffix` | The suffix stripped before matching (only with `stripSuffixes`), or `''`. |
|
|
134
168
|
| `rules` | The rules the word needed to match the entry (see below). Empty when it's spelled exactly like the entry. |
|
|
@@ -145,17 +179,22 @@ filter.inspect('وبعيرك', { includeSuppressed: true });
|
|
|
145
179
|
|
|
146
180
|
| Rule | Example |
|
|
147
181
|
|---|---|
|
|
182
|
+
| `unicode-forms` | `ﻋﻴﺮ`, `gahba`, `𝐠𝐚𝐡𝐛𝐚` (presentation forms, full-width and mathematical letters) |
|
|
148
183
|
| `diacritics` | `عَارَة`, `عـــارة` |
|
|
149
184
|
| `invisible` | `عارة` (zero-width character inside) |
|
|
150
185
|
| `spelling` | `عاره` (ة/ه, ى/ي, أ/ا) |
|
|
151
186
|
| `keyboard` | `عیر`, `کس` (Persian/Kurdish letters) |
|
|
187
|
+
| `lookalike` | `gаhba` (Cyrillic `а` in a Latin word) |
|
|
152
188
|
| `arabic-digits` | `ga٧ba` |
|
|
153
189
|
| `stretched` | `عاااارة`, `للكواد` |
|
|
154
190
|
| `glued-digits` | `عارة3` |
|
|
155
191
|
| `arabizi` | `qa7ba` → `gahba` |
|
|
156
192
|
| `gaf-as-qaf` / `gaf-as-kaf` | `گحبة` → قحبة, `گسها` → كسها |
|
|
157
|
-
| `prefix` / `suffix` | `بطيزك`, `طيزهم` (with `stripSuffixes`) |
|
|
193
|
+
| `prefix` / `suffix` | `بطيزك`, `ياعارة`, `طيزهم` (with `stripSuffixes`) |
|
|
158
194
|
| `mosul` | `شغموطة` → شرموطة (with `dialects: ['mosul']`) |
|
|
195
|
+
| `phrase` | `تعال العب بيه` (with `includePhrases`) |
|
|
196
|
+
| `split` | `ع*ي*ر`, `ع ي ر` (pieces joined again) |
|
|
197
|
+
| `symbols` | `g@hba`, `$harmoota` (@ as a, $ as s) |
|
|
159
198
|
|
|
160
199
|
Including the output of `inspect` in a [“Clean word flagged” issue](https://github.com/karam-fikre/iraqi-dialect-bad-words/issues/new?template=false-positive.yml) shows exactly which entry matched and which rules it needed; the issue form has a field for it.
|
|
161
200
|
|
|
@@ -171,6 +210,22 @@ Adds safe stems to this filter, like the `safeWords` option.
|
|
|
171
210
|
|
|
172
211
|
Removes words from this filter, for example a built-in word you don't want flagged in your app.
|
|
173
212
|
|
|
213
|
+
### `addPhrases(phrases: string[]): void` / `removePhrases(phrases: string[]): void`
|
|
214
|
+
|
|
215
|
+
Adds phrases to this filter, like the `customPhrases` option, or removes them, including built-in ones.
|
|
216
|
+
|
|
217
|
+
A phrase is its words separated by spaces. A word can list alternatives separated by `|`, so one entry covers every combination:
|
|
218
|
+
|
|
219
|
+
```json
|
|
220
|
+
[
|
|
221
|
+
"قوم|كوم بيه|بينو"
|
|
222
|
+
]
|
|
223
|
+
```
|
|
224
|
+
|
|
225
|
+
matches `قوم بيه`, `قوم بينو`, `كوم بيه` and `كوم بينو`. A phrase matches when its words appear one after another with only spaces or punctuation between them (`تعال، العب بيه!`). Each word is normalized like any other, so `گوم` matches the `قوم` alternative without listing it. The first word can carry an attached prefix (`وتعال العب بيه`) or the Iraqi progressive `د` (`دقوم بيه`). Other forms aren't stripped or guessed, so list each one you want as an alternative.
|
|
226
|
+
|
|
227
|
+
The whole phrase is reported as one match with the rule `phrase`, its `entry` being the line as listed, and `clean()` masks all of it. `removePhrases()` takes a line as listed; the order of its alternatives doesn't matter.
|
|
228
|
+
|
|
174
229
|
## What it catches
|
|
175
230
|
|
|
176
231
|
Matching is done per whole word, after normalizing both the text and the word list, so these all match the same entry:
|
|
@@ -184,18 +239,26 @@ Matching is done per whole word, after normalizing both the text and the word li
|
|
|
184
239
|
| Persian/Kurdish keyboard letters ی ک ە | `عیر`, `کس` |
|
|
185
240
|
| گ read as ق (Iraqi) and as ك (Kurdish typists) | `گحبة` = `قحبة`, `گس` = `كس` |
|
|
186
241
|
| Invisible characters hidden inside a word (zero-width, direction marks, soft hyphen) | `عارة` |
|
|
187
|
-
|
|
|
242
|
+
| Letters split by symbols, or by spaces between single letters | `ع*ي*ر`, `ع.ي.ر`, `ع ي ر`, `عي * ر`, `شرم.وطة`, `g-a-h-b-a` |
|
|
243
|
+
| Arabic presentation forms, full-width and mathematical letters | `ﻋﻴﺮ`, `gahba`, `𝐠𝐚𝐡𝐛𝐚` |
|
|
244
|
+
| Greek or Cyrillic look-alike letters in a Latin word | `gаhba` (Cyrillic `а`) |
|
|
245
|
+
| Attached prefixes و ف ب ل ك ال, and يا / ويا | `والعارة`, `للعارة`, `ياعارة`, `يالعارة`, `وياعارة` |
|
|
188
246
|
| Pronoun suffixes (with `stripSuffixes`) | `طيزهم`, `وطيزكم` |
|
|
189
247
|
| Punctuation next to the word | `عارة!` `عارة،` |
|
|
190
248
|
| Digits glued to or inside an Arabic word | `عارة3`, `عا٣رة` |
|
|
191
|
-
| Arabizi (Latin letters and numbers, Western or Arabic-Indic), including spelling variants | `ga7ba` = `ga٧ba` = `qahba` = `gahba`, `kosomak` = `kusumak`, `3ayri` = `ayri` |
|
|
249
|
+
| Arabizi (Latin letters and numbers, Western or Arabic-Indic), including spelling variants | `ga7ba` = `ga٧ba` = `qahba` = `gahba`, `kosomak` = `kusumak`, `3ayri` = `ayri`, `6eezak` = `teezak` (6 = ط), `9awad` = `8awad` = `gawad` (9, 8 = ق) |
|
|
250
|
+
| @ and $ standing in for letters in Arabizi | `g@hba`, `$harmoota` |
|
|
192
251
|
|
|
193
252
|
Because matching is whole-word, a clean word that merely contains a bad word is not flagged.
|
|
194
253
|
|
|
254
|
+
Split words are found by joining neighbouring pieces when the gap between them is short, on one line, and either has no spaces (`عي*ر`), sits between single letters (`ع ي ر`), or has a symbol between pieces of one or two letters (`عي * ر`). Normal sentences like `انا و انت` or `د. علي` stay apart, and the joined word must match an entry exactly. A split safe stem stays safe: `ب ع ي ر` is بعير (camel).
|
|
255
|
+
|
|
195
256
|
## Known limitations
|
|
196
257
|
|
|
197
|
-
- Phrases (
|
|
258
|
+
- Phrases match only as listed, word for word: another word in between (`تعال هنا العب بيه`) or an unlisted form (`بيهم`) isn't matched.
|
|
198
259
|
- Words glued together without a space (e.g. the same word typed twice) aren't detected.
|
|
260
|
+
- A letter replaced by a symbol (`ع*ر`, `ك*بة`) isn't detected: the symbol could stand for any letter, and `ع*ر` could just as well be عمر.
|
|
261
|
+
- Longer pieces split with spaces around the symbol (`شرم * وطة`) aren't joined, so normal sentences aren't either.
|
|
199
262
|
- No word list is complete. Use this as one layer of moderation, not the only one.
|
|
200
263
|
|
|
201
264
|
## Contributing
|
|
@@ -207,9 +270,9 @@ The word list is the heart of this package, and native speakers are the best jud
|
|
|
207
270
|
If it's flagged only because of a prefix (e.g. `بعير` "camel" = `ب` + a bad word), the fix is adding it to [`src/safe-stems.json`](src/safe-stems.json).
|
|
208
271
|
A safe stem written with گ protects only words spelled with گ, so `گلب` ("heart", Iraqi spelling of قلب) can be safe while `كلب` is not.
|
|
209
272
|
|
|
210
|
-
- **A sentence is judged wrong:** add it to [`test/fixtures/sentences.json`](test/fixtures/sentences.json) with the verdict a native speaker expects (`"profane": true` or `false`, plus `"options"` if it needs `includeInsults` or `stripSuffixes`). Every sentence there runs as a test, so later word-list changes can't silently flip it.
|
|
273
|
+
- **A sentence is judged wrong:** add it to [`test/fixtures/sentences.json`](test/fixtures/sentences.json) with the verdict a native speaker expects (`"profane": true` or `false`, plus `"options"` if it needs `includeInsults`, `includePhrases` or `stripSuffixes`). Every sentence there runs as a test, so later word-list changes can't silently flip it.
|
|
211
274
|
|
|
212
|
-
Words that also have a common, normal meaning (like كلب "dog") belong in [`src/insults.json`](src/insults.json), which is opt-in, not in `words.json`.
|
|
275
|
+
Words that also have a common, normal meaning (like كلب "dog") belong in [`src/insults.json`](src/insults.json), which is opt-in, not in `words.json`. Phrases of clean words that are vulgar together belong in [`src/phrases.json`](src/phrases.json), also opt-in.
|
|
213
276
|
|
|
214
277
|
### Development
|
|
215
278
|
|
|
@@ -218,8 +281,11 @@ npm install
|
|
|
218
281
|
npm test # run tests
|
|
219
282
|
npm run typecheck # type-check
|
|
220
283
|
npm run build # build dist/
|
|
284
|
+
npm run check -- article.txt --phrases # list every match in a text file
|
|
221
285
|
```
|
|
222
286
|
|
|
287
|
+
`npm run check` runs the filter over a text file (or stdin, e.g. `pbpaste | npm run check`) and prints each match with its line and column, the entry it matched, its source and rules, and the text around it. Flags turn on options: `--insults`, `--phrases`, `--suffixes`, `--mosul`, and `--clean` to also print the masked text. Running it on articles you know are clean is the quickest way to find false positives: each match is either a missing safe stem or a word that belongs in an opt-in list.
|
|
288
|
+
|
|
223
289
|
## License
|
|
224
290
|
|
|
225
291
|
[MIT](LICENSE)
|
package/dist/index.d.mts
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* A rule a word needed to match its list entry, reported in ProfanityMatch.rules.
|
|
3
3
|
* Only rules that made a difference are reported: عارة matching عارة has none.
|
|
4
4
|
*/
|
|
5
|
-
type MatchRule = "diacritics" | "invisible" | "spelling" | "keyboard" | "arabic-digits" | "stretched" | "glued-digits" | "arabizi" | "gaf-as-qaf" | "gaf-as-kaf" | "prefix" | "suffix" | "mosul";
|
|
5
|
+
type MatchRule = "unicode-forms" | "diacritics" | "invisible" | "spelling" | "keyboard" | "lookalike" | "arabic-digits" | "stretched" | "glued-digits" | "arabizi" | "gaf-as-qaf" | "gaf-as-kaf" | "prefix" | "suffix" | "mosul" | "split" | "symbols" | "phrase";
|
|
6
6
|
interface IraqiBadWordsFilterOptions {
|
|
7
7
|
/**
|
|
8
8
|
* Extra words to filter, on top of the built-in list. Can be your own JSON
|
|
@@ -22,6 +22,17 @@ interface IraqiBadWordsFilterOptions {
|
|
|
22
22
|
* many of them are everyday words ("عندي كلب", "جلب لي هدية").
|
|
23
23
|
*/
|
|
24
24
|
includeInsults?: boolean;
|
|
25
|
+
/**
|
|
26
|
+
* Also filter phrases (phrases.json) whose words are clean on their own but
|
|
27
|
+
* vulgar together (تعال العب بيه). Off by default because the same words can
|
|
28
|
+
* be meant literally ("come play with it").
|
|
29
|
+
*/
|
|
30
|
+
includePhrases?: boolean;
|
|
31
|
+
/**
|
|
32
|
+
* Extra phrases to filter, whatever includePhrases is. Can be your own JSON
|
|
33
|
+
* file: `customPhrases: require('./my-phrases.json')`.
|
|
34
|
+
*/
|
|
35
|
+
customPhrases?: string[];
|
|
25
36
|
/**
|
|
26
37
|
* Also match words with a pronoun suffix the list doesn't spell out
|
|
27
38
|
* (طيزهم -> طيز). Off by default: it widens matching, so it can flag
|
|
@@ -38,7 +49,7 @@ interface IraqiBadWordsFilterOptions {
|
|
|
38
49
|
/** A regional pronunciation the filter can also match. */
|
|
39
50
|
type Dialect = "mosul";
|
|
40
51
|
/** Which list a dictionary entry came from. */
|
|
41
|
-
type WordSource = "words" | "insults" | "custom";
|
|
52
|
+
type WordSource = "words" | "insults" | "phrases" | "custom";
|
|
42
53
|
interface InspectOptions {
|
|
43
54
|
/**
|
|
44
55
|
* Also return words that matched but were cleared by a safe stem
|
|
@@ -48,7 +59,7 @@ interface InspectOptions {
|
|
|
48
59
|
}
|
|
49
60
|
/** A bad word found by inspect(). */
|
|
50
61
|
interface ProfanityMatch {
|
|
51
|
-
/** The word exactly as written in the input. */
|
|
62
|
+
/** The word (or phrase) exactly as written in the input. */
|
|
52
63
|
token: string;
|
|
53
64
|
/**
|
|
54
65
|
* Offset of the word in the input; text.slice(start, end) === token.
|
|
@@ -63,7 +74,7 @@ interface ProfanityMatch {
|
|
|
63
74
|
*/
|
|
64
75
|
normalized: string;
|
|
65
76
|
/**
|
|
66
|
-
* The dictionary entry it matched, spelled as in its list. For a Mosul form
|
|
77
|
+
* The dictionary entry (word or phrase) it matched, spelled as in its list. For a Mosul form
|
|
67
78
|
* (rule "mosul"), the listed ر spelling: الشرموطة for الشغموطة.
|
|
68
79
|
*/
|
|
69
80
|
entry: string;
|
|
@@ -86,6 +97,10 @@ declare class IraqiBadWordsFilter {
|
|
|
86
97
|
private safeStems;
|
|
87
98
|
private stripSuffixes;
|
|
88
99
|
private mosul;
|
|
100
|
+
private entryStarts;
|
|
101
|
+
private longest;
|
|
102
|
+
private normalized;
|
|
103
|
+
private phrases;
|
|
89
104
|
constructor(options?: string[] | IraqiBadWordsFilterOptions);
|
|
90
105
|
/**
|
|
91
106
|
* Normalizes a word or token for comparison: removes diacritics, tatweel and
|
|
@@ -118,11 +133,30 @@ declare class IraqiBadWordsFilter {
|
|
|
118
133
|
* Adds words from one list. A word already present keeps its first source.
|
|
119
134
|
*/
|
|
120
135
|
private addEntries;
|
|
136
|
+
/**
|
|
137
|
+
* Reads a listed phrase: words separated by spaces, each with its alternatives
|
|
138
|
+
* separated by | (قوم|كوم بيه|بينو). Punctuation in a word is ignored, and
|
|
139
|
+
* alternatives that are empty after normalization (a lone tatweel) are left
|
|
140
|
+
* out. With the Mosul dialect, an alternative with ر also has its غ form.
|
|
141
|
+
* Phrases that match the same text get the same id.
|
|
142
|
+
*/
|
|
143
|
+
private parsePhrase;
|
|
144
|
+
/**
|
|
145
|
+
* Adds phrases from one list, indexed by each alternative of their first word.
|
|
146
|
+
* A phrase already present keeps its first source.
|
|
147
|
+
*/
|
|
148
|
+
private addPhraseEntries;
|
|
149
|
+
/**
|
|
150
|
+
* The first word of a phrase with attached prefixes removed, and also the
|
|
151
|
+
* Iraqi progressive د (دقوم بيه, ودقوم بيه).
|
|
152
|
+
*/
|
|
153
|
+
private phraseCandidates;
|
|
121
154
|
/**
|
|
122
155
|
* Checks a single word, also trying it with attached prefixes removed
|
|
123
156
|
* (والكحبة -> كحبة, بطيزك -> طيزك), and suffixes removed when stripSuffixes is on.
|
|
124
157
|
* Returns the matched entry, or null.
|
|
125
158
|
* A match cleared by a safe stem is returned with `suppressedBy` set.
|
|
159
|
+
* Pass `spelled` when the token is already normalized.
|
|
126
160
|
*/
|
|
127
161
|
private matchWord;
|
|
128
162
|
/**
|
|
@@ -139,12 +173,58 @@ declare class IraqiBadWordsFilter {
|
|
|
139
173
|
/** Matches one normalized reading of a word. */
|
|
140
174
|
private matchReading;
|
|
141
175
|
/**
|
|
142
|
-
* Finds every match in the text, including ones cleared by a safe stem
|
|
176
|
+
* Finds every match in the text, including ones cleared by a safe stem: whole
|
|
177
|
+
* words, phrases, words split into pieces, and Arabizi words with symbols as
|
|
178
|
+
* letters. Matches from different passes can overlap; inspect() resolves that.
|
|
179
|
+
*/
|
|
180
|
+
private findMatches;
|
|
181
|
+
/**
|
|
182
|
+
* Finds listed phrases: their words one after another, with only spaces or
|
|
183
|
+
* punctuation between them, each one of its alternatives. Each word is
|
|
184
|
+
* normalized like any other, and the first can carry an attached prefix
|
|
185
|
+
* (وتعال العب بيه) or the progressive د (دقوم بيه). The longest phrase wins.
|
|
186
|
+
*/
|
|
187
|
+
private findPhraseMatches;
|
|
188
|
+
/**
|
|
189
|
+
* Matches each word in the text.
|
|
143
190
|
* A token with invisible characters is matched whole first (عارة -> عارة);
|
|
144
191
|
* if that finds nothing profane, each visible piece is tried on its own, so an
|
|
145
192
|
* invisible character between two words can't hide a bad word either.
|
|
146
193
|
*/
|
|
147
|
-
private
|
|
194
|
+
private findWordMatches;
|
|
195
|
+
/**
|
|
196
|
+
* Whether two neighbouring pieces can be one split word. The gap between them
|
|
197
|
+
* must be short and on one line, and either have no spaces (عي*ر, شرم.وطة),
|
|
198
|
+
* sit between single letters (ع ي ر), or have a symbol between pieces of at
|
|
199
|
+
* most two letters (عي * ر). Normal sentences (انا و انت, كلمة، كلمة) stay apart.
|
|
200
|
+
* `size` gives a piece's normalized length, worked out only when needed.
|
|
201
|
+
*/
|
|
202
|
+
private joinable;
|
|
203
|
+
/**
|
|
204
|
+
* Whether a normalized word, or a longer one made by joining more pieces to
|
|
205
|
+
* it, could still match an entry: it must be the start of a prefix, or a
|
|
206
|
+
* prefix plus the start of an entry (or of an entry and a suffix). A word
|
|
207
|
+
* with no letters yet (digits only) always could.
|
|
208
|
+
*/
|
|
209
|
+
private couldMatch;
|
|
210
|
+
/**
|
|
211
|
+
* Finds bad words split into pieces by symbols or spaces (ع*ي*ر, ع ي ر,
|
|
212
|
+
* g-a-h-b-a) by joining neighbouring pieces again. From each piece, the
|
|
213
|
+
* longest joined stretch that matches wins, and its span covers the separators.
|
|
214
|
+
*/
|
|
215
|
+
private findSplitMatches;
|
|
216
|
+
/**
|
|
217
|
+
* Finds Arabizi words with @ or $ standing in for a letter (g@hba, $harmoota).
|
|
218
|
+
* A word glued to other letters, like an Arabic word, is left to the other passes.
|
|
219
|
+
*/
|
|
220
|
+
private findSymbolMatches;
|
|
221
|
+
/**
|
|
222
|
+
* Puts matches in text order and drops overlapping ones, so each span is
|
|
223
|
+
* reported and masked once. Real matches go before ones cleared by a safe
|
|
224
|
+
* stem, then earlier and longer before later or shorter: ع*ي*ر is one match,
|
|
225
|
+
* not also its pieces.
|
|
226
|
+
*/
|
|
227
|
+
private resolveOverlaps;
|
|
148
228
|
/**
|
|
149
229
|
* Checks if a given text contains profane words.
|
|
150
230
|
*/
|
|
@@ -170,6 +250,15 @@ declare class IraqiBadWordsFilter {
|
|
|
170
250
|
* Remove words from the active filter dictionary.
|
|
171
251
|
*/
|
|
172
252
|
removeWords(wordsToRemove: string[]): void;
|
|
253
|
+
/**
|
|
254
|
+
* Add phrases to this filter, like the customPhrases option.
|
|
255
|
+
*/
|
|
256
|
+
addPhrases(newPhrases: string[]): void;
|
|
257
|
+
/**
|
|
258
|
+
* Remove phrases from this filter, for example a built-in phrase you don't want
|
|
259
|
+
* flagged. Pass the phrase as listed, alternatives included.
|
|
260
|
+
*/
|
|
261
|
+
removePhrases(phrasesToRemove: string[]): void;
|
|
173
262
|
}
|
|
174
263
|
|
|
175
264
|
export { type Dialect, type InspectOptions, IraqiBadWordsFilter, type IraqiBadWordsFilterOptions, type MatchRule, type ProfanityMatch, type WordSource, IraqiBadWordsFilter as default };
|
package/dist/index.d.ts
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* A rule a word needed to match its list entry, reported in ProfanityMatch.rules.
|
|
3
3
|
* Only rules that made a difference are reported: عارة matching عارة has none.
|
|
4
4
|
*/
|
|
5
|
-
type MatchRule = "diacritics" | "invisible" | "spelling" | "keyboard" | "arabic-digits" | "stretched" | "glued-digits" | "arabizi" | "gaf-as-qaf" | "gaf-as-kaf" | "prefix" | "suffix" | "mosul";
|
|
5
|
+
type MatchRule = "unicode-forms" | "diacritics" | "invisible" | "spelling" | "keyboard" | "lookalike" | "arabic-digits" | "stretched" | "glued-digits" | "arabizi" | "gaf-as-qaf" | "gaf-as-kaf" | "prefix" | "suffix" | "mosul" | "split" | "symbols" | "phrase";
|
|
6
6
|
interface IraqiBadWordsFilterOptions {
|
|
7
7
|
/**
|
|
8
8
|
* Extra words to filter, on top of the built-in list. Can be your own JSON
|
|
@@ -22,6 +22,17 @@ interface IraqiBadWordsFilterOptions {
|
|
|
22
22
|
* many of them are everyday words ("عندي كلب", "جلب لي هدية").
|
|
23
23
|
*/
|
|
24
24
|
includeInsults?: boolean;
|
|
25
|
+
/**
|
|
26
|
+
* Also filter phrases (phrases.json) whose words are clean on their own but
|
|
27
|
+
* vulgar together (تعال العب بيه). Off by default because the same words can
|
|
28
|
+
* be meant literally ("come play with it").
|
|
29
|
+
*/
|
|
30
|
+
includePhrases?: boolean;
|
|
31
|
+
/**
|
|
32
|
+
* Extra phrases to filter, whatever includePhrases is. Can be your own JSON
|
|
33
|
+
* file: `customPhrases: require('./my-phrases.json')`.
|
|
34
|
+
*/
|
|
35
|
+
customPhrases?: string[];
|
|
25
36
|
/**
|
|
26
37
|
* Also match words with a pronoun suffix the list doesn't spell out
|
|
27
38
|
* (طيزهم -> طيز). Off by default: it widens matching, so it can flag
|
|
@@ -38,7 +49,7 @@ interface IraqiBadWordsFilterOptions {
|
|
|
38
49
|
/** A regional pronunciation the filter can also match. */
|
|
39
50
|
type Dialect = "mosul";
|
|
40
51
|
/** Which list a dictionary entry came from. */
|
|
41
|
-
type WordSource = "words" | "insults" | "custom";
|
|
52
|
+
type WordSource = "words" | "insults" | "phrases" | "custom";
|
|
42
53
|
interface InspectOptions {
|
|
43
54
|
/**
|
|
44
55
|
* Also return words that matched but were cleared by a safe stem
|
|
@@ -48,7 +59,7 @@ interface InspectOptions {
|
|
|
48
59
|
}
|
|
49
60
|
/** A bad word found by inspect(). */
|
|
50
61
|
interface ProfanityMatch {
|
|
51
|
-
/** The word exactly as written in the input. */
|
|
62
|
+
/** The word (or phrase) exactly as written in the input. */
|
|
52
63
|
token: string;
|
|
53
64
|
/**
|
|
54
65
|
* Offset of the word in the input; text.slice(start, end) === token.
|
|
@@ -63,7 +74,7 @@ interface ProfanityMatch {
|
|
|
63
74
|
*/
|
|
64
75
|
normalized: string;
|
|
65
76
|
/**
|
|
66
|
-
* The dictionary entry it matched, spelled as in its list. For a Mosul form
|
|
77
|
+
* The dictionary entry (word or phrase) it matched, spelled as in its list. For a Mosul form
|
|
67
78
|
* (rule "mosul"), the listed ر spelling: الشرموطة for الشغموطة.
|
|
68
79
|
*/
|
|
69
80
|
entry: string;
|
|
@@ -86,6 +97,10 @@ declare class IraqiBadWordsFilter {
|
|
|
86
97
|
private safeStems;
|
|
87
98
|
private stripSuffixes;
|
|
88
99
|
private mosul;
|
|
100
|
+
private entryStarts;
|
|
101
|
+
private longest;
|
|
102
|
+
private normalized;
|
|
103
|
+
private phrases;
|
|
89
104
|
constructor(options?: string[] | IraqiBadWordsFilterOptions);
|
|
90
105
|
/**
|
|
91
106
|
* Normalizes a word or token for comparison: removes diacritics, tatweel and
|
|
@@ -118,11 +133,30 @@ declare class IraqiBadWordsFilter {
|
|
|
118
133
|
* Adds words from one list. A word already present keeps its first source.
|
|
119
134
|
*/
|
|
120
135
|
private addEntries;
|
|
136
|
+
/**
|
|
137
|
+
* Reads a listed phrase: words separated by spaces, each with its alternatives
|
|
138
|
+
* separated by | (قوم|كوم بيه|بينو). Punctuation in a word is ignored, and
|
|
139
|
+
* alternatives that are empty after normalization (a lone tatweel) are left
|
|
140
|
+
* out. With the Mosul dialect, an alternative with ر also has its غ form.
|
|
141
|
+
* Phrases that match the same text get the same id.
|
|
142
|
+
*/
|
|
143
|
+
private parsePhrase;
|
|
144
|
+
/**
|
|
145
|
+
* Adds phrases from one list, indexed by each alternative of their first word.
|
|
146
|
+
* A phrase already present keeps its first source.
|
|
147
|
+
*/
|
|
148
|
+
private addPhraseEntries;
|
|
149
|
+
/**
|
|
150
|
+
* The first word of a phrase with attached prefixes removed, and also the
|
|
151
|
+
* Iraqi progressive د (دقوم بيه, ودقوم بيه).
|
|
152
|
+
*/
|
|
153
|
+
private phraseCandidates;
|
|
121
154
|
/**
|
|
122
155
|
* Checks a single word, also trying it with attached prefixes removed
|
|
123
156
|
* (والكحبة -> كحبة, بطيزك -> طيزك), and suffixes removed when stripSuffixes is on.
|
|
124
157
|
* Returns the matched entry, or null.
|
|
125
158
|
* A match cleared by a safe stem is returned with `suppressedBy` set.
|
|
159
|
+
* Pass `spelled` when the token is already normalized.
|
|
126
160
|
*/
|
|
127
161
|
private matchWord;
|
|
128
162
|
/**
|
|
@@ -139,12 +173,58 @@ declare class IraqiBadWordsFilter {
|
|
|
139
173
|
/** Matches one normalized reading of a word. */
|
|
140
174
|
private matchReading;
|
|
141
175
|
/**
|
|
142
|
-
* Finds every match in the text, including ones cleared by a safe stem
|
|
176
|
+
* Finds every match in the text, including ones cleared by a safe stem: whole
|
|
177
|
+
* words, phrases, words split into pieces, and Arabizi words with symbols as
|
|
178
|
+
* letters. Matches from different passes can overlap; inspect() resolves that.
|
|
179
|
+
*/
|
|
180
|
+
private findMatches;
|
|
181
|
+
/**
|
|
182
|
+
* Finds listed phrases: their words one after another, with only spaces or
|
|
183
|
+
* punctuation between them, each one of its alternatives. Each word is
|
|
184
|
+
* normalized like any other, and the first can carry an attached prefix
|
|
185
|
+
* (وتعال العب بيه) or the progressive د (دقوم بيه). The longest phrase wins.
|
|
186
|
+
*/
|
|
187
|
+
private findPhraseMatches;
|
|
188
|
+
/**
|
|
189
|
+
* Matches each word in the text.
|
|
143
190
|
* A token with invisible characters is matched whole first (عارة -> عارة);
|
|
144
191
|
* if that finds nothing profane, each visible piece is tried on its own, so an
|
|
145
192
|
* invisible character between two words can't hide a bad word either.
|
|
146
193
|
*/
|
|
147
|
-
private
|
|
194
|
+
private findWordMatches;
|
|
195
|
+
/**
|
|
196
|
+
* Whether two neighbouring pieces can be one split word. The gap between them
|
|
197
|
+
* must be short and on one line, and either have no spaces (عي*ر, شرم.وطة),
|
|
198
|
+
* sit between single letters (ع ي ر), or have a symbol between pieces of at
|
|
199
|
+
* most two letters (عي * ر). Normal sentences (انا و انت, كلمة، كلمة) stay apart.
|
|
200
|
+
* `size` gives a piece's normalized length, worked out only when needed.
|
|
201
|
+
*/
|
|
202
|
+
private joinable;
|
|
203
|
+
/**
|
|
204
|
+
* Whether a normalized word, or a longer one made by joining more pieces to
|
|
205
|
+
* it, could still match an entry: it must be the start of a prefix, or a
|
|
206
|
+
* prefix plus the start of an entry (or of an entry and a suffix). A word
|
|
207
|
+
* with no letters yet (digits only) always could.
|
|
208
|
+
*/
|
|
209
|
+
private couldMatch;
|
|
210
|
+
/**
|
|
211
|
+
* Finds bad words split into pieces by symbols or spaces (ع*ي*ر, ع ي ر,
|
|
212
|
+
* g-a-h-b-a) by joining neighbouring pieces again. From each piece, the
|
|
213
|
+
* longest joined stretch that matches wins, and its span covers the separators.
|
|
214
|
+
*/
|
|
215
|
+
private findSplitMatches;
|
|
216
|
+
/**
|
|
217
|
+
* Finds Arabizi words with @ or $ standing in for a letter (g@hba, $harmoota).
|
|
218
|
+
* A word glued to other letters, like an Arabic word, is left to the other passes.
|
|
219
|
+
*/
|
|
220
|
+
private findSymbolMatches;
|
|
221
|
+
/**
|
|
222
|
+
* Puts matches in text order and drops overlapping ones, so each span is
|
|
223
|
+
* reported and masked once. Real matches go before ones cleared by a safe
|
|
224
|
+
* stem, then earlier and longer before later or shorter: ع*ي*ر is one match,
|
|
225
|
+
* not also its pieces.
|
|
226
|
+
*/
|
|
227
|
+
private resolveOverlaps;
|
|
148
228
|
/**
|
|
149
229
|
* Checks if a given text contains profane words.
|
|
150
230
|
*/
|
|
@@ -170,6 +250,15 @@ declare class IraqiBadWordsFilter {
|
|
|
170
250
|
* Remove words from the active filter dictionary.
|
|
171
251
|
*/
|
|
172
252
|
removeWords(wordsToRemove: string[]): void;
|
|
253
|
+
/**
|
|
254
|
+
* Add phrases to this filter, like the customPhrases option.
|
|
255
|
+
*/
|
|
256
|
+
addPhrases(newPhrases: string[]): void;
|
|
257
|
+
/**
|
|
258
|
+
* Remove phrases from this filter, for example a built-in phrase you don't want
|
|
259
|
+
* flagged. Pass the phrase as listed, alternatives included.
|
|
260
|
+
*/
|
|
261
|
+
removePhrases(phrasesToRemove: string[]): void;
|
|
173
262
|
}
|
|
174
263
|
|
|
175
264
|
export { type Dialect, type InspectOptions, IraqiBadWordsFilter, type IraqiBadWordsFilterOptions, type MatchRule, type ProfanityMatch, type WordSource, IraqiBadWordsFilter as default };
|