@sideid/id-profanity-filter 1.10.6 → 1.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/constants/categories/blasphemy.d.ts +4 -0
- package/dist/constants/categories/disgusting.d.ts +4 -0
- package/dist/constants/categories/drugs.d.ts +4 -0
- package/dist/constants/categories/profanity.d.ts +4 -0
- package/dist/constants/categories/slur.d.ts +4 -0
- package/dist/constants/wordList.d.ts +1 -1
- package/dist/core/analyzer.d.ts +1 -1
- package/dist/core/filter.d.ts +1 -1
- package/dist/core/matcher.d.ts +1 -1
- package/dist/index.esm.js +536 -50
- package/dist/index.esm.js.map +1 -1
- package/dist/index.js +536 -50
- package/dist/index.js.map +1 -1
- package/dist/utils/ahoCorasick.d.ts +36 -0
- package/dist/utils/similarityUtils.d.ts +10 -0
- package/package.json +1 -1
- package/src/constants/categories/blasphemy.ts +25 -0
- package/src/constants/categories/disgusting.ts +82 -0
- package/src/constants/categories/drugs.ts +72 -0
- package/src/constants/categories/profanity.ts +139 -0
- package/src/constants/categories/slur.ts +102 -0
- package/src/constants/regions/jawa.ts +356 -354
- package/src/core/analyzer.ts +28 -7
- package/src/core/matcher.ts +25 -18
- package/src/utils/ahoCorasick.ts +179 -0
- package/src/utils/similarityUtils.ts +157 -20
- package/test.js +0 -184
package/dist/index.esm.js
CHANGED
|
@@ -214,7 +214,7 @@ const jawa = [
|
|
|
214
214
|
word: "jancok",
|
|
215
215
|
category: "sexual",
|
|
216
216
|
region: "jawa",
|
|
217
|
-
severity: 0.
|
|
217
|
+
severity: 0.9,
|
|
218
218
|
aliases: ["jancuk", "jncok", "jancuk", "jncuk", "dancok", "dancuk"],
|
|
219
219
|
description: "Kata umpatan kasar dalam Bahasa Jawa",
|
|
220
220
|
context: "Umpatan kasar yang umum digunakan di Jawa Timur",
|
|
@@ -250,7 +250,7 @@ const jawa = [
|
|
|
250
250
|
word: "mbokne ancok",
|
|
251
251
|
category: "insult",
|
|
252
252
|
region: "jawa",
|
|
253
|
-
severity: 0.
|
|
253
|
+
severity: 0.9,
|
|
254
254
|
aliases: ["mbokne", "mbokneancok"],
|
|
255
255
|
description: "Umpatan yang menyinggung ibu seseorang",
|
|
256
256
|
context: "Umpatan kasar yang menyinggung orangtua orang lain",
|
|
@@ -259,7 +259,7 @@ const jawa = [
|
|
|
259
259
|
word: "pekok",
|
|
260
260
|
category: "insult",
|
|
261
261
|
region: "jawa",
|
|
262
|
-
severity: 0.
|
|
262
|
+
severity: 0.7,
|
|
263
263
|
aliases: ["pekak", "pekilk"],
|
|
264
264
|
description: "Kata hinaan yang menunjukkan kebodohan",
|
|
265
265
|
context: "Hinaan untuk menyebut orang yang dianggap sangat bodoh",
|
|
@@ -291,6 +291,248 @@ const jawa = [
|
|
|
291
291
|
description: "Secara harfiah berarti kepalamu, digunakan sebagai umpatan ringan",
|
|
292
292
|
context: "Umpatan ringan untuk menanggapi sesuatu yang dianggap tidak benar",
|
|
293
293
|
},
|
|
294
|
+
{
|
|
295
|
+
word: "kontol",
|
|
296
|
+
category: "sexual",
|
|
297
|
+
region: "jawa",
|
|
298
|
+
severity: 0.8,
|
|
299
|
+
aliases: ["kntl", "kontl"],
|
|
300
|
+
description: "Mengacu ke alat kelamin laki-laki",
|
|
301
|
+
context: "Kata vulgar yang sering digunakan sebagai umpatan kasar",
|
|
302
|
+
},
|
|
303
|
+
{
|
|
304
|
+
word: "tempek",
|
|
305
|
+
category: "sexual",
|
|
306
|
+
region: "jawa",
|
|
307
|
+
severity: 0.8,
|
|
308
|
+
aliases: ["mpek", "torok", "tempk"],
|
|
309
|
+
description: "Mengacu pada alat kelamin perempuan",
|
|
310
|
+
context: "Kata vulgar yang digunakan sebagai umpatan atau hinaan",
|
|
311
|
+
},
|
|
312
|
+
{
|
|
313
|
+
word: "silit",
|
|
314
|
+
category: "insult",
|
|
315
|
+
region: "jawa",
|
|
316
|
+
severity: 0.6,
|
|
317
|
+
aliases: ["selet", "tilis"],
|
|
318
|
+
description: "Mengacu pada bagian dubur atau anus",
|
|
319
|
+
context: "Kata kasar yang digunakan sebagai hinaan",
|
|
320
|
+
},
|
|
321
|
+
{
|
|
322
|
+
word: "mbahmu",
|
|
323
|
+
category: "insult",
|
|
324
|
+
region: "jawa",
|
|
325
|
+
severity: 0.6,
|
|
326
|
+
aliases: ["mbahmu kiper", "mbah mu"],
|
|
327
|
+
description: "Hinaan yang menyinggung nenek/kakek seseorang",
|
|
328
|
+
context: "Umpatan ringan untuk menanggapi sesuatu yang tidak masuk akal",
|
|
329
|
+
},
|
|
330
|
+
{
|
|
331
|
+
word: "makmu",
|
|
332
|
+
category: "insult",
|
|
333
|
+
region: "jawa",
|
|
334
|
+
severity: 0.7,
|
|
335
|
+
aliases: ["mak mu", "mamamu"],
|
|
336
|
+
description: "Hinaan yang menyinggung ibu seseorang",
|
|
337
|
+
context: "Umpatan yang dianggap kasar karena menyinggung orang tua",
|
|
338
|
+
},
|
|
339
|
+
{
|
|
340
|
+
word: "bajingan",
|
|
341
|
+
category: "insult",
|
|
342
|
+
region: "jawa",
|
|
343
|
+
severity: 0.7,
|
|
344
|
+
aliases: [
|
|
345
|
+
"bajilak",
|
|
346
|
+
"bajhingan",
|
|
347
|
+
"bajingak",
|
|
348
|
+
"bajingseng",
|
|
349
|
+
"bajindul",
|
|
350
|
+
"bajigur",
|
|
351
|
+
"jingan",
|
|
352
|
+
],
|
|
353
|
+
description: "Sebutan untuk orang yang dianggap jahat atau tidak bermoral",
|
|
354
|
+
context: "Umpatan untuk mengekspresikan kemarahan atau kekesalan",
|
|
355
|
+
},
|
|
356
|
+
{
|
|
357
|
+
word: "cocote",
|
|
358
|
+
category: "insult",
|
|
359
|
+
region: "jawa",
|
|
360
|
+
severity: 0.6,
|
|
361
|
+
aliases: ["cocot", "bacot", "nyocot"],
|
|
362
|
+
description: "Mengacu pada mulut dengan konotasi negatif",
|
|
363
|
+
context: "Umpatan untuk menyuruh seseorang berhenti berbicara",
|
|
364
|
+
},
|
|
365
|
+
{
|
|
366
|
+
word: "ngentot",
|
|
367
|
+
category: "sexual",
|
|
368
|
+
region: "jawa",
|
|
369
|
+
severity: 0.9,
|
|
370
|
+
aliases: ["kentu", "kentot", "iclik", "ngtt", "iclk"],
|
|
371
|
+
description: "Mengacu pada aktivitas seksual",
|
|
372
|
+
context: "Kata vulgar yang digunakan sebagai umpatan kasar",
|
|
373
|
+
},
|
|
374
|
+
{
|
|
375
|
+
word: "edan",
|
|
376
|
+
category: "insult",
|
|
377
|
+
region: "jawa",
|
|
378
|
+
severity: 0.5,
|
|
379
|
+
aliases: ["gendeng", "gila", "gendheng", "sarap"],
|
|
380
|
+
description: "Secara harfiah berarti gila atau tidak waras",
|
|
381
|
+
context: "Umpatan untuk menyebut seseorang yang dianggap tidak masuk akal",
|
|
382
|
+
},
|
|
383
|
+
{
|
|
384
|
+
word: "dapuranmu",
|
|
385
|
+
category: "insult",
|
|
386
|
+
region: "jawa",
|
|
387
|
+
severity: 0.6,
|
|
388
|
+
aliases: ["raimu", "rai mu"],
|
|
389
|
+
description: "Secara harfiah mengacu pada wajah atau rupa seseorang",
|
|
390
|
+
context: "Umpatan untuk menghina penampilan atau wajah seseorang",
|
|
391
|
+
},
|
|
392
|
+
{
|
|
393
|
+
word: "damput",
|
|
394
|
+
category: "insult",
|
|
395
|
+
region: "jawa",
|
|
396
|
+
severity: 0.7,
|
|
397
|
+
aliases: ["diamput"],
|
|
398
|
+
description: "Variasi bentuk umpatan dengan makna serupa dengan jancok",
|
|
399
|
+
context: "Umpatan kasar untuk mengekspresikan kemarahan",
|
|
400
|
+
},
|
|
401
|
+
{
|
|
402
|
+
word: "mbathang",
|
|
403
|
+
category: "insult",
|
|
404
|
+
region: "jawa",
|
|
405
|
+
severity: 0.7,
|
|
406
|
+
aliases: ["mbatang"],
|
|
407
|
+
description: "Secara harfiah berarti bangkai",
|
|
408
|
+
context: "Umpatan kasar untuk menghina seseorang",
|
|
409
|
+
},
|
|
410
|
+
{
|
|
411
|
+
word: "ndlogok",
|
|
412
|
+
category: "insult",
|
|
413
|
+
region: "jawa",
|
|
414
|
+
severity: 0.6,
|
|
415
|
+
aliases: ["ndelodok", "ndlodok"],
|
|
416
|
+
description: "Mengacu pada tindakan yang dianggap bodoh atau tidak masuk akal",
|
|
417
|
+
context: "Hinaan untuk mengkritik tindakan seseorang",
|
|
418
|
+
},
|
|
419
|
+
{
|
|
420
|
+
word: "nggateli",
|
|
421
|
+
category: "insult",
|
|
422
|
+
region: "jawa",
|
|
423
|
+
severity: 0.5,
|
|
424
|
+
aliases: ["gateli", "gathel"],
|
|
425
|
+
description: "Secara harfiah berarti gatal atau menyebalkan",
|
|
426
|
+
context: "Ungkapan untuk menunjukkan kekesalan terhadap perilaku seseorang",
|
|
427
|
+
},
|
|
428
|
+
{
|
|
429
|
+
word: "perek",
|
|
430
|
+
category: "sexual",
|
|
431
|
+
region: "jawa",
|
|
432
|
+
severity: 0.8,
|
|
433
|
+
aliases: ["lonthe", "pelacur"],
|
|
434
|
+
description: "Istilah merendahkan untuk pekerja seks komersial",
|
|
435
|
+
context: "Kata kasar untuk menghina wanita",
|
|
436
|
+
},
|
|
437
|
+
{
|
|
438
|
+
word: "picek",
|
|
439
|
+
category: "insult",
|
|
440
|
+
region: "jawa",
|
|
441
|
+
severity: 0.6,
|
|
442
|
+
aliases: ["pcek", "buta"],
|
|
443
|
+
description: "Secara harfiah berarti buta atau tidak bisa melihat",
|
|
444
|
+
context: "Hinaan untuk orang yang dianggap tidak bisa melihat kenyataan",
|
|
445
|
+
},
|
|
446
|
+
{
|
|
447
|
+
word: "untumu",
|
|
448
|
+
category: "insult",
|
|
449
|
+
region: "jawa",
|
|
450
|
+
severity: 0.5,
|
|
451
|
+
aliases: ["gigimu", "untu mu"],
|
|
452
|
+
description: "Secara harfiah berarti gigimu",
|
|
453
|
+
context: "Umpatan ringan untuk menanggapi sesuatu yang tidak disetujui",
|
|
454
|
+
},
|
|
455
|
+
{
|
|
456
|
+
word: "goblog",
|
|
457
|
+
category: "insult",
|
|
458
|
+
region: "jawa",
|
|
459
|
+
severity: 0.7,
|
|
460
|
+
aliases: ["ghoblog", "goblok", "gobhlok", "pekok"],
|
|
461
|
+
description: "Kata hinaan yang menunjukkan kebodohan ekstrem",
|
|
462
|
+
context: "Hinaan untuk menyebut seseorang yang dianggap sangat bodoh",
|
|
463
|
+
},
|
|
464
|
+
{
|
|
465
|
+
word: "tolol",
|
|
466
|
+
category: "insult",
|
|
467
|
+
region: "jawa",
|
|
468
|
+
severity: 0.7,
|
|
469
|
+
aliases: ["tholol", "tlol"],
|
|
470
|
+
description: "Kata hinaan yang menunjukkan kebodohan",
|
|
471
|
+
context: "Hinaan untuk menyebut seseorang yang dianggap bodoh",
|
|
472
|
+
},
|
|
473
|
+
{
|
|
474
|
+
word: "budheg",
|
|
475
|
+
category: "insult",
|
|
476
|
+
region: "jawa",
|
|
477
|
+
severity: 0.6,
|
|
478
|
+
aliases: ["budeg", "bdeg"],
|
|
479
|
+
description: "Secara harfiah berarti tuli atau tidak bisa mendengar",
|
|
480
|
+
context: "Hinaan untuk orang yang dianggap tidak mau mendengarkan",
|
|
481
|
+
},
|
|
482
|
+
{
|
|
483
|
+
word: "jiangkrik",
|
|
484
|
+
category: "insult",
|
|
485
|
+
region: "jawa",
|
|
486
|
+
severity: 0.4,
|
|
487
|
+
aliases: ["jiangkrek", "jangkrik"],
|
|
488
|
+
description: "Secara harfiah berarti jangkrik, digunakan sebagai eufemisme",
|
|
489
|
+
context: "Umpatan ringan sebagai pengganti kata kasar yang lebih vulgar",
|
|
490
|
+
},
|
|
491
|
+
{
|
|
492
|
+
word: "diamput",
|
|
493
|
+
category: "insult",
|
|
494
|
+
region: "jawa",
|
|
495
|
+
severity: 0.8,
|
|
496
|
+
aliases: ["damput", "djamput"],
|
|
497
|
+
description: "Bentuk umpatan kasar dengan makna serupa jancok",
|
|
498
|
+
context: "Kata kasar untuk mengekspresikan kemarahan",
|
|
499
|
+
},
|
|
500
|
+
{
|
|
501
|
+
word: "celeng",
|
|
502
|
+
category: "insult",
|
|
503
|
+
region: "jawa",
|
|
504
|
+
severity: 0.6,
|
|
505
|
+
aliases: ["cleng", "babi hutan"],
|
|
506
|
+
description: "Secara harfiah berarti babi hutan",
|
|
507
|
+
context: "Hinaan untuk orang yang dianggap jorok atau rakus",
|
|
508
|
+
},
|
|
509
|
+
{
|
|
510
|
+
word: "kampret",
|
|
511
|
+
category: "insult",
|
|
512
|
+
region: "jawa",
|
|
513
|
+
severity: 0.5,
|
|
514
|
+
aliases: ["kmpret", "kmprt"],
|
|
515
|
+
description: "Secara harfiah berarti kelelawar kecil",
|
|
516
|
+
context: "Umpatan ringan untuk mengekspresikan kekesalan",
|
|
517
|
+
},
|
|
518
|
+
{
|
|
519
|
+
word: "ndeso",
|
|
520
|
+
category: "insult",
|
|
521
|
+
region: "jawa",
|
|
522
|
+
severity: 0.4,
|
|
523
|
+
aliases: ["ndesa", "deso"],
|
|
524
|
+
description: "Secara harfiah berarti dari desa atau kampungan",
|
|
525
|
+
context: "Hinaan untuk orang yang dianggap kurang modern atau berpendidikan",
|
|
526
|
+
},
|
|
527
|
+
{
|
|
528
|
+
word: "kere",
|
|
529
|
+
category: "insult",
|
|
530
|
+
region: "jawa",
|
|
531
|
+
severity: 0.5,
|
|
532
|
+
aliases: ["miskin", "mlarat"],
|
|
533
|
+
description: "Secara harfiah berarti miskin atau tidak punya uang",
|
|
534
|
+
context: "Hinaan untuk status ekonomi seseorang yang dianggap rendah",
|
|
535
|
+
},
|
|
294
536
|
{
|
|
295
537
|
word: "itil",
|
|
296
538
|
category: "sexual",
|
|
@@ -1084,29 +1326,73 @@ function clusterSimilarWords(words, threshold = 0.8) {
|
|
|
1084
1326
|
}
|
|
1085
1327
|
/**
|
|
1086
1328
|
* Cari kata-kata kotor yang mungkin dari teks menggunakan kesamaan string
|
|
1329
|
+
* dengan optimasi untuk mengurangi kompleksitas
|
|
1087
1330
|
*
|
|
1088
1331
|
* @param text Teks yang akan diperiksa
|
|
1089
1332
|
* @param profanityWords Daftar kata kotor
|
|
1090
1333
|
* @param threshold Batas minimum kesamaan (default: 0.8)
|
|
1091
1334
|
* @returns Array kata yang mungkin merupakan kata kotor
|
|
1092
1335
|
*/
|
|
1336
|
+
/**
|
|
1337
|
+
* Cari kata-kata kotor yang mungkin dari teks berdasarkan kemiripan string
|
|
1338
|
+
* dengan optimasi biar prosesnya nggak terlalu berat
|
|
1339
|
+
*
|
|
1340
|
+
* @param text Teks yang mau dicek
|
|
1341
|
+
* @param profanityWords Daftar kata-kata kotor/kasar
|
|
1342
|
+
* @param threshold Batas minimal kemiripan (default: 0.8)
|
|
1343
|
+
* @returns Array kata yang kemungkinan kata kotor/kasar
|
|
1344
|
+
*/
|
|
1093
1345
|
function findPossibleProfanityBySimiliarity(text, profanityWords, threshold = 0.8) {
|
|
1094
1346
|
const result = [];
|
|
1347
|
+
// Optimasi dengan bikin map kata-kata kotor berdasarkan huruf pertama
|
|
1348
|
+
const profanityMap = new Map();
|
|
1349
|
+
// Kelompokkan kata-kata kotor berdasarkan huruf pertama biar pencariannya lebih cepat
|
|
1350
|
+
for (const word of profanityWords) {
|
|
1351
|
+
if (word.length < 1)
|
|
1352
|
+
continue;
|
|
1353
|
+
const firstChar = word[0].toLowerCase();
|
|
1354
|
+
if (!profanityMap.has(firstChar)) {
|
|
1355
|
+
profanityMap.set(firstChar, []);
|
|
1356
|
+
}
|
|
1357
|
+
profanityMap.get(firstChar).push(word);
|
|
1358
|
+
}
|
|
1095
1359
|
const words = text.toLowerCase().split(/\s+/);
|
|
1096
1360
|
for (const word of words) {
|
|
1097
1361
|
if (word.length < 3)
|
|
1098
1362
|
continue;
|
|
1099
|
-
|
|
1363
|
+
// Cuma bandingin dengan kata-kata kotor yang huruf pertamanya sama
|
|
1364
|
+
// atau yang perbedaan panjangnya masih masuk akal
|
|
1365
|
+
const firstChar = word[0];
|
|
1366
|
+
const candidateWords = profanityMap.get(firstChar) || [];
|
|
1367
|
+
// Cek juga huruf-huruf yang berdekatan (buat antisipasi typo di huruf pertama)
|
|
1368
|
+
// Ini opsional tapi bikin deteksinya lebih bagus
|
|
1369
|
+
const charCode = firstChar.charCodeAt(0);
|
|
1370
|
+
const prevChar = String.fromCharCode(charCode - 1);
|
|
1371
|
+
const nextChar = String.fromCharCode(charCode + 1);
|
|
1372
|
+
const adjacentCandidates = [
|
|
1373
|
+
...(profanityMap.get(prevChar) || []),
|
|
1374
|
+
...(profanityMap.get(nextChar) || []),
|
|
1375
|
+
];
|
|
1376
|
+
// Gabungin kandidat-kandidatnya, prioritasin yang huruf pertamanya sama persis
|
|
1377
|
+
const allCandidates = [...candidateWords, ...adjacentCandidates];
|
|
1378
|
+
// Filter kandidat berdasarkan perbedaan panjang sebelum ngitung kemiripannya
|
|
1379
|
+
const lengthFilteredCandidates = allCandidates.filter((candidate) => Math.abs(candidate.length - word.length) <= 2);
|
|
1380
|
+
// Cari yang paling cocok
|
|
1381
|
+
let bestMatch = null;
|
|
1382
|
+
for (const profanity of lengthFilteredCandidates) {
|
|
1100
1383
|
const similarity = stringSimilarity(word, profanity);
|
|
1101
|
-
if (similarity >= threshold
|
|
1102
|
-
|
|
1384
|
+
if (similarity >= threshold &&
|
|
1385
|
+
(!bestMatch || similarity > bestMatch.similarity)) {
|
|
1386
|
+
bestMatch = {
|
|
1103
1387
|
word,
|
|
1104
1388
|
original: profanity,
|
|
1105
1389
|
similarity,
|
|
1106
|
-
}
|
|
1107
|
-
break;
|
|
1390
|
+
};
|
|
1108
1391
|
}
|
|
1109
1392
|
}
|
|
1393
|
+
if (bestMatch) {
|
|
1394
|
+
result.push(bestMatch);
|
|
1395
|
+
}
|
|
1110
1396
|
}
|
|
1111
1397
|
return result;
|
|
1112
1398
|
}
|
|
@@ -1121,30 +1407,75 @@ function findPossibleProfanityBySimiliarity(text, profanityWords, threshold = 0.
|
|
|
1121
1407
|
*/
|
|
1122
1408
|
function findProfanityByLevenshteinDistance(text, profanityWords, threshold = 0.8, maxDistance = 2) {
|
|
1123
1409
|
const result = [];
|
|
1410
|
+
// map kata-kata kotor dikelompokkan sesuai panjangnya
|
|
1411
|
+
const profanityByLength = new Map();
|
|
1412
|
+
// Kelompokkin kata-kata kotor berdasarkan panjangnya biar pencarian lebih cepat
|
|
1413
|
+
for (const word of profanityWords) {
|
|
1414
|
+
const length = word.length;
|
|
1415
|
+
if (!profanityByLength.has(length)) {
|
|
1416
|
+
profanityByLength.set(length, []);
|
|
1417
|
+
}
|
|
1418
|
+
profanityByLength.get(length).push(word);
|
|
1419
|
+
}
|
|
1124
1420
|
const words = text.toLowerCase().split(/\s+/);
|
|
1125
1421
|
for (const word of words) {
|
|
1126
1422
|
if (word.length < 3)
|
|
1127
1423
|
continue;
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
const
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1424
|
+
let bestMatch = null;
|
|
1425
|
+
for (let len = Math.max(3, word.length - maxDistance); len <= word.length + maxDistance; len++) {
|
|
1426
|
+
const candidates = profanityByLength.get(len) || [];
|
|
1427
|
+
for (const profanity of candidates) {
|
|
1428
|
+
if (!isCharacterCountSimilar(word, profanity, maxDistance)) {
|
|
1429
|
+
continue;
|
|
1430
|
+
}
|
|
1431
|
+
const distance = levenshteinDistance(word, profanity);
|
|
1432
|
+
if (distance <= maxDistance) {
|
|
1433
|
+
const similarity = 1 - distance / Math.max(word.length, profanity.length);
|
|
1434
|
+
if (similarity >= threshold &&
|
|
1435
|
+
(!bestMatch || similarity > bestMatch.similarity)) {
|
|
1436
|
+
bestMatch = {
|
|
1437
|
+
word,
|
|
1438
|
+
original: profanity,
|
|
1439
|
+
similarity,
|
|
1440
|
+
distance,
|
|
1441
|
+
};
|
|
1442
|
+
if (distance === 0 || similarity > 0.95) {
|
|
1443
|
+
break;
|
|
1444
|
+
}
|
|
1445
|
+
}
|
|
1142
1446
|
}
|
|
1143
1447
|
}
|
|
1144
1448
|
}
|
|
1449
|
+
if (bestMatch) {
|
|
1450
|
+
result.push(bestMatch);
|
|
1451
|
+
}
|
|
1145
1452
|
}
|
|
1146
1453
|
return result;
|
|
1147
1454
|
}
|
|
1455
|
+
/**
|
|
1456
|
+
* Helper function to efficiently check if character counts between two strings
|
|
1457
|
+
* are similar enough to warrant a full Levenshtein calculation
|
|
1458
|
+
*/
|
|
1459
|
+
function isCharacterCountSimilar(str1, str2, maxDifference) {
|
|
1460
|
+
const charCount1 = {};
|
|
1461
|
+
const charCount2 = {};
|
|
1462
|
+
for (const char of str1) {
|
|
1463
|
+
charCount1[char] = (charCount1[char] || 0) + 1;
|
|
1464
|
+
}
|
|
1465
|
+
for (const char of str2) {
|
|
1466
|
+
charCount2[char] = (charCount2[char] || 0) + 1;
|
|
1467
|
+
}
|
|
1468
|
+
let diffCount = 0;
|
|
1469
|
+
for (const char in charCount1) {
|
|
1470
|
+
diffCount += Math.abs((charCount1[char] || 0) - (charCount2[char] || 0));
|
|
1471
|
+
}
|
|
1472
|
+
for (const char in charCount2) {
|
|
1473
|
+
if (!charCount1[char]) {
|
|
1474
|
+
diffCount += charCount2[char];
|
|
1475
|
+
}
|
|
1476
|
+
}
|
|
1477
|
+
return diffCount <= maxDifference * 2;
|
|
1478
|
+
}
|
|
1148
1479
|
|
|
1149
1480
|
const DEFAULT_OPTIONS = {
|
|
1150
1481
|
replaceWith: "*",
|
|
@@ -1299,6 +1630,157 @@ function makeRandomGrawlixString(length) {
|
|
|
1299
1630
|
return result;
|
|
1300
1631
|
}
|
|
1301
1632
|
|
|
1633
|
+
/**
|
|
1634
|
+
* Implementasi algoritma Aho-Corasick untuk pencocokan string
|
|
1635
|
+
* Algoritma ini memungkinkan pencarian banyak pola dalam satu kali pembacaan teks
|
|
1636
|
+
*/
|
|
1637
|
+
class AhoCorasick {
|
|
1638
|
+
constructor() {
|
|
1639
|
+
this.built = false;
|
|
1640
|
+
this.root = {
|
|
1641
|
+
children: new Map(),
|
|
1642
|
+
fail: null,
|
|
1643
|
+
output: new Set(),
|
|
1644
|
+
depth: 0,
|
|
1645
|
+
};
|
|
1646
|
+
}
|
|
1647
|
+
/**
|
|
1648
|
+
* Menambahkan pola ke dalam trie
|
|
1649
|
+
* @param pattern Pola yang akan ditambahkan
|
|
1650
|
+
*/
|
|
1651
|
+
addPattern(pattern) {
|
|
1652
|
+
if (this.built) {
|
|
1653
|
+
throw new Error("Cannot add patterns after the automaton is built");
|
|
1654
|
+
}
|
|
1655
|
+
let node = this.root;
|
|
1656
|
+
const normalizedPattern = pattern.toLowerCase();
|
|
1657
|
+
for (let i = 0; i < normalizedPattern.length; i++) {
|
|
1658
|
+
const char = normalizedPattern[i];
|
|
1659
|
+
if (!node.children.has(char)) {
|
|
1660
|
+
node.children.set(char, {
|
|
1661
|
+
children: new Map(),
|
|
1662
|
+
fail: null,
|
|
1663
|
+
output: new Set(),
|
|
1664
|
+
depth: node.depth + 1,
|
|
1665
|
+
char,
|
|
1666
|
+
});
|
|
1667
|
+
}
|
|
1668
|
+
node = node.children.get(char);
|
|
1669
|
+
}
|
|
1670
|
+
node.output.add(normalizedPattern);
|
|
1671
|
+
}
|
|
1672
|
+
/**
|
|
1673
|
+
* Membangun fungsi failure
|
|
1674
|
+
*/
|
|
1675
|
+
build() {
|
|
1676
|
+
if (this.built)
|
|
1677
|
+
return;
|
|
1678
|
+
const queue = [];
|
|
1679
|
+
// Set fail pointer for depth 1 nodes to root
|
|
1680
|
+
for (const child of this.root.children.values()) {
|
|
1681
|
+
child.fail = this.root;
|
|
1682
|
+
queue.push(child);
|
|
1683
|
+
}
|
|
1684
|
+
// BFS to build failure links
|
|
1685
|
+
while (queue.length > 0) {
|
|
1686
|
+
const current = queue.shift();
|
|
1687
|
+
for (const [char, child] of current.children.entries()) {
|
|
1688
|
+
queue.push(child);
|
|
1689
|
+
let failNode = current.fail;
|
|
1690
|
+
// Find the longest proper suffix that is also a prefix
|
|
1691
|
+
while (failNode !== null && !failNode.children.has(char)) {
|
|
1692
|
+
failNode = failNode.fail;
|
|
1693
|
+
}
|
|
1694
|
+
if (failNode === null) {
|
|
1695
|
+
child.fail = this.root;
|
|
1696
|
+
}
|
|
1697
|
+
else {
|
|
1698
|
+
child.fail = failNode.children.get(char);
|
|
1699
|
+
// Add outputs from the fail state to this node
|
|
1700
|
+
for (const output of child.fail.output) {
|
|
1701
|
+
child.output.add(output);
|
|
1702
|
+
}
|
|
1703
|
+
}
|
|
1704
|
+
}
|
|
1705
|
+
}
|
|
1706
|
+
this.built = true;
|
|
1707
|
+
}
|
|
1708
|
+
/**
|
|
1709
|
+
* Mencari semua kemunculan pola dalam teks
|
|
1710
|
+
* @param text Teks yang akan dicari
|
|
1711
|
+
* @returns Map pola yang ditemukan dengan jumlah kemunculannya
|
|
1712
|
+
*/
|
|
1713
|
+
search(text) {
|
|
1714
|
+
if (!this.built) {
|
|
1715
|
+
this.build();
|
|
1716
|
+
}
|
|
1717
|
+
const matches = new Map();
|
|
1718
|
+
const normalizedText = text.toLowerCase();
|
|
1719
|
+
let node = this.root;
|
|
1720
|
+
for (let i = 0; i < normalizedText.length; i++) {
|
|
1721
|
+
const char = normalizedText[i];
|
|
1722
|
+
// Follow failure links until we find a matching transition or reach root
|
|
1723
|
+
while (node !== this.root && !node.children.has(char)) {
|
|
1724
|
+
node = node.fail;
|
|
1725
|
+
}
|
|
1726
|
+
// Try to follow the transition
|
|
1727
|
+
if (node.children.has(char)) {
|
|
1728
|
+
node = node.children.get(char);
|
|
1729
|
+
}
|
|
1730
|
+
// Check for any matches at this node
|
|
1731
|
+
for (const match of node.output) {
|
|
1732
|
+
matches.set(match, (matches.get(match) || 0) + 1);
|
|
1733
|
+
}
|
|
1734
|
+
}
|
|
1735
|
+
return matches;
|
|
1736
|
+
}
|
|
1737
|
+
/**
|
|
1738
|
+
* Mencari semua kemunculan pola dalam teks dan mengembalikan hanya pola unik
|
|
1739
|
+
* @param text Teks yang akan dicari
|
|
1740
|
+
* @returns Set pola yang ditemukan
|
|
1741
|
+
*/
|
|
1742
|
+
searchUnique(text) {
|
|
1743
|
+
const matches = this.search(text);
|
|
1744
|
+
return new Set(matches.keys());
|
|
1745
|
+
}
|
|
1746
|
+
/**
|
|
1747
|
+
* Mengecek apakah teks mengandung setidaknya satu pola
|
|
1748
|
+
* @param text Teks yang akan dicari
|
|
1749
|
+
* @returns Boolean apakah pola ditemukan
|
|
1750
|
+
*/
|
|
1751
|
+
containsAny(text) {
|
|
1752
|
+
if (!this.built) {
|
|
1753
|
+
this.build();
|
|
1754
|
+
}
|
|
1755
|
+
const normalizedText = text.toLowerCase();
|
|
1756
|
+
let node = this.root;
|
|
1757
|
+
for (let i = 0; i < normalizedText.length; i++) {
|
|
1758
|
+
const char = normalizedText[i];
|
|
1759
|
+
while (node !== this.root && !node.children.has(char)) {
|
|
1760
|
+
node = node.fail;
|
|
1761
|
+
}
|
|
1762
|
+
if (node.children.has(char)) {
|
|
1763
|
+
node = node.children.get(char);
|
|
1764
|
+
}
|
|
1765
|
+
if (node.output.size > 0) {
|
|
1766
|
+
return true;
|
|
1767
|
+
}
|
|
1768
|
+
}
|
|
1769
|
+
return false;
|
|
1770
|
+
}
|
|
1771
|
+
}
|
|
1772
|
+
|
|
1773
|
+
const globalAhoCorasick = new AhoCorasick();
|
|
1774
|
+
let ahoCorasickInitialized = false;
|
|
1775
|
+
function initializeAhoCorasick(words) {
|
|
1776
|
+
if (ahoCorasickInitialized)
|
|
1777
|
+
return;
|
|
1778
|
+
for (const word of words) {
|
|
1779
|
+
globalAhoCorasick.addPattern(word);
|
|
1780
|
+
}
|
|
1781
|
+
globalAhoCorasick.build();
|
|
1782
|
+
ahoCorasickInitialized = true;
|
|
1783
|
+
}
|
|
1302
1784
|
function findProfanity(text, options = {}) {
|
|
1303
1785
|
const { wordList = [], detectLeetSpeak = true, checkSubstring = false, whitelist = [], categories, regions, severityThreshold = 0, indonesianVariation = false, detectSimilarity = false, similarityThreshold = 0.8, detectSplit = false, useLevenshtein = false, maxLevenshteinDistance = 2, } = { ...DEFAULT_OPTIONS, ...options };
|
|
1304
1786
|
const normalizedText = normalizeText(text);
|
|
@@ -1338,24 +1820,16 @@ function findProfanity(text, options = {}) {
|
|
|
1338
1820
|
}
|
|
1339
1821
|
const matches = new Set();
|
|
1340
1822
|
const actualMatches = new Map();
|
|
1341
|
-
wordsToCheck
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
});
|
|
1349
|
-
let match;
|
|
1350
|
-
while ((match = regex.exec(normalizedText)) !== null) {
|
|
1351
|
-
const originalWord = aliasMap.get(word.toLowerCase()) || word.toLowerCase();
|
|
1352
|
-
matches.add(originalWord);
|
|
1353
|
-
if (!actualMatches.has(originalWord)) {
|
|
1354
|
-
actualMatches.set(originalWord, []);
|
|
1355
|
-
}
|
|
1356
|
-
actualMatches.get(originalWord)?.push(match[0]);
|
|
1823
|
+
initializeAhoCorasick(wordsToCheck);
|
|
1824
|
+
const basicMatches = globalAhoCorasick.searchUnique(normalizedText);
|
|
1825
|
+
for (const match of basicMatches) {
|
|
1826
|
+
const originalWord = aliasMap.get(match.toLowerCase()) || match.toLowerCase();
|
|
1827
|
+
matches.add(originalWord);
|
|
1828
|
+
if (!actualMatches.has(originalWord)) {
|
|
1829
|
+
actualMatches.set(originalWord, []);
|
|
1357
1830
|
}
|
|
1358
|
-
|
|
1831
|
+
actualMatches.get(originalWord)?.push(match);
|
|
1832
|
+
}
|
|
1359
1833
|
if (detectLeetSpeak) {
|
|
1360
1834
|
wordsToCheck.forEach((word) => {
|
|
1361
1835
|
const leetRegex = createWordRegex(word, {
|
|
@@ -1532,7 +2006,7 @@ function calculateSeverity(matchDetails) {
|
|
|
1532
2006
|
* @returns FilterResult dengan hasil filter
|
|
1533
2007
|
*/
|
|
1534
2008
|
function filter(text, options = {}) {
|
|
1535
|
-
const { replaceWith =
|
|
2009
|
+
const { replaceWith = "*", fullWordCensor = true, detectLeetSpeak = true, whitelist = [], checkSubstring = false, useRandomGrawlix = false, keepFirstAndLast = false, indonesianVariation = false, detectSplit = false, detectSimilarity = false, useLevenshtein = false, maxLevenshteinDistance = 2, similarityThreshold = 0.8, } = { ...DEFAULT_OPTIONS, ...options };
|
|
1536
2010
|
const matches = findProfanity(text, {
|
|
1537
2011
|
...options,
|
|
1538
2012
|
detectLeetSpeak,
|
|
@@ -1564,7 +2038,7 @@ function filter(text, options = {}) {
|
|
|
1564
2038
|
variants.push(word);
|
|
1565
2039
|
const uniqueVariants = [...new Set(variants)];
|
|
1566
2040
|
uniqueVariants.forEach((variant) => {
|
|
1567
|
-
const regex = new RegExp(`\\b${escapeRegExp(variant)}\\b`,
|
|
2041
|
+
const regex = new RegExp(`\\b${escapeRegExp(variant)}\\b`, "gi");
|
|
1568
2042
|
let match;
|
|
1569
2043
|
while ((match = regex.exec(filteredText)) !== null) {
|
|
1570
2044
|
const originalWord = match[0];
|
|
@@ -1582,7 +2056,7 @@ function filter(text, options = {}) {
|
|
|
1582
2056
|
censored: censoredWord,
|
|
1583
2057
|
metadata,
|
|
1584
2058
|
});
|
|
1585
|
-
filteredText = filteredText.replace(new RegExp(`\\b${escapeRegExp(originalWord)}\\b`,
|
|
2059
|
+
filteredText = filteredText.replace(new RegExp(`\\b${escapeRegExp(originalWord)}\\b`, "g"), censoredWord);
|
|
1586
2060
|
}
|
|
1587
2061
|
});
|
|
1588
2062
|
if (detectSplit || detectLeetSpeak) {
|
|
@@ -1611,7 +2085,7 @@ function filter(text, options = {}) {
|
|
|
1611
2085
|
censored: censoredWord,
|
|
1612
2086
|
metadata,
|
|
1613
2087
|
});
|
|
1614
|
-
filteredText = filteredText.replace(new RegExp(escapeRegExp(originalWord),
|
|
2088
|
+
filteredText = filteredText.replace(new RegExp(escapeRegExp(originalWord), "g"), censoredWord);
|
|
1615
2089
|
}
|
|
1616
2090
|
}
|
|
1617
2091
|
if (detectSplit) {
|
|
@@ -1639,7 +2113,7 @@ function filter(text, options = {}) {
|
|
|
1639
2113
|
censored: censoredWord,
|
|
1640
2114
|
metadata,
|
|
1641
2115
|
});
|
|
1642
|
-
filteredText = filteredText.replace(new RegExp(escapeRegExp(originalWord),
|
|
2116
|
+
filteredText = filteredText.replace(new RegExp(escapeRegExp(originalWord), "g"), censoredWord);
|
|
1643
2117
|
}
|
|
1644
2118
|
}
|
|
1645
2119
|
}
|
|
@@ -1651,7 +2125,7 @@ function filter(text, options = {}) {
|
|
|
1651
2125
|
m.aliases.some((alias) => alias.toLowerCase() === word.toLowerCase())));
|
|
1652
2126
|
const variants = actualMatches.get(word.toLowerCase()) || [];
|
|
1653
2127
|
variants.forEach((variant) => {
|
|
1654
|
-
const exactVariantRegex = new RegExp(`\\b${escapeRegExp(variant)}\\b`,
|
|
2128
|
+
const exactVariantRegex = new RegExp(`\\b${escapeRegExp(variant)}\\b`, "gi");
|
|
1655
2129
|
let match;
|
|
1656
2130
|
while ((match = exactVariantRegex.exec(filteredText)) !== null) {
|
|
1657
2131
|
const originalWord = match[0];
|
|
@@ -1669,7 +2143,7 @@ function filter(text, options = {}) {
|
|
|
1669
2143
|
censored: censoredWord,
|
|
1670
2144
|
metadata,
|
|
1671
2145
|
});
|
|
1672
|
-
filteredText = filteredText.replace(new RegExp(`\\b${escapeRegExp(originalWord)}\\b`,
|
|
2146
|
+
filteredText = filteredText.replace(new RegExp(`\\b${escapeRegExp(originalWord)}\\b`, "g"), censoredWord);
|
|
1673
2147
|
}
|
|
1674
2148
|
});
|
|
1675
2149
|
});
|
|
@@ -1718,8 +2192,20 @@ function analyze(text, options = {}) {
|
|
|
1718
2192
|
const severityScore = calculateSeverity(matchDetails);
|
|
1719
2193
|
let similarWords = [];
|
|
1720
2194
|
if (mergedOptions.detectSimilarity) {
|
|
1721
|
-
|
|
1722
|
-
|
|
2195
|
+
if (matchDetails.length > 0) {
|
|
2196
|
+
const wordList = matchDetails.map((word) => word.word);
|
|
2197
|
+
if (mergedOptions.useLevenshtein) {
|
|
2198
|
+
const levenshteinResults = findProfanityByLevenshteinDistance(text, wordList, mergedOptions.similarityThreshold || 0.8, mergedOptions.maxLevenshteinDistance || 2);
|
|
2199
|
+
similarWords = levenshteinResults.map((item) => ({
|
|
2200
|
+
word: item.word,
|
|
2201
|
+
original: item.original,
|
|
2202
|
+
similarity: item.similarity,
|
|
2203
|
+
}));
|
|
2204
|
+
}
|
|
2205
|
+
else {
|
|
2206
|
+
similarWords = findPossibleProfanityBySimiliarity(text, wordList, mergedOptions.similarityThreshold || 0.8);
|
|
2207
|
+
}
|
|
2208
|
+
}
|
|
1723
2209
|
}
|
|
1724
2210
|
return {
|
|
1725
2211
|
hasProfanity: true,
|
|
@@ -1816,9 +2302,9 @@ function analyzeWithContext(text, contextWindowSize = 5, options = {}) {
|
|
|
1816
2302
|
const regex = createContextRegex(word, contextWindowSize);
|
|
1817
2303
|
let match;
|
|
1818
2304
|
while ((match = regex.exec(text)) !== null) {
|
|
1819
|
-
const beforeContext = match[1] ||
|
|
2305
|
+
const beforeContext = match[1] || "";
|
|
1820
2306
|
const wordMatch = match[2];
|
|
1821
|
-
const afterContext = match[3] ||
|
|
2307
|
+
const afterContext = match[3] || "";
|
|
1822
2308
|
result.push({
|
|
1823
2309
|
word: wordMatch,
|
|
1824
2310
|
context: beforeContext + wordMatch + afterContext,
|