@sideid/id-profanity-filter 1.10.6 → 1.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/constants/categories/blasphemy.d.ts +4 -0
- package/dist/constants/categories/disgusting.d.ts +4 -0
- package/dist/constants/categories/drugs.d.ts +4 -0
- package/dist/constants/categories/profanity.d.ts +4 -0
- package/dist/constants/categories/slur.d.ts +4 -0
- package/dist/constants/wordList.d.ts +1 -1
- package/dist/core/analyzer.d.ts +1 -1
- package/dist/core/filter.d.ts +1 -1
- package/dist/core/matcher.d.ts +1 -1
- package/dist/index.esm.js +536 -50
- package/dist/index.esm.js.map +1 -1
- package/dist/index.js +536 -50
- package/dist/index.js.map +1 -1
- package/dist/utils/ahoCorasick.d.ts +36 -0
- package/dist/utils/similarityUtils.d.ts +10 -0
- package/package.json +1 -1
- package/src/constants/categories/blasphemy.ts +25 -0
- package/src/constants/categories/disgusting.ts +82 -0
- package/src/constants/categories/drugs.ts +72 -0
- package/src/constants/categories/profanity.ts +139 -0
- package/src/constants/categories/slur.ts +102 -0
- package/src/constants/regions/jawa.ts +356 -354
- package/src/core/analyzer.ts +28 -7
- package/src/core/matcher.ts +25 -18
- package/src/utils/ahoCorasick.ts +179 -0
- package/src/utils/similarityUtils.ts +157 -20
- package/test.js +0 -184
package/dist/index.js
CHANGED
|
@@ -218,7 +218,7 @@ const jawa = [
|
|
|
218
218
|
word: "jancok",
|
|
219
219
|
category: "sexual",
|
|
220
220
|
region: "jawa",
|
|
221
|
-
severity: 0.
|
|
221
|
+
severity: 0.9,
|
|
222
222
|
aliases: ["jancuk", "jncok", "jancuk", "jncuk", "dancok", "dancuk"],
|
|
223
223
|
description: "Kata umpatan kasar dalam Bahasa Jawa",
|
|
224
224
|
context: "Umpatan kasar yang umum digunakan di Jawa Timur",
|
|
@@ -254,7 +254,7 @@ const jawa = [
|
|
|
254
254
|
word: "mbokne ancok",
|
|
255
255
|
category: "insult",
|
|
256
256
|
region: "jawa",
|
|
257
|
-
severity: 0.
|
|
257
|
+
severity: 0.9,
|
|
258
258
|
aliases: ["mbokne", "mbokneancok"],
|
|
259
259
|
description: "Umpatan yang menyinggung ibu seseorang",
|
|
260
260
|
context: "Umpatan kasar yang menyinggung orangtua orang lain",
|
|
@@ -263,7 +263,7 @@ const jawa = [
|
|
|
263
263
|
word: "pekok",
|
|
264
264
|
category: "insult",
|
|
265
265
|
region: "jawa",
|
|
266
|
-
severity: 0.
|
|
266
|
+
severity: 0.7,
|
|
267
267
|
aliases: ["pekak", "pekilk"],
|
|
268
268
|
description: "Kata hinaan yang menunjukkan kebodohan",
|
|
269
269
|
context: "Hinaan untuk menyebut orang yang dianggap sangat bodoh",
|
|
@@ -295,6 +295,248 @@ const jawa = [
|
|
|
295
295
|
description: "Secara harfiah berarti kepalamu, digunakan sebagai umpatan ringan",
|
|
296
296
|
context: "Umpatan ringan untuk menanggapi sesuatu yang dianggap tidak benar",
|
|
297
297
|
},
|
|
298
|
+
{
|
|
299
|
+
word: "kontol",
|
|
300
|
+
category: "sexual",
|
|
301
|
+
region: "jawa",
|
|
302
|
+
severity: 0.8,
|
|
303
|
+
aliases: ["kntl", "kontl"],
|
|
304
|
+
description: "Mengacu ke alat kelamin laki-laki",
|
|
305
|
+
context: "Kata vulgar yang sering digunakan sebagai umpatan kasar",
|
|
306
|
+
},
|
|
307
|
+
{
|
|
308
|
+
word: "tempek",
|
|
309
|
+
category: "sexual",
|
|
310
|
+
region: "jawa",
|
|
311
|
+
severity: 0.8,
|
|
312
|
+
aliases: ["mpek", "torok", "tempk"],
|
|
313
|
+
description: "Mengacu pada alat kelamin perempuan",
|
|
314
|
+
context: "Kata vulgar yang digunakan sebagai umpatan atau hinaan",
|
|
315
|
+
},
|
|
316
|
+
{
|
|
317
|
+
word: "silit",
|
|
318
|
+
category: "insult",
|
|
319
|
+
region: "jawa",
|
|
320
|
+
severity: 0.6,
|
|
321
|
+
aliases: ["selet", "tilis"],
|
|
322
|
+
description: "Mengacu pada bagian dubur atau anus",
|
|
323
|
+
context: "Kata kasar yang digunakan sebagai hinaan",
|
|
324
|
+
},
|
|
325
|
+
{
|
|
326
|
+
word: "mbahmu",
|
|
327
|
+
category: "insult",
|
|
328
|
+
region: "jawa",
|
|
329
|
+
severity: 0.6,
|
|
330
|
+
aliases: ["mbahmu kiper", "mbah mu"],
|
|
331
|
+
description: "Hinaan yang menyinggung nenek/kakek seseorang",
|
|
332
|
+
context: "Umpatan ringan untuk menanggapi sesuatu yang tidak masuk akal",
|
|
333
|
+
},
|
|
334
|
+
{
|
|
335
|
+
word: "makmu",
|
|
336
|
+
category: "insult",
|
|
337
|
+
region: "jawa",
|
|
338
|
+
severity: 0.7,
|
|
339
|
+
aliases: ["mak mu", "mamamu"],
|
|
340
|
+
description: "Hinaan yang menyinggung ibu seseorang",
|
|
341
|
+
context: "Umpatan yang dianggap kasar karena menyinggung orang tua",
|
|
342
|
+
},
|
|
343
|
+
{
|
|
344
|
+
word: "bajingan",
|
|
345
|
+
category: "insult",
|
|
346
|
+
region: "jawa",
|
|
347
|
+
severity: 0.7,
|
|
348
|
+
aliases: [
|
|
349
|
+
"bajilak",
|
|
350
|
+
"bajhingan",
|
|
351
|
+
"bajingak",
|
|
352
|
+
"bajingseng",
|
|
353
|
+
"bajindul",
|
|
354
|
+
"bajigur",
|
|
355
|
+
"jingan",
|
|
356
|
+
],
|
|
357
|
+
description: "Sebutan untuk orang yang dianggap jahat atau tidak bermoral",
|
|
358
|
+
context: "Umpatan untuk mengekspresikan kemarahan atau kekesalan",
|
|
359
|
+
},
|
|
360
|
+
{
|
|
361
|
+
word: "cocote",
|
|
362
|
+
category: "insult",
|
|
363
|
+
region: "jawa",
|
|
364
|
+
severity: 0.6,
|
|
365
|
+
aliases: ["cocot", "bacot", "nyocot"],
|
|
366
|
+
description: "Mengacu pada mulut dengan konotasi negatif",
|
|
367
|
+
context: "Umpatan untuk menyuruh seseorang berhenti berbicara",
|
|
368
|
+
},
|
|
369
|
+
{
|
|
370
|
+
word: "ngentot",
|
|
371
|
+
category: "sexual",
|
|
372
|
+
region: "jawa",
|
|
373
|
+
severity: 0.9,
|
|
374
|
+
aliases: ["kentu", "kentot", "iclik", "ngtt", "iclk"],
|
|
375
|
+
description: "Mengacu pada aktivitas seksual",
|
|
376
|
+
context: "Kata vulgar yang digunakan sebagai umpatan kasar",
|
|
377
|
+
},
|
|
378
|
+
{
|
|
379
|
+
word: "edan",
|
|
380
|
+
category: "insult",
|
|
381
|
+
region: "jawa",
|
|
382
|
+
severity: 0.5,
|
|
383
|
+
aliases: ["gendeng", "gila", "gendheng", "sarap"],
|
|
384
|
+
description: "Secara harfiah berarti gila atau tidak waras",
|
|
385
|
+
context: "Umpatan untuk menyebut seseorang yang dianggap tidak masuk akal",
|
|
386
|
+
},
|
|
387
|
+
{
|
|
388
|
+
word: "dapuranmu",
|
|
389
|
+
category: "insult",
|
|
390
|
+
region: "jawa",
|
|
391
|
+
severity: 0.6,
|
|
392
|
+
aliases: ["raimu", "rai mu"],
|
|
393
|
+
description: "Secara harfiah mengacu pada wajah atau rupa seseorang",
|
|
394
|
+
context: "Umpatan untuk menghina penampilan atau wajah seseorang",
|
|
395
|
+
},
|
|
396
|
+
{
|
|
397
|
+
word: "damput",
|
|
398
|
+
category: "insult",
|
|
399
|
+
region: "jawa",
|
|
400
|
+
severity: 0.7,
|
|
401
|
+
aliases: ["diamput"],
|
|
402
|
+
description: "Variasi bentuk umpatan dengan makna serupa dengan jancok",
|
|
403
|
+
context: "Umpatan kasar untuk mengekspresikan kemarahan",
|
|
404
|
+
},
|
|
405
|
+
{
|
|
406
|
+
word: "mbathang",
|
|
407
|
+
category: "insult",
|
|
408
|
+
region: "jawa",
|
|
409
|
+
severity: 0.7,
|
|
410
|
+
aliases: ["mbatang"],
|
|
411
|
+
description: "Secara harfiah berarti bangkai",
|
|
412
|
+
context: "Umpatan kasar untuk menghina seseorang",
|
|
413
|
+
},
|
|
414
|
+
{
|
|
415
|
+
word: "ndlogok",
|
|
416
|
+
category: "insult",
|
|
417
|
+
region: "jawa",
|
|
418
|
+
severity: 0.6,
|
|
419
|
+
aliases: ["ndelodok", "ndlodok"],
|
|
420
|
+
description: "Mengacu pada tindakan yang dianggap bodoh atau tidak masuk akal",
|
|
421
|
+
context: "Hinaan untuk mengkritik tindakan seseorang",
|
|
422
|
+
},
|
|
423
|
+
{
|
|
424
|
+
word: "nggateli",
|
|
425
|
+
category: "insult",
|
|
426
|
+
region: "jawa",
|
|
427
|
+
severity: 0.5,
|
|
428
|
+
aliases: ["gateli", "gathel"],
|
|
429
|
+
description: "Secara harfiah berarti gatal atau menyebalkan",
|
|
430
|
+
context: "Ungkapan untuk menunjukkan kekesalan terhadap perilaku seseorang",
|
|
431
|
+
},
|
|
432
|
+
{
|
|
433
|
+
word: "perek",
|
|
434
|
+
category: "sexual",
|
|
435
|
+
region: "jawa",
|
|
436
|
+
severity: 0.8,
|
|
437
|
+
aliases: ["lonthe", "pelacur"],
|
|
438
|
+
description: "Istilah merendahkan untuk pekerja seks komersial",
|
|
439
|
+
context: "Kata kasar untuk menghina wanita",
|
|
440
|
+
},
|
|
441
|
+
{
|
|
442
|
+
word: "picek",
|
|
443
|
+
category: "insult",
|
|
444
|
+
region: "jawa",
|
|
445
|
+
severity: 0.6,
|
|
446
|
+
aliases: ["pcek", "buta"],
|
|
447
|
+
description: "Secara harfiah berarti buta atau tidak bisa melihat",
|
|
448
|
+
context: "Hinaan untuk orang yang dianggap tidak bisa melihat kenyataan",
|
|
449
|
+
},
|
|
450
|
+
{
|
|
451
|
+
word: "untumu",
|
|
452
|
+
category: "insult",
|
|
453
|
+
region: "jawa",
|
|
454
|
+
severity: 0.5,
|
|
455
|
+
aliases: ["gigimu", "untu mu"],
|
|
456
|
+
description: "Secara harfiah berarti gigimu",
|
|
457
|
+
context: "Umpatan ringan untuk menanggapi sesuatu yang tidak disetujui",
|
|
458
|
+
},
|
|
459
|
+
{
|
|
460
|
+
word: "goblog",
|
|
461
|
+
category: "insult",
|
|
462
|
+
region: "jawa",
|
|
463
|
+
severity: 0.7,
|
|
464
|
+
aliases: ["ghoblog", "goblok", "gobhlok", "pekok"],
|
|
465
|
+
description: "Kata hinaan yang menunjukkan kebodohan ekstrem",
|
|
466
|
+
context: "Hinaan untuk menyebut seseorang yang dianggap sangat bodoh",
|
|
467
|
+
},
|
|
468
|
+
{
|
|
469
|
+
word: "tolol",
|
|
470
|
+
category: "insult",
|
|
471
|
+
region: "jawa",
|
|
472
|
+
severity: 0.7,
|
|
473
|
+
aliases: ["tholol", "tlol"],
|
|
474
|
+
description: "Kata hinaan yang menunjukkan kebodohan",
|
|
475
|
+
context: "Hinaan untuk menyebut seseorang yang dianggap bodoh",
|
|
476
|
+
},
|
|
477
|
+
{
|
|
478
|
+
word: "budheg",
|
|
479
|
+
category: "insult",
|
|
480
|
+
region: "jawa",
|
|
481
|
+
severity: 0.6,
|
|
482
|
+
aliases: ["budeg", "bdeg"],
|
|
483
|
+
description: "Secara harfiah berarti tuli atau tidak bisa mendengar",
|
|
484
|
+
context: "Hinaan untuk orang yang dianggap tidak mau mendengarkan",
|
|
485
|
+
},
|
|
486
|
+
{
|
|
487
|
+
word: "jiangkrik",
|
|
488
|
+
category: "insult",
|
|
489
|
+
region: "jawa",
|
|
490
|
+
severity: 0.4,
|
|
491
|
+
aliases: ["jiangkrek", "jangkrik"],
|
|
492
|
+
description: "Secara harfiah berarti jangkrik, digunakan sebagai eufemisme",
|
|
493
|
+
context: "Umpatan ringan sebagai pengganti kata kasar yang lebih vulgar",
|
|
494
|
+
},
|
|
495
|
+
{
|
|
496
|
+
word: "diamput",
|
|
497
|
+
category: "insult",
|
|
498
|
+
region: "jawa",
|
|
499
|
+
severity: 0.8,
|
|
500
|
+
aliases: ["damput", "djamput"],
|
|
501
|
+
description: "Bentuk umpatan kasar dengan makna serupa jancok",
|
|
502
|
+
context: "Kata kasar untuk mengekspresikan kemarahan",
|
|
503
|
+
},
|
|
504
|
+
{
|
|
505
|
+
word: "celeng",
|
|
506
|
+
category: "insult",
|
|
507
|
+
region: "jawa",
|
|
508
|
+
severity: 0.6,
|
|
509
|
+
aliases: ["cleng", "babi hutan"],
|
|
510
|
+
description: "Secara harfiah berarti babi hutan",
|
|
511
|
+
context: "Hinaan untuk orang yang dianggap jorok atau rakus",
|
|
512
|
+
},
|
|
513
|
+
{
|
|
514
|
+
word: "kampret",
|
|
515
|
+
category: "insult",
|
|
516
|
+
region: "jawa",
|
|
517
|
+
severity: 0.5,
|
|
518
|
+
aliases: ["kmpret", "kmprt"],
|
|
519
|
+
description: "Secara harfiah berarti kelelawar kecil",
|
|
520
|
+
context: "Umpatan ringan untuk mengekspresikan kekesalan",
|
|
521
|
+
},
|
|
522
|
+
{
|
|
523
|
+
word: "ndeso",
|
|
524
|
+
category: "insult",
|
|
525
|
+
region: "jawa",
|
|
526
|
+
severity: 0.4,
|
|
527
|
+
aliases: ["ndesa", "deso"],
|
|
528
|
+
description: "Secara harfiah berarti dari desa atau kampungan",
|
|
529
|
+
context: "Hinaan untuk orang yang dianggap kurang modern atau berpendidikan",
|
|
530
|
+
},
|
|
531
|
+
{
|
|
532
|
+
word: "kere",
|
|
533
|
+
category: "insult",
|
|
534
|
+
region: "jawa",
|
|
535
|
+
severity: 0.5,
|
|
536
|
+
aliases: ["miskin", "mlarat"],
|
|
537
|
+
description: "Secara harfiah berarti miskin atau tidak punya uang",
|
|
538
|
+
context: "Hinaan untuk status ekonomi seseorang yang dianggap rendah",
|
|
539
|
+
},
|
|
298
540
|
{
|
|
299
541
|
word: "itil",
|
|
300
542
|
category: "sexual",
|
|
@@ -1088,29 +1330,73 @@ function clusterSimilarWords(words, threshold = 0.8) {
|
|
|
1088
1330
|
}
|
|
1089
1331
|
/**
|
|
1090
1332
|
* Cari kata-kata kotor yang mungkin dari teks menggunakan kesamaan string
|
|
1333
|
+
* dengan optimasi untuk mengurangi kompleksitas
|
|
1091
1334
|
*
|
|
1092
1335
|
* @param text Teks yang akan diperiksa
|
|
1093
1336
|
* @param profanityWords Daftar kata kotor
|
|
1094
1337
|
* @param threshold Batas minimum kesamaan (default: 0.8)
|
|
1095
1338
|
* @returns Array kata yang mungkin merupakan kata kotor
|
|
1096
1339
|
*/
|
|
1340
|
+
/**
|
|
1341
|
+
* Cari kata-kata kotor yang mungkin dari teks berdasarkan kemiripan string
|
|
1342
|
+
* dengan optimasi biar prosesnya nggak terlalu berat
|
|
1343
|
+
*
|
|
1344
|
+
* @param text Teks yang mau dicek
|
|
1345
|
+
* @param profanityWords Daftar kata-kata kotor/kasar
|
|
1346
|
+
* @param threshold Batas minimal kemiripan (default: 0.8)
|
|
1347
|
+
* @returns Array kata yang kemungkinan kata kotor/kasar
|
|
1348
|
+
*/
|
|
1097
1349
|
function findPossibleProfanityBySimiliarity(text, profanityWords, threshold = 0.8) {
|
|
1098
1350
|
const result = [];
|
|
1351
|
+
// Optimasi dengan bikin map kata-kata kotor berdasarkan huruf pertama
|
|
1352
|
+
const profanityMap = new Map();
|
|
1353
|
+
// Kelompokkan kata-kata kotor berdasarkan huruf pertama biar pencariannya lebih cepat
|
|
1354
|
+
for (const word of profanityWords) {
|
|
1355
|
+
if (word.length < 1)
|
|
1356
|
+
continue;
|
|
1357
|
+
const firstChar = word[0].toLowerCase();
|
|
1358
|
+
if (!profanityMap.has(firstChar)) {
|
|
1359
|
+
profanityMap.set(firstChar, []);
|
|
1360
|
+
}
|
|
1361
|
+
profanityMap.get(firstChar).push(word);
|
|
1362
|
+
}
|
|
1099
1363
|
const words = text.toLowerCase().split(/\s+/);
|
|
1100
1364
|
for (const word of words) {
|
|
1101
1365
|
if (word.length < 3)
|
|
1102
1366
|
continue;
|
|
1103
|
-
|
|
1367
|
+
// Cuma bandingin dengan kata-kata kotor yang huruf pertamanya sama
|
|
1368
|
+
// atau yang perbedaan panjangnya masih masuk akal
|
|
1369
|
+
const firstChar = word[0];
|
|
1370
|
+
const candidateWords = profanityMap.get(firstChar) || [];
|
|
1371
|
+
// Cek juga huruf-huruf yang berdekatan (buat antisipasi typo di huruf pertama)
|
|
1372
|
+
// Ini opsional tapi bikin deteksinya lebih bagus
|
|
1373
|
+
const charCode = firstChar.charCodeAt(0);
|
|
1374
|
+
const prevChar = String.fromCharCode(charCode - 1);
|
|
1375
|
+
const nextChar = String.fromCharCode(charCode + 1);
|
|
1376
|
+
const adjacentCandidates = [
|
|
1377
|
+
...(profanityMap.get(prevChar) || []),
|
|
1378
|
+
...(profanityMap.get(nextChar) || []),
|
|
1379
|
+
];
|
|
1380
|
+
// Gabungin kandidat-kandidatnya, prioritasin yang huruf pertamanya sama persis
|
|
1381
|
+
const allCandidates = [...candidateWords, ...adjacentCandidates];
|
|
1382
|
+
// Filter kandidat berdasarkan perbedaan panjang sebelum ngitung kemiripannya
|
|
1383
|
+
const lengthFilteredCandidates = allCandidates.filter((candidate) => Math.abs(candidate.length - word.length) <= 2);
|
|
1384
|
+
// Cari yang paling cocok
|
|
1385
|
+
let bestMatch = null;
|
|
1386
|
+
for (const profanity of lengthFilteredCandidates) {
|
|
1104
1387
|
const similarity = stringSimilarity(word, profanity);
|
|
1105
|
-
if (similarity >= threshold
|
|
1106
|
-
|
|
1388
|
+
if (similarity >= threshold &&
|
|
1389
|
+
(!bestMatch || similarity > bestMatch.similarity)) {
|
|
1390
|
+
bestMatch = {
|
|
1107
1391
|
word,
|
|
1108
1392
|
original: profanity,
|
|
1109
1393
|
similarity,
|
|
1110
|
-
}
|
|
1111
|
-
break;
|
|
1394
|
+
};
|
|
1112
1395
|
}
|
|
1113
1396
|
}
|
|
1397
|
+
if (bestMatch) {
|
|
1398
|
+
result.push(bestMatch);
|
|
1399
|
+
}
|
|
1114
1400
|
}
|
|
1115
1401
|
return result;
|
|
1116
1402
|
}
|
|
@@ -1125,30 +1411,75 @@ function findPossibleProfanityBySimiliarity(text, profanityWords, threshold = 0.
|
|
|
1125
1411
|
*/
|
|
1126
1412
|
function findProfanityByLevenshteinDistance(text, profanityWords, threshold = 0.8, maxDistance = 2) {
|
|
1127
1413
|
const result = [];
|
|
1414
|
+
// map kata-kata kotor dikelompokkan sesuai panjangnya
|
|
1415
|
+
const profanityByLength = new Map();
|
|
1416
|
+
// Kelompokkin kata-kata kotor berdasarkan panjangnya biar pencarian lebih cepat
|
|
1417
|
+
for (const word of profanityWords) {
|
|
1418
|
+
const length = word.length;
|
|
1419
|
+
if (!profanityByLength.has(length)) {
|
|
1420
|
+
profanityByLength.set(length, []);
|
|
1421
|
+
}
|
|
1422
|
+
profanityByLength.get(length).push(word);
|
|
1423
|
+
}
|
|
1128
1424
|
const words = text.toLowerCase().split(/\s+/);
|
|
1129
1425
|
for (const word of words) {
|
|
1130
1426
|
if (word.length < 3)
|
|
1131
1427
|
continue;
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
const
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1428
|
+
let bestMatch = null;
|
|
1429
|
+
for (let len = Math.max(3, word.length - maxDistance); len <= word.length + maxDistance; len++) {
|
|
1430
|
+
const candidates = profanityByLength.get(len) || [];
|
|
1431
|
+
for (const profanity of candidates) {
|
|
1432
|
+
if (!isCharacterCountSimilar(word, profanity, maxDistance)) {
|
|
1433
|
+
continue;
|
|
1434
|
+
}
|
|
1435
|
+
const distance = levenshteinDistance(word, profanity);
|
|
1436
|
+
if (distance <= maxDistance) {
|
|
1437
|
+
const similarity = 1 - distance / Math.max(word.length, profanity.length);
|
|
1438
|
+
if (similarity >= threshold &&
|
|
1439
|
+
(!bestMatch || similarity > bestMatch.similarity)) {
|
|
1440
|
+
bestMatch = {
|
|
1441
|
+
word,
|
|
1442
|
+
original: profanity,
|
|
1443
|
+
similarity,
|
|
1444
|
+
distance,
|
|
1445
|
+
};
|
|
1446
|
+
if (distance === 0 || similarity > 0.95) {
|
|
1447
|
+
break;
|
|
1448
|
+
}
|
|
1449
|
+
}
|
|
1146
1450
|
}
|
|
1147
1451
|
}
|
|
1148
1452
|
}
|
|
1453
|
+
if (bestMatch) {
|
|
1454
|
+
result.push(bestMatch);
|
|
1455
|
+
}
|
|
1149
1456
|
}
|
|
1150
1457
|
return result;
|
|
1151
1458
|
}
|
|
1459
|
+
/**
|
|
1460
|
+
* Helper function to efficiently check if character counts between two strings
|
|
1461
|
+
* are similar enough to warrant a full Levenshtein calculation
|
|
1462
|
+
*/
|
|
1463
|
+
function isCharacterCountSimilar(str1, str2, maxDifference) {
|
|
1464
|
+
const charCount1 = {};
|
|
1465
|
+
const charCount2 = {};
|
|
1466
|
+
for (const char of str1) {
|
|
1467
|
+
charCount1[char] = (charCount1[char] || 0) + 1;
|
|
1468
|
+
}
|
|
1469
|
+
for (const char of str2) {
|
|
1470
|
+
charCount2[char] = (charCount2[char] || 0) + 1;
|
|
1471
|
+
}
|
|
1472
|
+
let diffCount = 0;
|
|
1473
|
+
for (const char in charCount1) {
|
|
1474
|
+
diffCount += Math.abs((charCount1[char] || 0) - (charCount2[char] || 0));
|
|
1475
|
+
}
|
|
1476
|
+
for (const char in charCount2) {
|
|
1477
|
+
if (!charCount1[char]) {
|
|
1478
|
+
diffCount += charCount2[char];
|
|
1479
|
+
}
|
|
1480
|
+
}
|
|
1481
|
+
return diffCount <= maxDifference * 2;
|
|
1482
|
+
}
|
|
1152
1483
|
|
|
1153
1484
|
const DEFAULT_OPTIONS = {
|
|
1154
1485
|
replaceWith: "*",
|
|
@@ -1303,6 +1634,157 @@ function makeRandomGrawlixString(length) {
|
|
|
1303
1634
|
return result;
|
|
1304
1635
|
}
|
|
1305
1636
|
|
|
1637
|
+
/**
|
|
1638
|
+
* Implementasi algoritma Aho-Corasick untuk pencocokan string
|
|
1639
|
+
* Algoritma ini memungkinkan pencarian banyak pola dalam satu kali pembacaan teks
|
|
1640
|
+
*/
|
|
1641
|
+
class AhoCorasick {
|
|
1642
|
+
constructor() {
|
|
1643
|
+
this.built = false;
|
|
1644
|
+
this.root = {
|
|
1645
|
+
children: new Map(),
|
|
1646
|
+
fail: null,
|
|
1647
|
+
output: new Set(),
|
|
1648
|
+
depth: 0,
|
|
1649
|
+
};
|
|
1650
|
+
}
|
|
1651
|
+
/**
|
|
1652
|
+
* Menambahkan pola ke dalam trie
|
|
1653
|
+
* @param pattern Pola yang akan ditambahkan
|
|
1654
|
+
*/
|
|
1655
|
+
addPattern(pattern) {
|
|
1656
|
+
if (this.built) {
|
|
1657
|
+
throw new Error("Cannot add patterns after the automaton is built");
|
|
1658
|
+
}
|
|
1659
|
+
let node = this.root;
|
|
1660
|
+
const normalizedPattern = pattern.toLowerCase();
|
|
1661
|
+
for (let i = 0; i < normalizedPattern.length; i++) {
|
|
1662
|
+
const char = normalizedPattern[i];
|
|
1663
|
+
if (!node.children.has(char)) {
|
|
1664
|
+
node.children.set(char, {
|
|
1665
|
+
children: new Map(),
|
|
1666
|
+
fail: null,
|
|
1667
|
+
output: new Set(),
|
|
1668
|
+
depth: node.depth + 1,
|
|
1669
|
+
char,
|
|
1670
|
+
});
|
|
1671
|
+
}
|
|
1672
|
+
node = node.children.get(char);
|
|
1673
|
+
}
|
|
1674
|
+
node.output.add(normalizedPattern);
|
|
1675
|
+
}
|
|
1676
|
+
/**
|
|
1677
|
+
* Membangun fungsi failure
|
|
1678
|
+
*/
|
|
1679
|
+
build() {
|
|
1680
|
+
if (this.built)
|
|
1681
|
+
return;
|
|
1682
|
+
const queue = [];
|
|
1683
|
+
// Set fail pointer for depth 1 nodes to root
|
|
1684
|
+
for (const child of this.root.children.values()) {
|
|
1685
|
+
child.fail = this.root;
|
|
1686
|
+
queue.push(child);
|
|
1687
|
+
}
|
|
1688
|
+
// BFS to build failure links
|
|
1689
|
+
while (queue.length > 0) {
|
|
1690
|
+
const current = queue.shift();
|
|
1691
|
+
for (const [char, child] of current.children.entries()) {
|
|
1692
|
+
queue.push(child);
|
|
1693
|
+
let failNode = current.fail;
|
|
1694
|
+
// Find the longest proper suffix that is also a prefix
|
|
1695
|
+
while (failNode !== null && !failNode.children.has(char)) {
|
|
1696
|
+
failNode = failNode.fail;
|
|
1697
|
+
}
|
|
1698
|
+
if (failNode === null) {
|
|
1699
|
+
child.fail = this.root;
|
|
1700
|
+
}
|
|
1701
|
+
else {
|
|
1702
|
+
child.fail = failNode.children.get(char);
|
|
1703
|
+
// Add outputs from the fail state to this node
|
|
1704
|
+
for (const output of child.fail.output) {
|
|
1705
|
+
child.output.add(output);
|
|
1706
|
+
}
|
|
1707
|
+
}
|
|
1708
|
+
}
|
|
1709
|
+
}
|
|
1710
|
+
this.built = true;
|
|
1711
|
+
}
|
|
1712
|
+
/**
|
|
1713
|
+
* Mencari semua kemunculan pola dalam teks
|
|
1714
|
+
* @param text Teks yang akan dicari
|
|
1715
|
+
* @returns Map pola yang ditemukan dengan jumlah kemunculannya
|
|
1716
|
+
*/
|
|
1717
|
+
search(text) {
|
|
1718
|
+
if (!this.built) {
|
|
1719
|
+
this.build();
|
|
1720
|
+
}
|
|
1721
|
+
const matches = new Map();
|
|
1722
|
+
const normalizedText = text.toLowerCase();
|
|
1723
|
+
let node = this.root;
|
|
1724
|
+
for (let i = 0; i < normalizedText.length; i++) {
|
|
1725
|
+
const char = normalizedText[i];
|
|
1726
|
+
// Follow failure links until we find a matching transition or reach root
|
|
1727
|
+
while (node !== this.root && !node.children.has(char)) {
|
|
1728
|
+
node = node.fail;
|
|
1729
|
+
}
|
|
1730
|
+
// Try to follow the transition
|
|
1731
|
+
if (node.children.has(char)) {
|
|
1732
|
+
node = node.children.get(char);
|
|
1733
|
+
}
|
|
1734
|
+
// Check for any matches at this node
|
|
1735
|
+
for (const match of node.output) {
|
|
1736
|
+
matches.set(match, (matches.get(match) || 0) + 1);
|
|
1737
|
+
}
|
|
1738
|
+
}
|
|
1739
|
+
return matches;
|
|
1740
|
+
}
|
|
1741
|
+
/**
|
|
1742
|
+
* Mencari semua kemunculan pola dalam teks dan mengembalikan hanya pola unik
|
|
1743
|
+
* @param text Teks yang akan dicari
|
|
1744
|
+
* @returns Set pola yang ditemukan
|
|
1745
|
+
*/
|
|
1746
|
+
searchUnique(text) {
|
|
1747
|
+
const matches = this.search(text);
|
|
1748
|
+
return new Set(matches.keys());
|
|
1749
|
+
}
|
|
1750
|
+
/**
|
|
1751
|
+
* Mengecek apakah teks mengandung setidaknya satu pola
|
|
1752
|
+
* @param text Teks yang akan dicari
|
|
1753
|
+
* @returns Boolean apakah pola ditemukan
|
|
1754
|
+
*/
|
|
1755
|
+
containsAny(text) {
|
|
1756
|
+
if (!this.built) {
|
|
1757
|
+
this.build();
|
|
1758
|
+
}
|
|
1759
|
+
const normalizedText = text.toLowerCase();
|
|
1760
|
+
let node = this.root;
|
|
1761
|
+
for (let i = 0; i < normalizedText.length; i++) {
|
|
1762
|
+
const char = normalizedText[i];
|
|
1763
|
+
while (node !== this.root && !node.children.has(char)) {
|
|
1764
|
+
node = node.fail;
|
|
1765
|
+
}
|
|
1766
|
+
if (node.children.has(char)) {
|
|
1767
|
+
node = node.children.get(char);
|
|
1768
|
+
}
|
|
1769
|
+
if (node.output.size > 0) {
|
|
1770
|
+
return true;
|
|
1771
|
+
}
|
|
1772
|
+
}
|
|
1773
|
+
return false;
|
|
1774
|
+
}
|
|
1775
|
+
}
|
|
1776
|
+
|
|
1777
|
+
const globalAhoCorasick = new AhoCorasick();
|
|
1778
|
+
let ahoCorasickInitialized = false;
|
|
1779
|
+
function initializeAhoCorasick(words) {
|
|
1780
|
+
if (ahoCorasickInitialized)
|
|
1781
|
+
return;
|
|
1782
|
+
for (const word of words) {
|
|
1783
|
+
globalAhoCorasick.addPattern(word);
|
|
1784
|
+
}
|
|
1785
|
+
globalAhoCorasick.build();
|
|
1786
|
+
ahoCorasickInitialized = true;
|
|
1787
|
+
}
|
|
1306
1788
|
function findProfanity(text, options = {}) {
|
|
1307
1789
|
const { wordList = [], detectLeetSpeak = true, checkSubstring = false, whitelist = [], categories, regions, severityThreshold = 0, indonesianVariation = false, detectSimilarity = false, similarityThreshold = 0.8, detectSplit = false, useLevenshtein = false, maxLevenshteinDistance = 2, } = { ...DEFAULT_OPTIONS, ...options };
|
|
1308
1790
|
const normalizedText = normalizeText(text);
|
|
@@ -1342,24 +1824,16 @@ function findProfanity(text, options = {}) {
|
|
|
1342
1824
|
}
|
|
1343
1825
|
const matches = new Set();
|
|
1344
1826
|
const actualMatches = new Map();
|
|
1345
|
-
wordsToCheck
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
});
|
|
1353
|
-
let match;
|
|
1354
|
-
while ((match = regex.exec(normalizedText)) !== null) {
|
|
1355
|
-
const originalWord = aliasMap.get(word.toLowerCase()) || word.toLowerCase();
|
|
1356
|
-
matches.add(originalWord);
|
|
1357
|
-
if (!actualMatches.has(originalWord)) {
|
|
1358
|
-
actualMatches.set(originalWord, []);
|
|
1359
|
-
}
|
|
1360
|
-
actualMatches.get(originalWord)?.push(match[0]);
|
|
1827
|
+
initializeAhoCorasick(wordsToCheck);
|
|
1828
|
+
const basicMatches = globalAhoCorasick.searchUnique(normalizedText);
|
|
1829
|
+
for (const match of basicMatches) {
|
|
1830
|
+
const originalWord = aliasMap.get(match.toLowerCase()) || match.toLowerCase();
|
|
1831
|
+
matches.add(originalWord);
|
|
1832
|
+
if (!actualMatches.has(originalWord)) {
|
|
1833
|
+
actualMatches.set(originalWord, []);
|
|
1361
1834
|
}
|
|
1362
|
-
|
|
1835
|
+
actualMatches.get(originalWord)?.push(match);
|
|
1836
|
+
}
|
|
1363
1837
|
if (detectLeetSpeak) {
|
|
1364
1838
|
wordsToCheck.forEach((word) => {
|
|
1365
1839
|
const leetRegex = createWordRegex(word, {
|
|
@@ -1536,7 +2010,7 @@ function calculateSeverity(matchDetails) {
|
|
|
1536
2010
|
* @returns FilterResult dengan hasil filter
|
|
1537
2011
|
*/
|
|
1538
2012
|
function filter(text, options = {}) {
|
|
1539
|
-
const { replaceWith =
|
|
2013
|
+
const { replaceWith = "*", fullWordCensor = true, detectLeetSpeak = true, whitelist = [], checkSubstring = false, useRandomGrawlix = false, keepFirstAndLast = false, indonesianVariation = false, detectSplit = false, detectSimilarity = false, useLevenshtein = false, maxLevenshteinDistance = 2, similarityThreshold = 0.8, } = { ...DEFAULT_OPTIONS, ...options };
|
|
1540
2014
|
const matches = findProfanity(text, {
|
|
1541
2015
|
...options,
|
|
1542
2016
|
detectLeetSpeak,
|
|
@@ -1568,7 +2042,7 @@ function filter(text, options = {}) {
|
|
|
1568
2042
|
variants.push(word);
|
|
1569
2043
|
const uniqueVariants = [...new Set(variants)];
|
|
1570
2044
|
uniqueVariants.forEach((variant) => {
|
|
1571
|
-
const regex = new RegExp(`\\b${escapeRegExp(variant)}\\b`,
|
|
2045
|
+
const regex = new RegExp(`\\b${escapeRegExp(variant)}\\b`, "gi");
|
|
1572
2046
|
let match;
|
|
1573
2047
|
while ((match = regex.exec(filteredText)) !== null) {
|
|
1574
2048
|
const originalWord = match[0];
|
|
@@ -1586,7 +2060,7 @@ function filter(text, options = {}) {
|
|
|
1586
2060
|
censored: censoredWord,
|
|
1587
2061
|
metadata,
|
|
1588
2062
|
});
|
|
1589
|
-
filteredText = filteredText.replace(new RegExp(`\\b${escapeRegExp(originalWord)}\\b`,
|
|
2063
|
+
filteredText = filteredText.replace(new RegExp(`\\b${escapeRegExp(originalWord)}\\b`, "g"), censoredWord);
|
|
1590
2064
|
}
|
|
1591
2065
|
});
|
|
1592
2066
|
if (detectSplit || detectLeetSpeak) {
|
|
@@ -1615,7 +2089,7 @@ function filter(text, options = {}) {
|
|
|
1615
2089
|
censored: censoredWord,
|
|
1616
2090
|
metadata,
|
|
1617
2091
|
});
|
|
1618
|
-
filteredText = filteredText.replace(new RegExp(escapeRegExp(originalWord),
|
|
2092
|
+
filteredText = filteredText.replace(new RegExp(escapeRegExp(originalWord), "g"), censoredWord);
|
|
1619
2093
|
}
|
|
1620
2094
|
}
|
|
1621
2095
|
if (detectSplit) {
|
|
@@ -1643,7 +2117,7 @@ function filter(text, options = {}) {
|
|
|
1643
2117
|
censored: censoredWord,
|
|
1644
2118
|
metadata,
|
|
1645
2119
|
});
|
|
1646
|
-
filteredText = filteredText.replace(new RegExp(escapeRegExp(originalWord),
|
|
2120
|
+
filteredText = filteredText.replace(new RegExp(escapeRegExp(originalWord), "g"), censoredWord);
|
|
1647
2121
|
}
|
|
1648
2122
|
}
|
|
1649
2123
|
}
|
|
@@ -1655,7 +2129,7 @@ function filter(text, options = {}) {
|
|
|
1655
2129
|
m.aliases.some((alias) => alias.toLowerCase() === word.toLowerCase())));
|
|
1656
2130
|
const variants = actualMatches.get(word.toLowerCase()) || [];
|
|
1657
2131
|
variants.forEach((variant) => {
|
|
1658
|
-
const exactVariantRegex = new RegExp(`\\b${escapeRegExp(variant)}\\b`,
|
|
2132
|
+
const exactVariantRegex = new RegExp(`\\b${escapeRegExp(variant)}\\b`, "gi");
|
|
1659
2133
|
let match;
|
|
1660
2134
|
while ((match = exactVariantRegex.exec(filteredText)) !== null) {
|
|
1661
2135
|
const originalWord = match[0];
|
|
@@ -1673,7 +2147,7 @@ function filter(text, options = {}) {
|
|
|
1673
2147
|
censored: censoredWord,
|
|
1674
2148
|
metadata,
|
|
1675
2149
|
});
|
|
1676
|
-
filteredText = filteredText.replace(new RegExp(`\\b${escapeRegExp(originalWord)}\\b`,
|
|
2150
|
+
filteredText = filteredText.replace(new RegExp(`\\b${escapeRegExp(originalWord)}\\b`, "g"), censoredWord);
|
|
1677
2151
|
}
|
|
1678
2152
|
});
|
|
1679
2153
|
});
|
|
@@ -1722,8 +2196,20 @@ function analyze(text, options = {}) {
|
|
|
1722
2196
|
const severityScore = calculateSeverity(matchDetails);
|
|
1723
2197
|
let similarWords = [];
|
|
1724
2198
|
if (mergedOptions.detectSimilarity) {
|
|
1725
|
-
|
|
1726
|
-
|
|
2199
|
+
if (matchDetails.length > 0) {
|
|
2200
|
+
const wordList = matchDetails.map((word) => word.word);
|
|
2201
|
+
if (mergedOptions.useLevenshtein) {
|
|
2202
|
+
const levenshteinResults = findProfanityByLevenshteinDistance(text, wordList, mergedOptions.similarityThreshold || 0.8, mergedOptions.maxLevenshteinDistance || 2);
|
|
2203
|
+
similarWords = levenshteinResults.map((item) => ({
|
|
2204
|
+
word: item.word,
|
|
2205
|
+
original: item.original,
|
|
2206
|
+
similarity: item.similarity,
|
|
2207
|
+
}));
|
|
2208
|
+
}
|
|
2209
|
+
else {
|
|
2210
|
+
similarWords = findPossibleProfanityBySimiliarity(text, wordList, mergedOptions.similarityThreshold || 0.8);
|
|
2211
|
+
}
|
|
2212
|
+
}
|
|
1727
2213
|
}
|
|
1728
2214
|
return {
|
|
1729
2215
|
hasProfanity: true,
|
|
@@ -1820,9 +2306,9 @@ function analyzeWithContext(text, contextWindowSize = 5, options = {}) {
|
|
|
1820
2306
|
const regex = createContextRegex(word, contextWindowSize);
|
|
1821
2307
|
let match;
|
|
1822
2308
|
while ((match = regex.exec(text)) !== null) {
|
|
1823
|
-
const beforeContext = match[1] ||
|
|
2309
|
+
const beforeContext = match[1] || "";
|
|
1824
2310
|
const wordMatch = match[2];
|
|
1825
|
-
const afterContext = match[3] ||
|
|
2311
|
+
const afterContext = match[3] || "";
|
|
1826
2312
|
result.push({
|
|
1827
2313
|
word: wordMatch,
|
|
1828
2314
|
context: beforeContext + wordMatch + afterContext,
|