natural 0.5.4 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.dockerignore +2 -0
- package/.npmignore +1 -0
- package/.travis.yml +1 -1
- package/README.md +412 -89
- package/bayes_classifier.json +1 -1
- package/classifier.json +418 -0
- package/data/kata-dasar.txt +29933 -0
- package/lib/natural/brill_pos_tagger/lib/Brill_POS_Tagger.js +8 -9
- package/lib/natural/brill_pos_tagger/lib/Brill_POS_Tester.js +5 -5
- package/lib/natural/brill_pos_tagger/lib/Brill_POS_Trainer.js +10 -6
- package/lib/natural/brill_pos_tagger/lib/Corpus.js +21 -16
- package/lib/natural/brill_pos_tagger/lib/Lexicon.js +22 -13
- package/lib/natural/brill_pos_tagger/lib/Predicate.js +2 -2
- package/lib/natural/brill_pos_tagger/lib/RuleTemplate.js +2 -2
- package/lib/natural/brill_pos_tagger/lib/RuleTemplates.js +132 -146
- package/lib/natural/brill_pos_tagger/lib/Sentence.js +41 -0
- package/lib/natural/brill_pos_tagger/lib/TF_Parser.js +83 -76
- package/lib/natural/brill_pos_tagger/lib/TF_Parser.pegjs +19 -4
- package/lib/natural/brill_pos_tagger/lib/TransformationRule.js +10 -11
- package/lib/natural/classifiers/classifier.js +15 -8
- package/lib/natural/classifiers/maxent/Classifier.js +125 -0
- package/lib/natural/classifiers/maxent/Context.js +33 -0
- package/lib/natural/classifiers/maxent/Distribution.js +145 -0
- package/lib/natural/classifiers/maxent/Element.js +34 -0
- package/lib/natural/classifiers/maxent/Feature.js +88 -0
- package/lib/natural/classifiers/maxent/FeatureSet.js +67 -0
- package/lib/natural/classifiers/maxent/GISScaler.js +148 -0
- package/lib/natural/classifiers/maxent/POS/ME_Corpus.js +56 -0
- package/lib/natural/classifiers/maxent/POS/ME_Sentence.js +73 -0
- package/lib/natural/classifiers/maxent/POS/POS_Element.js +172 -0
- package/lib/natural/classifiers/maxent/Sample.js +141 -0
- package/lib/natural/classifiers/maxent/SimpleExample/SE_Element.js +49 -0
- package/lib/natural/distance/hamming_distance.js +52 -0
- package/lib/natural/distance/jaro-winkler_distance.js +16 -6
- package/lib/natural/distance/levenshtein_distance.js +164 -11
- package/lib/natural/index.js +26 -5
- package/lib/natural/inflectors/noun_inflector.js +15 -13
- package/lib/natural/ngrams/ngrams.js +83 -25
- package/lib/natural/normalizers/normalizer_sv.js +41 -0
- package/lib/natural/sentiment/Basque/senticon_eu.json +21557 -0
- package/lib/natural/sentiment/Catalan/senticon_ca.json +36352 -0
- package/lib/natural/sentiment/Dutch/negations_du.json +3 -0
- package/lib/natural/sentiment/Dutch/pattern-sentiment-nl.json +39650 -0
- package/lib/natural/sentiment/English/negations_en.json +3 -0
- package/lib/natural/sentiment/English/pattern-sentiment-en.json +16024 -0
- package/lib/natural/sentiment/English/senticon_en.json +124197 -0
- package/lib/natural/sentiment/French/pattern-sentiment-fr.json +40906 -0
- package/lib/natural/sentiment/Galician/senticon_gl.json +24427 -0
- package/lib/natural/sentiment/Italian/pattern-sentiment-it.json +24522 -0
- package/lib/natural/sentiment/SentimentAnalyzer.js +116 -0
- package/lib/natural/sentiment/Spanish/afinnShortSortedSpanish.json +1655 -0
- package/lib/natural/sentiment/Spanish/negations_es.json +3 -0
- package/lib/natural/sentiment/Spanish/senticon_es.json +56722 -0
- package/lib/natural/sentiment/tools/README.md +31 -0
- package/lib/natural/sentiment/tools/XmlParser4PatternData.js +31 -0
- package/lib/natural/sentiment/tools/sentimentXmlParser.js +30 -0
- package/lib/natural/stemmers/indonesian/base_stemmer_id.js +86 -0
- package/lib/natural/stemmers/indonesian/prefix_rules.js +1027 -0
- package/lib/natural/stemmers/indonesian/removal.js +49 -0
- package/lib/natural/stemmers/indonesian/stemmer_id.js +307 -0
- package/lib/natural/stemmers/indonesian/suffix_rules.js +75 -0
- package/lib/natural/stemmers/porter_stemmer_es.js +265 -199
- package/lib/natural/stemmers/porter_stemmer_fr.js +1 -1
- package/lib/natural/stemmers/porter_stemmer_nl.js +362 -0
- package/lib/natural/stemmers/porter_stemmer_sv.js +99 -0
- package/lib/natural/stemmers/stemmer_es.js +18 -12
- package/lib/natural/stemmers/stemmer_nl.js +64 -0
- package/lib/natural/stemmers/stemmer_sv.js +61 -0
- package/lib/natural/tfidf/tfidf.js +27 -4
- package/lib/natural/tokenizers/aggressive_tokenizer_id.js +44 -0
- package/lib/natural/tokenizers/aggressive_tokenizer_sv.js +40 -0
- package/lib/natural/tokenizers/aggressive_tokenizer_vi.js +37 -0
- package/lib/natural/tokenizers/orthography_matchers.js +33 -0
- package/lib/natural/tokenizers/regexp_tokenizer.js +23 -3
- package/lib/natural/tokenizers/sentence_tokenizer.js +1 -1
- package/lib/natural/util/stopwords_id.js +119 -0
- package/lib/natural/util/stopwords_no.js +1 -1
- package/lib/natural/util/stopwords_sv.js +70 -0
- package/lib/natural/wordnet/wordnet_file.js +7 -1
- package/lr_classifier.json +1 -1
- package/package-lock.json +378 -0
- package/package.json +29 -21
|
@@ -43,40 +43,51 @@ module.exports = (function() {
|
|
|
43
43
|
}
|
|
44
44
|
return(result);
|
|
45
45
|
},
|
|
46
|
-
peg$c3 =
|
|
47
|
-
|
|
46
|
+
peg$c3 = function(c1, c2, pred, pars) {
|
|
47
|
+
var result = null;
|
|
48
|
+
|
|
48
49
|
// Construct rule
|
|
49
|
-
|
|
50
|
+
if (pars.length === 1) {
|
|
51
|
+
result = new TransformationRule(c1, c2, pred, pars[0]);
|
|
52
|
+
}
|
|
53
|
+
else {
|
|
54
|
+
if (pars.length === 2) {
|
|
55
|
+
result = new TransformationRule(c1, c2, pred, pars[0], pars[1]);
|
|
56
|
+
}
|
|
57
|
+
else {
|
|
58
|
+
result = new TransformationRule(c1, c2, pred);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
50
61
|
return(result);
|
|
51
62
|
},
|
|
52
|
-
peg$
|
|
53
|
-
peg$
|
|
54
|
-
peg$
|
|
63
|
+
peg$c4 = /^[!-~\xA1-\xFF]/,
|
|
64
|
+
peg$c5 = { type: "class", value: "[!-~\\xA1-\\xFF]", description: "[!-~\\xA1-\\xFF]" },
|
|
65
|
+
peg$c6 = function(characters) {
|
|
55
66
|
var s = "";
|
|
56
67
|
for (var i = 0; i < characters.length; i++) {
|
|
57
68
|
s += characters[i];
|
|
58
69
|
}
|
|
59
70
|
return(s);
|
|
60
71
|
},
|
|
61
|
-
peg$
|
|
62
|
-
peg$
|
|
63
|
-
peg$
|
|
72
|
+
peg$c7 = "*",
|
|
73
|
+
peg$c8 = { type: "literal", value: "*", description: "\"*\"" },
|
|
74
|
+
peg$c9 = function(wc) {
|
|
64
75
|
return(wc)
|
|
65
76
|
},
|
|
66
|
-
peg$
|
|
67
|
-
peg$
|
|
68
|
-
peg$
|
|
69
|
-
peg$
|
|
70
|
-
peg$
|
|
71
|
-
peg$
|
|
72
|
-
peg$
|
|
73
|
-
peg$
|
|
74
|
-
peg$
|
|
75
|
-
peg$
|
|
76
|
-
peg$
|
|
77
|
-
peg$
|
|
78
|
-
peg$
|
|
79
|
-
peg$
|
|
77
|
+
peg$c10 = "\r\n",
|
|
78
|
+
peg$c11 = { type: "literal", value: "\r\n", description: "\"\\r\\n\"" },
|
|
79
|
+
peg$c12 = "\n",
|
|
80
|
+
peg$c13 = { type: "literal", value: "\n", description: "\"\\n\"" },
|
|
81
|
+
peg$c14 = "\r",
|
|
82
|
+
peg$c15 = { type: "literal", value: "\r", description: "\"\\r\"" },
|
|
83
|
+
peg$c16 = "//",
|
|
84
|
+
peg$c17 = { type: "literal", value: "//", description: "\"//\"" },
|
|
85
|
+
peg$c18 = void 0,
|
|
86
|
+
peg$c19 = { type: "any", description: "any character" },
|
|
87
|
+
peg$c20 = " ",
|
|
88
|
+
peg$c21 = { type: "literal", value: " ", description: "\" \"" },
|
|
89
|
+
peg$c22 = "\t",
|
|
90
|
+
peg$c23 = { type: "literal", value: "\t", description: "\"\\t\"" },
|
|
80
91
|
|
|
81
92
|
peg$currPos = 0,
|
|
82
93
|
peg$reportedPos = 0,
|
|
@@ -317,20 +328,16 @@ module.exports = (function() {
|
|
|
317
328
|
if (s2 !== peg$FAILED) {
|
|
318
329
|
s3 = peg$parseidentifier();
|
|
319
330
|
if (s3 !== peg$FAILED) {
|
|
320
|
-
s4 =
|
|
321
|
-
|
|
331
|
+
s4 = [];
|
|
332
|
+
s5 = peg$parseidentifier();
|
|
333
|
+
while (s5 !== peg$FAILED) {
|
|
334
|
+
s4.push(s5);
|
|
322
335
|
s5 = peg$parseidentifier();
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
s1 = peg$c4(s1, s2, s3, s4, s5);
|
|
329
|
-
s0 = s1;
|
|
330
|
-
} else {
|
|
331
|
-
peg$currPos = s0;
|
|
332
|
-
s0 = peg$c1;
|
|
333
|
-
}
|
|
336
|
+
}
|
|
337
|
+
if (s4 !== peg$FAILED) {
|
|
338
|
+
peg$reportedPos = s0;
|
|
339
|
+
s1 = peg$c3(s1, s2, s3, s4);
|
|
340
|
+
s0 = s1;
|
|
334
341
|
} else {
|
|
335
342
|
peg$currPos = s0;
|
|
336
343
|
s0 = peg$c1;
|
|
@@ -367,22 +374,22 @@ module.exports = (function() {
|
|
|
367
374
|
|
|
368
375
|
s0 = peg$currPos;
|
|
369
376
|
s1 = [];
|
|
370
|
-
if (peg$
|
|
377
|
+
if (peg$c4.test(input.charAt(peg$currPos))) {
|
|
371
378
|
s2 = input.charAt(peg$currPos);
|
|
372
379
|
peg$currPos++;
|
|
373
380
|
} else {
|
|
374
381
|
s2 = peg$FAILED;
|
|
375
|
-
if (peg$silentFails === 0) { peg$fail(peg$
|
|
382
|
+
if (peg$silentFails === 0) { peg$fail(peg$c5); }
|
|
376
383
|
}
|
|
377
384
|
if (s2 !== peg$FAILED) {
|
|
378
385
|
while (s2 !== peg$FAILED) {
|
|
379
386
|
s1.push(s2);
|
|
380
|
-
if (peg$
|
|
387
|
+
if (peg$c4.test(input.charAt(peg$currPos))) {
|
|
381
388
|
s2 = input.charAt(peg$currPos);
|
|
382
389
|
peg$currPos++;
|
|
383
390
|
} else {
|
|
384
391
|
s2 = peg$FAILED;
|
|
385
|
-
if (peg$silentFails === 0) { peg$fail(peg$
|
|
392
|
+
if (peg$silentFails === 0) { peg$fail(peg$c5); }
|
|
386
393
|
}
|
|
387
394
|
}
|
|
388
395
|
} else {
|
|
@@ -392,7 +399,7 @@ module.exports = (function() {
|
|
|
392
399
|
s2 = peg$parseS_no_eol();
|
|
393
400
|
if (s2 !== peg$FAILED) {
|
|
394
401
|
peg$reportedPos = s0;
|
|
395
|
-
s1 = peg$
|
|
402
|
+
s1 = peg$c6(s1);
|
|
396
403
|
s0 = s1;
|
|
397
404
|
} else {
|
|
398
405
|
peg$currPos = s0;
|
|
@@ -411,17 +418,17 @@ module.exports = (function() {
|
|
|
411
418
|
|
|
412
419
|
s0 = peg$currPos;
|
|
413
420
|
if (input.charCodeAt(peg$currPos) === 42) {
|
|
414
|
-
s1 = peg$
|
|
421
|
+
s1 = peg$c7;
|
|
415
422
|
peg$currPos++;
|
|
416
423
|
} else {
|
|
417
424
|
s1 = peg$FAILED;
|
|
418
|
-
if (peg$silentFails === 0) { peg$fail(peg$
|
|
425
|
+
if (peg$silentFails === 0) { peg$fail(peg$c8); }
|
|
419
426
|
}
|
|
420
427
|
if (s1 !== peg$FAILED) {
|
|
421
428
|
s2 = peg$parseS_no_eol();
|
|
422
429
|
if (s2 !== peg$FAILED) {
|
|
423
430
|
peg$reportedPos = s0;
|
|
424
|
-
s1 = peg$
|
|
431
|
+
s1 = peg$c9(s1);
|
|
425
432
|
s0 = s1;
|
|
426
433
|
} else {
|
|
427
434
|
peg$currPos = s0;
|
|
@@ -438,28 +445,28 @@ module.exports = (function() {
|
|
|
438
445
|
function peg$parseEOL() {
|
|
439
446
|
var s0;
|
|
440
447
|
|
|
441
|
-
if (input.substr(peg$currPos, 2) === peg$
|
|
442
|
-
s0 = peg$
|
|
448
|
+
if (input.substr(peg$currPos, 2) === peg$c10) {
|
|
449
|
+
s0 = peg$c10;
|
|
443
450
|
peg$currPos += 2;
|
|
444
451
|
} else {
|
|
445
452
|
s0 = peg$FAILED;
|
|
446
|
-
if (peg$silentFails === 0) { peg$fail(peg$
|
|
453
|
+
if (peg$silentFails === 0) { peg$fail(peg$c11); }
|
|
447
454
|
}
|
|
448
455
|
if (s0 === peg$FAILED) {
|
|
449
456
|
if (input.charCodeAt(peg$currPos) === 10) {
|
|
450
|
-
s0 = peg$
|
|
457
|
+
s0 = peg$c12;
|
|
451
458
|
peg$currPos++;
|
|
452
459
|
} else {
|
|
453
460
|
s0 = peg$FAILED;
|
|
454
|
-
if (peg$silentFails === 0) { peg$fail(peg$
|
|
461
|
+
if (peg$silentFails === 0) { peg$fail(peg$c13); }
|
|
455
462
|
}
|
|
456
463
|
if (s0 === peg$FAILED) {
|
|
457
464
|
if (input.charCodeAt(peg$currPos) === 13) {
|
|
458
|
-
s0 = peg$
|
|
465
|
+
s0 = peg$c14;
|
|
459
466
|
peg$currPos++;
|
|
460
467
|
} else {
|
|
461
468
|
s0 = peg$FAILED;
|
|
462
|
-
if (peg$silentFails === 0) { peg$fail(peg$
|
|
469
|
+
if (peg$silentFails === 0) { peg$fail(peg$c15); }
|
|
463
470
|
}
|
|
464
471
|
}
|
|
465
472
|
}
|
|
@@ -471,12 +478,12 @@ module.exports = (function() {
|
|
|
471
478
|
var s0, s1, s2, s3, s4, s5;
|
|
472
479
|
|
|
473
480
|
s0 = peg$currPos;
|
|
474
|
-
if (input.substr(peg$currPos, 2) === peg$
|
|
475
|
-
s1 = peg$
|
|
481
|
+
if (input.substr(peg$currPos, 2) === peg$c16) {
|
|
482
|
+
s1 = peg$c16;
|
|
476
483
|
peg$currPos += 2;
|
|
477
484
|
} else {
|
|
478
485
|
s1 = peg$FAILED;
|
|
479
|
-
if (peg$silentFails === 0) { peg$fail(peg$
|
|
486
|
+
if (peg$silentFails === 0) { peg$fail(peg$c17); }
|
|
480
487
|
}
|
|
481
488
|
if (s1 !== peg$FAILED) {
|
|
482
489
|
s2 = [];
|
|
@@ -486,7 +493,7 @@ module.exports = (function() {
|
|
|
486
493
|
s5 = peg$parseEOL();
|
|
487
494
|
peg$silentFails--;
|
|
488
495
|
if (s5 === peg$FAILED) {
|
|
489
|
-
s4 = peg$
|
|
496
|
+
s4 = peg$c18;
|
|
490
497
|
} else {
|
|
491
498
|
peg$currPos = s4;
|
|
492
499
|
s4 = peg$c1;
|
|
@@ -497,7 +504,7 @@ module.exports = (function() {
|
|
|
497
504
|
peg$currPos++;
|
|
498
505
|
} else {
|
|
499
506
|
s5 = peg$FAILED;
|
|
500
|
-
if (peg$silentFails === 0) { peg$fail(peg$
|
|
507
|
+
if (peg$silentFails === 0) { peg$fail(peg$c19); }
|
|
501
508
|
}
|
|
502
509
|
if (s5 !== peg$FAILED) {
|
|
503
510
|
s4 = [s4, s5];
|
|
@@ -518,7 +525,7 @@ module.exports = (function() {
|
|
|
518
525
|
s5 = peg$parseEOL();
|
|
519
526
|
peg$silentFails--;
|
|
520
527
|
if (s5 === peg$FAILED) {
|
|
521
|
-
s4 = peg$
|
|
528
|
+
s4 = peg$c18;
|
|
522
529
|
} else {
|
|
523
530
|
peg$currPos = s4;
|
|
524
531
|
s4 = peg$c1;
|
|
@@ -529,7 +536,7 @@ module.exports = (function() {
|
|
|
529
536
|
peg$currPos++;
|
|
530
537
|
} else {
|
|
531
538
|
s5 = peg$FAILED;
|
|
532
|
-
if (peg$silentFails === 0) { peg$fail(peg$
|
|
539
|
+
if (peg$silentFails === 0) { peg$fail(peg$c19); }
|
|
533
540
|
}
|
|
534
541
|
if (s5 !== peg$FAILED) {
|
|
535
542
|
s4 = [s4, s5];
|
|
@@ -572,19 +579,19 @@ module.exports = (function() {
|
|
|
572
579
|
|
|
573
580
|
s0 = [];
|
|
574
581
|
if (input.charCodeAt(peg$currPos) === 32) {
|
|
575
|
-
s1 = peg$
|
|
582
|
+
s1 = peg$c20;
|
|
576
583
|
peg$currPos++;
|
|
577
584
|
} else {
|
|
578
585
|
s1 = peg$FAILED;
|
|
579
|
-
if (peg$silentFails === 0) { peg$fail(peg$
|
|
586
|
+
if (peg$silentFails === 0) { peg$fail(peg$c21); }
|
|
580
587
|
}
|
|
581
588
|
if (s1 === peg$FAILED) {
|
|
582
589
|
if (input.charCodeAt(peg$currPos) === 9) {
|
|
583
|
-
s1 = peg$
|
|
590
|
+
s1 = peg$c22;
|
|
584
591
|
peg$currPos++;
|
|
585
592
|
} else {
|
|
586
593
|
s1 = peg$FAILED;
|
|
587
|
-
if (peg$silentFails === 0) { peg$fail(peg$
|
|
594
|
+
if (peg$silentFails === 0) { peg$fail(peg$c23); }
|
|
588
595
|
}
|
|
589
596
|
if (s1 === peg$FAILED) {
|
|
590
597
|
s1 = peg$parseEOL();
|
|
@@ -596,19 +603,19 @@ module.exports = (function() {
|
|
|
596
603
|
while (s1 !== peg$FAILED) {
|
|
597
604
|
s0.push(s1);
|
|
598
605
|
if (input.charCodeAt(peg$currPos) === 32) {
|
|
599
|
-
s1 = peg$
|
|
606
|
+
s1 = peg$c20;
|
|
600
607
|
peg$currPos++;
|
|
601
608
|
} else {
|
|
602
609
|
s1 = peg$FAILED;
|
|
603
|
-
if (peg$silentFails === 0) { peg$fail(peg$
|
|
610
|
+
if (peg$silentFails === 0) { peg$fail(peg$c21); }
|
|
604
611
|
}
|
|
605
612
|
if (s1 === peg$FAILED) {
|
|
606
613
|
if (input.charCodeAt(peg$currPos) === 9) {
|
|
607
|
-
s1 = peg$
|
|
614
|
+
s1 = peg$c22;
|
|
608
615
|
peg$currPos++;
|
|
609
616
|
} else {
|
|
610
617
|
s1 = peg$FAILED;
|
|
611
|
-
if (peg$silentFails === 0) { peg$fail(peg$
|
|
618
|
+
if (peg$silentFails === 0) { peg$fail(peg$c23); }
|
|
612
619
|
}
|
|
613
620
|
if (s1 === peg$FAILED) {
|
|
614
621
|
s1 = peg$parseEOL();
|
|
@@ -627,19 +634,19 @@ module.exports = (function() {
|
|
|
627
634
|
|
|
628
635
|
s0 = [];
|
|
629
636
|
if (input.charCodeAt(peg$currPos) === 32) {
|
|
630
|
-
s1 = peg$
|
|
637
|
+
s1 = peg$c20;
|
|
631
638
|
peg$currPos++;
|
|
632
639
|
} else {
|
|
633
640
|
s1 = peg$FAILED;
|
|
634
|
-
if (peg$silentFails === 0) { peg$fail(peg$
|
|
641
|
+
if (peg$silentFails === 0) { peg$fail(peg$c21); }
|
|
635
642
|
}
|
|
636
643
|
if (s1 === peg$FAILED) {
|
|
637
644
|
if (input.charCodeAt(peg$currPos) === 9) {
|
|
638
|
-
s1 = peg$
|
|
645
|
+
s1 = peg$c22;
|
|
639
646
|
peg$currPos++;
|
|
640
647
|
} else {
|
|
641
648
|
s1 = peg$FAILED;
|
|
642
|
-
if (peg$silentFails === 0) { peg$fail(peg$
|
|
649
|
+
if (peg$silentFails === 0) { peg$fail(peg$c23); }
|
|
643
650
|
}
|
|
644
651
|
if (s1 === peg$FAILED) {
|
|
645
652
|
s1 = peg$parseComment();
|
|
@@ -648,19 +655,19 @@ module.exports = (function() {
|
|
|
648
655
|
while (s1 !== peg$FAILED) {
|
|
649
656
|
s0.push(s1);
|
|
650
657
|
if (input.charCodeAt(peg$currPos) === 32) {
|
|
651
|
-
s1 = peg$
|
|
658
|
+
s1 = peg$c20;
|
|
652
659
|
peg$currPos++;
|
|
653
660
|
} else {
|
|
654
661
|
s1 = peg$FAILED;
|
|
655
|
-
if (peg$silentFails === 0) { peg$fail(peg$
|
|
662
|
+
if (peg$silentFails === 0) { peg$fail(peg$c21); }
|
|
656
663
|
}
|
|
657
664
|
if (s1 === peg$FAILED) {
|
|
658
665
|
if (input.charCodeAt(peg$currPos) === 9) {
|
|
659
|
-
s1 = peg$
|
|
666
|
+
s1 = peg$c22;
|
|
660
667
|
peg$currPos++;
|
|
661
668
|
} else {
|
|
662
669
|
s1 = peg$FAILED;
|
|
663
|
-
if (peg$silentFails === 0) { peg$fail(peg$
|
|
670
|
+
if (peg$silentFails === 0) { peg$fail(peg$c23); }
|
|
664
671
|
}
|
|
665
672
|
if (s1 === peg$FAILED) {
|
|
666
673
|
s1 = peg$parseComment();
|
|
@@ -681,11 +688,11 @@ module.exports = (function() {
|
|
|
681
688
|
peg$currPos++;
|
|
682
689
|
} else {
|
|
683
690
|
s1 = peg$FAILED;
|
|
684
|
-
if (peg$silentFails === 0) { peg$fail(peg$
|
|
691
|
+
if (peg$silentFails === 0) { peg$fail(peg$c19); }
|
|
685
692
|
}
|
|
686
693
|
peg$silentFails--;
|
|
687
694
|
if (s1 === peg$FAILED) {
|
|
688
|
-
s0 = peg$
|
|
695
|
+
s0 = peg$c18;
|
|
689
696
|
} else {
|
|
690
697
|
peg$currPos = s0;
|
|
691
698
|
s0 = peg$c1;
|
|
@@ -13,10 +13,22 @@ transformation_rules = rules: (S transformation_rule S) +
|
|
|
13
13
|
return(result);
|
|
14
14
|
}
|
|
15
15
|
|
|
16
|
-
transformation_rule = c1: category1 c2: category2 pred: predicate
|
|
16
|
+
transformation_rule = c1: category1 c2: category2 pred: predicate pars: parameter *
|
|
17
17
|
{
|
|
18
|
+
var result = null;
|
|
19
|
+
|
|
18
20
|
// Construct rule
|
|
19
|
-
|
|
21
|
+
if (pars.length === 1) {
|
|
22
|
+
result = new TransformationRule(c1, c2, pred, pars[0]);
|
|
23
|
+
}
|
|
24
|
+
else {
|
|
25
|
+
if (pars.length === 2) {
|
|
26
|
+
result = new TransformationRule(c1, c2, pred, pars[0], pars[1]);
|
|
27
|
+
}
|
|
28
|
+
else {
|
|
29
|
+
result = new TransformationRule(c1, c2, pred);
|
|
30
|
+
}
|
|
31
|
+
}
|
|
20
32
|
return(result);
|
|
21
33
|
}
|
|
22
34
|
|
|
@@ -29,7 +41,10 @@ predicate = identifier
|
|
|
29
41
|
parameter = identifier
|
|
30
42
|
|
|
31
43
|
identifier =
|
|
32
|
-
characters: [a-zA-Z_0-9_\-\.,()]+ S_no_eol
|
|
44
|
+
// characters: [a-zA-Z_0-9_\-\.,()]+ S_no_eol
|
|
45
|
+
// https://en.wikipedia.org/wiki/List_of_Unicode_characters
|
|
46
|
+
// Ranges supported: U+0021 - U+007E plus U+00A1 - U+00FF
|
|
47
|
+
characters: [!-~¡-ÿ]+ S_no_eol
|
|
33
48
|
{
|
|
34
49
|
var s = "";
|
|
35
50
|
for (var i = 0; i < characters.length; i++) {
|
|
@@ -52,5 +67,5 @@ S =
|
|
|
52
67
|
(' ' / '\t' / EOL / Comment)*
|
|
53
68
|
S_no_eol =
|
|
54
69
|
(' ' / '\t' / Comment)*
|
|
55
|
-
EOI=
|
|
70
|
+
EOI=
|
|
56
71
|
!.
|
|
@@ -20,6 +20,7 @@
|
|
|
20
20
|
//var logger = log4js.getLogger();
|
|
21
21
|
|
|
22
22
|
var Predicate = require("./Predicate");
|
|
23
|
+
var Sentence = require('./Sentence');
|
|
23
24
|
|
|
24
25
|
//logger.setLevel('INFO');
|
|
25
26
|
|
|
@@ -42,11 +43,11 @@ TransformationRule.prototype.key = function() {
|
|
|
42
43
|
return(this.literal.toString());
|
|
43
44
|
};
|
|
44
45
|
|
|
45
|
-
TransformationRule.prototype.apply = function(
|
|
46
|
-
if ((
|
|
46
|
+
TransformationRule.prototype.apply = function(sentence, position) {
|
|
47
|
+
if ((sentence.taggedWords[position].tag === this.old_category) ||
|
|
47
48
|
(this.old_category === category_wild_card)) {
|
|
48
|
-
if (this.predicate.evaluate(
|
|
49
|
-
|
|
49
|
+
if (this.predicate.evaluate(sentence, position)) {
|
|
50
|
+
sentence.taggedWords[position].tag = this.new_category;
|
|
50
51
|
//logger.debug('TransformationRule.apply: changed category ' +
|
|
51
52
|
//this.old_category + ' to ' + this.new_category +
|
|
52
53
|
//' at position ' + position);
|
|
@@ -62,13 +63,13 @@ TransformationRule.prototype.apply = function(tagged_sentence, position) {
|
|
|
62
63
|
// category to newTag
|
|
63
64
|
TransformationRule.prototype.isApplicableAt = function(sentence, taggedSentence, i) {
|
|
64
65
|
//logger.debug("TransformationRule.prototype.isApplicableAt: " + taggedSentence);
|
|
65
|
-
var applies = (taggedSentence[i]
|
|
66
|
+
var applies = (taggedSentence.taggedWords[i].tag === this.old_category) &&
|
|
66
67
|
this.predicate.evaluate(taggedSentence, i);
|
|
67
68
|
//logger.debug("TransformationRule.prototype.isApplicableAt: " + applies);
|
|
68
69
|
|
|
69
70
|
// Set newTag to let the trainer know what the new tag would become
|
|
70
71
|
if (applies) {
|
|
71
|
-
sentence[i].newTag = this.new_category;
|
|
72
|
+
sentence.taggedWords[i].newTag = this.new_category;
|
|
72
73
|
}
|
|
73
74
|
return(applies);
|
|
74
75
|
};
|
|
@@ -92,16 +93,14 @@ TransformationRule.prototype.prettyPrint = function() {
|
|
|
92
93
|
|
|
93
94
|
// Applies the rule the given location (if it applies)
|
|
94
95
|
TransformationRule.prototype.applyAt = function(sentence, i) {
|
|
95
|
-
var taggedSentence = sentence.
|
|
96
|
-
return [token.token, token.tag];
|
|
97
|
-
});
|
|
96
|
+
var taggedSentence = sentence.clone();
|
|
98
97
|
|
|
99
98
|
//logger.debug("TransformationRule.prototype.applyAt: input sentence length: " + sentence.length);
|
|
100
99
|
//logger.debug("TransformationRule.prototype.applyAt: tagged sentence length: " + taggedSentence.length);
|
|
101
100
|
|
|
102
|
-
this.apply(
|
|
101
|
+
this.apply(sentence, i);
|
|
103
102
|
// Assign the new tag to the corpus site
|
|
104
|
-
sentence[i].testTag = taggedSentence[i]
|
|
103
|
+
sentence.taggedWords[i].testTag = taggedSentence.taggedWords[i].tag;
|
|
105
104
|
};
|
|
106
105
|
|
|
107
106
|
// Calculate the net score of this rule
|
|
@@ -58,7 +58,7 @@ function addDocument(text, classification) {
|
|
|
58
58
|
}
|
|
59
59
|
|
|
60
60
|
if(typeof text === 'string')
|
|
61
|
-
text = this.stemmer.tokenizeAndStem(text);
|
|
61
|
+
text = this.stemmer.tokenizeAndStem(text, this.keepStops);
|
|
62
62
|
|
|
63
63
|
if(text.length === 0) {
|
|
64
64
|
// ignore empty documents
|
|
@@ -82,7 +82,7 @@ function removeDocument(text, classification) {
|
|
|
82
82
|
, pos;
|
|
83
83
|
|
|
84
84
|
if (typeof text === 'string') {
|
|
85
|
-
text = this.stemmer.tokenizeAndStem(text);
|
|
85
|
+
text = this.stemmer.tokenizeAndStem(text, this.keepStops);
|
|
86
86
|
}
|
|
87
87
|
|
|
88
88
|
for (var i = 0, ii = docs.length; i < ii; i++) {
|
|
@@ -107,7 +107,7 @@ function textToFeatures(observation) {
|
|
|
107
107
|
var features = [];
|
|
108
108
|
|
|
109
109
|
if(typeof observation === 'string')
|
|
110
|
-
observation = this.stemmer.tokenizeAndStem(observation);
|
|
110
|
+
observation = this.stemmer.tokenizeAndStem(observation, this.keepStops);
|
|
111
111
|
|
|
112
112
|
for(var feature in this.features) {
|
|
113
113
|
if(observation.indexOf(feature) > -1)
|
|
@@ -180,7 +180,7 @@ function trainParallel(numThreads, callback) {
|
|
|
180
180
|
for (var i = this.lastAdded; i < totalDocs; i++) {
|
|
181
181
|
var observation = this.docs[i].text;
|
|
182
182
|
if (typeof observation === 'string')
|
|
183
|
-
observation = this.stemmer.tokenizeAndStem(observation);
|
|
183
|
+
observation = this.stemmer.tokenizeAndStem(observation, this.keepStops);
|
|
184
184
|
obsDocs.push({
|
|
185
185
|
index: i,
|
|
186
186
|
observation: observation
|
|
@@ -289,7 +289,7 @@ function trainParallelBatches(options) {
|
|
|
289
289
|
for (var i = this.lastAdded; i < totalDocs; i++) {
|
|
290
290
|
var observation = this.docs[i].text;
|
|
291
291
|
if (typeof observation === 'string')
|
|
292
|
-
observation = this.stemmer.tokenizeAndStem(observation);
|
|
292
|
+
observation = this.stemmer.tokenizeAndStem(observation, this.keepStops);
|
|
293
293
|
obsDocs.push({
|
|
294
294
|
index: i,
|
|
295
295
|
observation: observation
|
|
@@ -437,17 +437,24 @@ function load(filename, callback) {
|
|
|
437
437
|
});
|
|
438
438
|
}
|
|
439
439
|
|
|
440
|
+
function setOptions(options){
|
|
441
|
+
this.keepStops = (options.keepStops) ? true : false;
|
|
442
|
+
}
|
|
443
|
+
|
|
440
444
|
Classifier.prototype.addDocument = addDocument;
|
|
441
445
|
Classifier.prototype.removeDocument = removeDocument;
|
|
442
446
|
Classifier.prototype.train = train;
|
|
443
|
-
|
|
444
|
-
Classifier.prototype.
|
|
447
|
+
if (Threads) {
|
|
448
|
+
Classifier.prototype.trainParallel = trainParallel;
|
|
449
|
+
Classifier.prototype.trainParallelBatches = trainParallelBatches;
|
|
450
|
+
Classifier.prototype.retrainParallel = retrainParallel;
|
|
451
|
+
}
|
|
445
452
|
Classifier.prototype.retrain = retrain;
|
|
446
|
-
Classifier.prototype.retrainParallel = retrainParallel;
|
|
447
453
|
Classifier.prototype.classify = classify;
|
|
448
454
|
Classifier.prototype.textToFeatures = textToFeatures;
|
|
449
455
|
Classifier.prototype.save = save;
|
|
450
456
|
Classifier.prototype.getClassifications = getClassifications;
|
|
457
|
+
Classifier.prototype.setOptions = setOptions;
|
|
451
458
|
Classifier.restore = restore;
|
|
452
459
|
Classifier.load = load;
|
|
453
460
|
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
/*
|
|
2
|
+
Classifier class that provides functionality for training and
|
|
3
|
+
classification
|
|
4
|
+
Copyright (C) 2017 Hugo W.L. ter Doest
|
|
5
|
+
|
|
6
|
+
This program is free software: you can redistribute it and/or modify
|
|
7
|
+
it under the terms of the GNU General Public License as published by
|
|
8
|
+
the Free Software Foundation, either version 3 of the License, or
|
|
9
|
+
(at your option) any later version.
|
|
10
|
+
|
|
11
|
+
This program is distributed in the hope that it will be useful,
|
|
12
|
+
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
13
|
+
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
14
|
+
GNU General Public License for more details.
|
|
15
|
+
|
|
16
|
+
You should have received a copy of the GNU General Public License
|
|
17
|
+
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
var fs = require('fs');
|
|
21
|
+
|
|
22
|
+
var Context = require('./Context');
|
|
23
|
+
var Element = require('./Element');
|
|
24
|
+
var Sample = require('./Sample');
|
|
25
|
+
var Scaler = require('./GISScaler');
|
|
26
|
+
var FeatureSet = require('./FeatureSet');
|
|
27
|
+
|
|
28
|
+
function Classifier(features, sample) {
|
|
29
|
+
if (features) {
|
|
30
|
+
this.features = features;
|
|
31
|
+
}
|
|
32
|
+
else {
|
|
33
|
+
this.features = new featureSet();
|
|
34
|
+
}
|
|
35
|
+
this.features = features;
|
|
36
|
+
if (sample) {
|
|
37
|
+
this.sample = sample;
|
|
38
|
+
}
|
|
39
|
+
else {
|
|
40
|
+
this.sample = new Sample();
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
// Loads a classifier from file.
|
|
45
|
+
// Caveat: feature functions are generated from the sample elements. You need
|
|
46
|
+
// to create your own specialisation of the Element class that can generate
|
|
47
|
+
// your own specific feature functions
|
|
48
|
+
Classifier.prototype.load = function(filename, elementClass, callback) {
|
|
49
|
+
fs.readFile(filename, 'utf8', function(err, data) {
|
|
50
|
+
|
|
51
|
+
if(!err) {
|
|
52
|
+
classifierData = JSON.parse(data);
|
|
53
|
+
var sample = new Sample();
|
|
54
|
+
classifierData.sample.elements.forEach(function(elementData) {
|
|
55
|
+
var elt = new elementClass(elementData.a, new Context(elementData.b.data));
|
|
56
|
+
sample.addElement(elt);
|
|
57
|
+
});
|
|
58
|
+
var featureSet = new FeatureSet();
|
|
59
|
+
sample.generateFeatures(featureSet);
|
|
60
|
+
var classifier = new Classifier(featureSet, sample);
|
|
61
|
+
callback(err, classifier);
|
|
62
|
+
}
|
|
63
|
+
else {
|
|
64
|
+
if(callback) {
|
|
65
|
+
callback(err);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
});
|
|
69
|
+
};
|
|
70
|
+
|
|
71
|
+
Classifier.prototype.save = function(filename, callback) {
|
|
72
|
+
var data = JSON.stringify(this, null, 2);
|
|
73
|
+
var classifier = this;
|
|
74
|
+
fs.writeFile(filename, data, 'utf8', function(err) {
|
|
75
|
+
if(callback) {
|
|
76
|
+
callback(err, err ? null : classifier);
|
|
77
|
+
}
|
|
78
|
+
});
|
|
79
|
+
};
|
|
80
|
+
|
|
81
|
+
Classifier.prototype.addElement = function(x) {
|
|
82
|
+
this.sample.addElement(x);
|
|
83
|
+
};
|
|
84
|
+
|
|
85
|
+
Classifier.prototype.addDocument = function(context, classification, elementClass) {
|
|
86
|
+
Classifier.prototype.addElement(new elementClass(classification, context));
|
|
87
|
+
};
|
|
88
|
+
|
|
89
|
+
Classifier.prototype.train = function(maxIterations, minImprovement, approxExpectation) {
|
|
90
|
+
this.scaler = new Scaler(this.features, this.sample);
|
|
91
|
+
this.p = this.scaler.run(maxIterations, minImprovement, approxExpectation);
|
|
92
|
+
};
|
|
93
|
+
|
|
94
|
+
Classifier.prototype.getClassifications = function(b) {
|
|
95
|
+
var scores = [];
|
|
96
|
+
var that = this;
|
|
97
|
+
this.sample.getClasses().forEach(function(a) {
|
|
98
|
+
var x = new Element(a, b);
|
|
99
|
+
scores.push({
|
|
100
|
+
"label": a,
|
|
101
|
+
"value": that.p.calculateAPriori(x)
|
|
102
|
+
});
|
|
103
|
+
});
|
|
104
|
+
return scores;
|
|
105
|
+
};
|
|
106
|
+
|
|
107
|
+
Classifier.prototype.classify = function(b) {
|
|
108
|
+
var scores = this.getClassifications(b);
|
|
109
|
+
// Sort the scores in an array
|
|
110
|
+
scores.sort(function(a, b) {
|
|
111
|
+
return b.value - a.value;
|
|
112
|
+
});
|
|
113
|
+
// Check if the classifier discriminates
|
|
114
|
+
var min = scores[scores.length - 1].value;
|
|
115
|
+
var max = scores[0].value;
|
|
116
|
+
if (min === max) {
|
|
117
|
+
return "";
|
|
118
|
+
}
|
|
119
|
+
else {
|
|
120
|
+
// Return the highest scoring classes
|
|
121
|
+
return scores[0].label;
|
|
122
|
+
}
|
|
123
|
+
};
|
|
124
|
+
|
|
125
|
+
module.exports = Classifier;
|