natural 0.5.4 → 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. package/.dockerignore +2 -0
  2. package/.npmignore +1 -0
  3. package/.travis.yml +1 -1
  4. package/README.md +412 -89
  5. package/bayes_classifier.json +1 -1
  6. package/classifier.json +418 -0
  7. package/data/kata-dasar.txt +29933 -0
  8. package/lib/natural/brill_pos_tagger/lib/Brill_POS_Tagger.js +8 -9
  9. package/lib/natural/brill_pos_tagger/lib/Brill_POS_Tester.js +5 -5
  10. package/lib/natural/brill_pos_tagger/lib/Brill_POS_Trainer.js +10 -6
  11. package/lib/natural/brill_pos_tagger/lib/Corpus.js +21 -16
  12. package/lib/natural/brill_pos_tagger/lib/Lexicon.js +22 -13
  13. package/lib/natural/brill_pos_tagger/lib/Predicate.js +2 -2
  14. package/lib/natural/brill_pos_tagger/lib/RuleTemplate.js +2 -2
  15. package/lib/natural/brill_pos_tagger/lib/RuleTemplates.js +132 -146
  16. package/lib/natural/brill_pos_tagger/lib/Sentence.js +41 -0
  17. package/lib/natural/brill_pos_tagger/lib/TF_Parser.js +83 -76
  18. package/lib/natural/brill_pos_tagger/lib/TF_Parser.pegjs +19 -4
  19. package/lib/natural/brill_pos_tagger/lib/TransformationRule.js +10 -11
  20. package/lib/natural/classifiers/classifier.js +15 -8
  21. package/lib/natural/classifiers/maxent/Classifier.js +125 -0
  22. package/lib/natural/classifiers/maxent/Context.js +33 -0
  23. package/lib/natural/classifiers/maxent/Distribution.js +145 -0
  24. package/lib/natural/classifiers/maxent/Element.js +34 -0
  25. package/lib/natural/classifiers/maxent/Feature.js +88 -0
  26. package/lib/natural/classifiers/maxent/FeatureSet.js +67 -0
  27. package/lib/natural/classifiers/maxent/GISScaler.js +148 -0
  28. package/lib/natural/classifiers/maxent/POS/ME_Corpus.js +56 -0
  29. package/lib/natural/classifiers/maxent/POS/ME_Sentence.js +73 -0
  30. package/lib/natural/classifiers/maxent/POS/POS_Element.js +172 -0
  31. package/lib/natural/classifiers/maxent/Sample.js +141 -0
  32. package/lib/natural/classifiers/maxent/SimpleExample/SE_Element.js +49 -0
  33. package/lib/natural/distance/hamming_distance.js +52 -0
  34. package/lib/natural/distance/jaro-winkler_distance.js +16 -6
  35. package/lib/natural/distance/levenshtein_distance.js +164 -11
  36. package/lib/natural/index.js +26 -5
  37. package/lib/natural/inflectors/noun_inflector.js +15 -13
  38. package/lib/natural/ngrams/ngrams.js +83 -25
  39. package/lib/natural/normalizers/normalizer_sv.js +41 -0
  40. package/lib/natural/sentiment/Basque/senticon_eu.json +21557 -0
  41. package/lib/natural/sentiment/Catalan/senticon_ca.json +36352 -0
  42. package/lib/natural/sentiment/Dutch/negations_du.json +3 -0
  43. package/lib/natural/sentiment/Dutch/pattern-sentiment-nl.json +39650 -0
  44. package/lib/natural/sentiment/English/negations_en.json +3 -0
  45. package/lib/natural/sentiment/English/pattern-sentiment-en.json +16024 -0
  46. package/lib/natural/sentiment/English/senticon_en.json +124197 -0
  47. package/lib/natural/sentiment/French/pattern-sentiment-fr.json +40906 -0
  48. package/lib/natural/sentiment/Galician/senticon_gl.json +24427 -0
  49. package/lib/natural/sentiment/Italian/pattern-sentiment-it.json +24522 -0
  50. package/lib/natural/sentiment/SentimentAnalyzer.js +116 -0
  51. package/lib/natural/sentiment/Spanish/afinnShortSortedSpanish.json +1655 -0
  52. package/lib/natural/sentiment/Spanish/negations_es.json +3 -0
  53. package/lib/natural/sentiment/Spanish/senticon_es.json +56722 -0
  54. package/lib/natural/sentiment/tools/README.md +31 -0
  55. package/lib/natural/sentiment/tools/XmlParser4PatternData.js +31 -0
  56. package/lib/natural/sentiment/tools/sentimentXmlParser.js +30 -0
  57. package/lib/natural/stemmers/indonesian/base_stemmer_id.js +86 -0
  58. package/lib/natural/stemmers/indonesian/prefix_rules.js +1027 -0
  59. package/lib/natural/stemmers/indonesian/removal.js +49 -0
  60. package/lib/natural/stemmers/indonesian/stemmer_id.js +307 -0
  61. package/lib/natural/stemmers/indonesian/suffix_rules.js +75 -0
  62. package/lib/natural/stemmers/porter_stemmer_es.js +265 -199
  63. package/lib/natural/stemmers/porter_stemmer_fr.js +1 -1
  64. package/lib/natural/stemmers/porter_stemmer_nl.js +362 -0
  65. package/lib/natural/stemmers/porter_stemmer_sv.js +99 -0
  66. package/lib/natural/stemmers/stemmer_es.js +18 -12
  67. package/lib/natural/stemmers/stemmer_nl.js +64 -0
  68. package/lib/natural/stemmers/stemmer_sv.js +61 -0
  69. package/lib/natural/tfidf/tfidf.js +27 -4
  70. package/lib/natural/tokenizers/aggressive_tokenizer_id.js +44 -0
  71. package/lib/natural/tokenizers/aggressive_tokenizer_sv.js +40 -0
  72. package/lib/natural/tokenizers/aggressive_tokenizer_vi.js +37 -0
  73. package/lib/natural/tokenizers/orthography_matchers.js +33 -0
  74. package/lib/natural/tokenizers/regexp_tokenizer.js +23 -3
  75. package/lib/natural/tokenizers/sentence_tokenizer.js +1 -1
  76. package/lib/natural/util/stopwords_id.js +119 -0
  77. package/lib/natural/util/stopwords_no.js +1 -1
  78. package/lib/natural/util/stopwords_sv.js +70 -0
  79. package/lib/natural/wordnet/wordnet_file.js +7 -1
  80. package/lr_classifier.json +1 -1
  81. package/package-lock.json +378 -0
  82. package/package.json +29 -21
@@ -43,40 +43,51 @@ module.exports = (function() {
43
43
  }
44
44
  return(result);
45
45
  },
46
- peg$c3 = null,
47
- peg$c4 = function(c1, c2, pred, par1, par2) {
46
+ peg$c3 = function(c1, c2, pred, pars) {
47
+ var result = null;
48
+
48
49
  // Construct rule
49
- var result = new TransformationRule(c1, c2, pred, par1, par2);
50
+ if (pars.length === 1) {
51
+ result = new TransformationRule(c1, c2, pred, pars[0]);
52
+ }
53
+ else {
54
+ if (pars.length === 2) {
55
+ result = new TransformationRule(c1, c2, pred, pars[0], pars[1]);
56
+ }
57
+ else {
58
+ result = new TransformationRule(c1, c2, pred);
59
+ }
60
+ }
50
61
  return(result);
51
62
  },
52
- peg$c5 = /^[a-zA-Z_0-9_\-.,()]/,
53
- peg$c6 = { type: "class", value: "[a-zA-Z_0-9_\\-.,()]", description: "[a-zA-Z_0-9_\\-.,()]" },
54
- peg$c7 = function(characters) {
63
+ peg$c4 = /^[!-~\xA1-\xFF]/,
64
+ peg$c5 = { type: "class", value: "[!-~\\xA1-\\xFF]", description: "[!-~\\xA1-\\xFF]" },
65
+ peg$c6 = function(characters) {
55
66
  var s = "";
56
67
  for (var i = 0; i < characters.length; i++) {
57
68
  s += characters[i];
58
69
  }
59
70
  return(s);
60
71
  },
61
- peg$c8 = "*",
62
- peg$c9 = { type: "literal", value: "*", description: "\"*\"" },
63
- peg$c10 = function(wc) {
72
+ peg$c7 = "*",
73
+ peg$c8 = { type: "literal", value: "*", description: "\"*\"" },
74
+ peg$c9 = function(wc) {
64
75
  return(wc)
65
76
  },
66
- peg$c11 = "\r\n",
67
- peg$c12 = { type: "literal", value: "\r\n", description: "\"\\r\\n\"" },
68
- peg$c13 = "\n",
69
- peg$c14 = { type: "literal", value: "\n", description: "\"\\n\"" },
70
- peg$c15 = "\r",
71
- peg$c16 = { type: "literal", value: "\r", description: "\"\\r\"" },
72
- peg$c17 = "//",
73
- peg$c18 = { type: "literal", value: "//", description: "\"//\"" },
74
- peg$c19 = void 0,
75
- peg$c20 = { type: "any", description: "any character" },
76
- peg$c21 = " ",
77
- peg$c22 = { type: "literal", value: " ", description: "\" \"" },
78
- peg$c23 = "\t",
79
- peg$c24 = { type: "literal", value: "\t", description: "\"\\t\"" },
77
+ peg$c10 = "\r\n",
78
+ peg$c11 = { type: "literal", value: "\r\n", description: "\"\\r\\n\"" },
79
+ peg$c12 = "\n",
80
+ peg$c13 = { type: "literal", value: "\n", description: "\"\\n\"" },
81
+ peg$c14 = "\r",
82
+ peg$c15 = { type: "literal", value: "\r", description: "\"\\r\"" },
83
+ peg$c16 = "//",
84
+ peg$c17 = { type: "literal", value: "//", description: "\"//\"" },
85
+ peg$c18 = void 0,
86
+ peg$c19 = { type: "any", description: "any character" },
87
+ peg$c20 = " ",
88
+ peg$c21 = { type: "literal", value: " ", description: "\" \"" },
89
+ peg$c22 = "\t",
90
+ peg$c23 = { type: "literal", value: "\t", description: "\"\\t\"" },
80
91
 
81
92
  peg$currPos = 0,
82
93
  peg$reportedPos = 0,
@@ -317,20 +328,16 @@ module.exports = (function() {
317
328
  if (s2 !== peg$FAILED) {
318
329
  s3 = peg$parseidentifier();
319
330
  if (s3 !== peg$FAILED) {
320
- s4 = peg$parseidentifier();
321
- if (s4 !== peg$FAILED) {
331
+ s4 = [];
332
+ s5 = peg$parseidentifier();
333
+ while (s5 !== peg$FAILED) {
334
+ s4.push(s5);
322
335
  s5 = peg$parseidentifier();
323
- if (s5 === peg$FAILED) {
324
- s5 = peg$c3;
325
- }
326
- if (s5 !== peg$FAILED) {
327
- peg$reportedPos = s0;
328
- s1 = peg$c4(s1, s2, s3, s4, s5);
329
- s0 = s1;
330
- } else {
331
- peg$currPos = s0;
332
- s0 = peg$c1;
333
- }
336
+ }
337
+ if (s4 !== peg$FAILED) {
338
+ peg$reportedPos = s0;
339
+ s1 = peg$c3(s1, s2, s3, s4);
340
+ s0 = s1;
334
341
  } else {
335
342
  peg$currPos = s0;
336
343
  s0 = peg$c1;
@@ -367,22 +374,22 @@ module.exports = (function() {
367
374
 
368
375
  s0 = peg$currPos;
369
376
  s1 = [];
370
- if (peg$c5.test(input.charAt(peg$currPos))) {
377
+ if (peg$c4.test(input.charAt(peg$currPos))) {
371
378
  s2 = input.charAt(peg$currPos);
372
379
  peg$currPos++;
373
380
  } else {
374
381
  s2 = peg$FAILED;
375
- if (peg$silentFails === 0) { peg$fail(peg$c6); }
382
+ if (peg$silentFails === 0) { peg$fail(peg$c5); }
376
383
  }
377
384
  if (s2 !== peg$FAILED) {
378
385
  while (s2 !== peg$FAILED) {
379
386
  s1.push(s2);
380
- if (peg$c5.test(input.charAt(peg$currPos))) {
387
+ if (peg$c4.test(input.charAt(peg$currPos))) {
381
388
  s2 = input.charAt(peg$currPos);
382
389
  peg$currPos++;
383
390
  } else {
384
391
  s2 = peg$FAILED;
385
- if (peg$silentFails === 0) { peg$fail(peg$c6); }
392
+ if (peg$silentFails === 0) { peg$fail(peg$c5); }
386
393
  }
387
394
  }
388
395
  } else {
@@ -392,7 +399,7 @@ module.exports = (function() {
392
399
  s2 = peg$parseS_no_eol();
393
400
  if (s2 !== peg$FAILED) {
394
401
  peg$reportedPos = s0;
395
- s1 = peg$c7(s1);
402
+ s1 = peg$c6(s1);
396
403
  s0 = s1;
397
404
  } else {
398
405
  peg$currPos = s0;
@@ -411,17 +418,17 @@ module.exports = (function() {
411
418
 
412
419
  s0 = peg$currPos;
413
420
  if (input.charCodeAt(peg$currPos) === 42) {
414
- s1 = peg$c8;
421
+ s1 = peg$c7;
415
422
  peg$currPos++;
416
423
  } else {
417
424
  s1 = peg$FAILED;
418
- if (peg$silentFails === 0) { peg$fail(peg$c9); }
425
+ if (peg$silentFails === 0) { peg$fail(peg$c8); }
419
426
  }
420
427
  if (s1 !== peg$FAILED) {
421
428
  s2 = peg$parseS_no_eol();
422
429
  if (s2 !== peg$FAILED) {
423
430
  peg$reportedPos = s0;
424
- s1 = peg$c10(s1);
431
+ s1 = peg$c9(s1);
425
432
  s0 = s1;
426
433
  } else {
427
434
  peg$currPos = s0;
@@ -438,28 +445,28 @@ module.exports = (function() {
438
445
  function peg$parseEOL() {
439
446
  var s0;
440
447
 
441
- if (input.substr(peg$currPos, 2) === peg$c11) {
442
- s0 = peg$c11;
448
+ if (input.substr(peg$currPos, 2) === peg$c10) {
449
+ s0 = peg$c10;
443
450
  peg$currPos += 2;
444
451
  } else {
445
452
  s0 = peg$FAILED;
446
- if (peg$silentFails === 0) { peg$fail(peg$c12); }
453
+ if (peg$silentFails === 0) { peg$fail(peg$c11); }
447
454
  }
448
455
  if (s0 === peg$FAILED) {
449
456
  if (input.charCodeAt(peg$currPos) === 10) {
450
- s0 = peg$c13;
457
+ s0 = peg$c12;
451
458
  peg$currPos++;
452
459
  } else {
453
460
  s0 = peg$FAILED;
454
- if (peg$silentFails === 0) { peg$fail(peg$c14); }
461
+ if (peg$silentFails === 0) { peg$fail(peg$c13); }
455
462
  }
456
463
  if (s0 === peg$FAILED) {
457
464
  if (input.charCodeAt(peg$currPos) === 13) {
458
- s0 = peg$c15;
465
+ s0 = peg$c14;
459
466
  peg$currPos++;
460
467
  } else {
461
468
  s0 = peg$FAILED;
462
- if (peg$silentFails === 0) { peg$fail(peg$c16); }
469
+ if (peg$silentFails === 0) { peg$fail(peg$c15); }
463
470
  }
464
471
  }
465
472
  }
@@ -471,12 +478,12 @@ module.exports = (function() {
471
478
  var s0, s1, s2, s3, s4, s5;
472
479
 
473
480
  s0 = peg$currPos;
474
- if (input.substr(peg$currPos, 2) === peg$c17) {
475
- s1 = peg$c17;
481
+ if (input.substr(peg$currPos, 2) === peg$c16) {
482
+ s1 = peg$c16;
476
483
  peg$currPos += 2;
477
484
  } else {
478
485
  s1 = peg$FAILED;
479
- if (peg$silentFails === 0) { peg$fail(peg$c18); }
486
+ if (peg$silentFails === 0) { peg$fail(peg$c17); }
480
487
  }
481
488
  if (s1 !== peg$FAILED) {
482
489
  s2 = [];
@@ -486,7 +493,7 @@ module.exports = (function() {
486
493
  s5 = peg$parseEOL();
487
494
  peg$silentFails--;
488
495
  if (s5 === peg$FAILED) {
489
- s4 = peg$c19;
496
+ s4 = peg$c18;
490
497
  } else {
491
498
  peg$currPos = s4;
492
499
  s4 = peg$c1;
@@ -497,7 +504,7 @@ module.exports = (function() {
497
504
  peg$currPos++;
498
505
  } else {
499
506
  s5 = peg$FAILED;
500
- if (peg$silentFails === 0) { peg$fail(peg$c20); }
507
+ if (peg$silentFails === 0) { peg$fail(peg$c19); }
501
508
  }
502
509
  if (s5 !== peg$FAILED) {
503
510
  s4 = [s4, s5];
@@ -518,7 +525,7 @@ module.exports = (function() {
518
525
  s5 = peg$parseEOL();
519
526
  peg$silentFails--;
520
527
  if (s5 === peg$FAILED) {
521
- s4 = peg$c19;
528
+ s4 = peg$c18;
522
529
  } else {
523
530
  peg$currPos = s4;
524
531
  s4 = peg$c1;
@@ -529,7 +536,7 @@ module.exports = (function() {
529
536
  peg$currPos++;
530
537
  } else {
531
538
  s5 = peg$FAILED;
532
- if (peg$silentFails === 0) { peg$fail(peg$c20); }
539
+ if (peg$silentFails === 0) { peg$fail(peg$c19); }
533
540
  }
534
541
  if (s5 !== peg$FAILED) {
535
542
  s4 = [s4, s5];
@@ -572,19 +579,19 @@ module.exports = (function() {
572
579
 
573
580
  s0 = [];
574
581
  if (input.charCodeAt(peg$currPos) === 32) {
575
- s1 = peg$c21;
582
+ s1 = peg$c20;
576
583
  peg$currPos++;
577
584
  } else {
578
585
  s1 = peg$FAILED;
579
- if (peg$silentFails === 0) { peg$fail(peg$c22); }
586
+ if (peg$silentFails === 0) { peg$fail(peg$c21); }
580
587
  }
581
588
  if (s1 === peg$FAILED) {
582
589
  if (input.charCodeAt(peg$currPos) === 9) {
583
- s1 = peg$c23;
590
+ s1 = peg$c22;
584
591
  peg$currPos++;
585
592
  } else {
586
593
  s1 = peg$FAILED;
587
- if (peg$silentFails === 0) { peg$fail(peg$c24); }
594
+ if (peg$silentFails === 0) { peg$fail(peg$c23); }
588
595
  }
589
596
  if (s1 === peg$FAILED) {
590
597
  s1 = peg$parseEOL();
@@ -596,19 +603,19 @@ module.exports = (function() {
596
603
  while (s1 !== peg$FAILED) {
597
604
  s0.push(s1);
598
605
  if (input.charCodeAt(peg$currPos) === 32) {
599
- s1 = peg$c21;
606
+ s1 = peg$c20;
600
607
  peg$currPos++;
601
608
  } else {
602
609
  s1 = peg$FAILED;
603
- if (peg$silentFails === 0) { peg$fail(peg$c22); }
610
+ if (peg$silentFails === 0) { peg$fail(peg$c21); }
604
611
  }
605
612
  if (s1 === peg$FAILED) {
606
613
  if (input.charCodeAt(peg$currPos) === 9) {
607
- s1 = peg$c23;
614
+ s1 = peg$c22;
608
615
  peg$currPos++;
609
616
  } else {
610
617
  s1 = peg$FAILED;
611
- if (peg$silentFails === 0) { peg$fail(peg$c24); }
618
+ if (peg$silentFails === 0) { peg$fail(peg$c23); }
612
619
  }
613
620
  if (s1 === peg$FAILED) {
614
621
  s1 = peg$parseEOL();
@@ -627,19 +634,19 @@ module.exports = (function() {
627
634
 
628
635
  s0 = [];
629
636
  if (input.charCodeAt(peg$currPos) === 32) {
630
- s1 = peg$c21;
637
+ s1 = peg$c20;
631
638
  peg$currPos++;
632
639
  } else {
633
640
  s1 = peg$FAILED;
634
- if (peg$silentFails === 0) { peg$fail(peg$c22); }
641
+ if (peg$silentFails === 0) { peg$fail(peg$c21); }
635
642
  }
636
643
  if (s1 === peg$FAILED) {
637
644
  if (input.charCodeAt(peg$currPos) === 9) {
638
- s1 = peg$c23;
645
+ s1 = peg$c22;
639
646
  peg$currPos++;
640
647
  } else {
641
648
  s1 = peg$FAILED;
642
- if (peg$silentFails === 0) { peg$fail(peg$c24); }
649
+ if (peg$silentFails === 0) { peg$fail(peg$c23); }
643
650
  }
644
651
  if (s1 === peg$FAILED) {
645
652
  s1 = peg$parseComment();
@@ -648,19 +655,19 @@ module.exports = (function() {
648
655
  while (s1 !== peg$FAILED) {
649
656
  s0.push(s1);
650
657
  if (input.charCodeAt(peg$currPos) === 32) {
651
- s1 = peg$c21;
658
+ s1 = peg$c20;
652
659
  peg$currPos++;
653
660
  } else {
654
661
  s1 = peg$FAILED;
655
- if (peg$silentFails === 0) { peg$fail(peg$c22); }
662
+ if (peg$silentFails === 0) { peg$fail(peg$c21); }
656
663
  }
657
664
  if (s1 === peg$FAILED) {
658
665
  if (input.charCodeAt(peg$currPos) === 9) {
659
- s1 = peg$c23;
666
+ s1 = peg$c22;
660
667
  peg$currPos++;
661
668
  } else {
662
669
  s1 = peg$FAILED;
663
- if (peg$silentFails === 0) { peg$fail(peg$c24); }
670
+ if (peg$silentFails === 0) { peg$fail(peg$c23); }
664
671
  }
665
672
  if (s1 === peg$FAILED) {
666
673
  s1 = peg$parseComment();
@@ -681,11 +688,11 @@ module.exports = (function() {
681
688
  peg$currPos++;
682
689
  } else {
683
690
  s1 = peg$FAILED;
684
- if (peg$silentFails === 0) { peg$fail(peg$c20); }
691
+ if (peg$silentFails === 0) { peg$fail(peg$c19); }
685
692
  }
686
693
  peg$silentFails--;
687
694
  if (s1 === peg$FAILED) {
688
- s0 = peg$c19;
695
+ s0 = peg$c18;
689
696
  } else {
690
697
  peg$currPos = s0;
691
698
  s0 = peg$c1;
@@ -13,10 +13,22 @@ transformation_rules = rules: (S transformation_rule S) +
13
13
  return(result);
14
14
  }
15
15
 
16
- transformation_rule = c1: category1 c2: category2 pred: predicate par1: parameter par2: parameter ?
16
+ transformation_rule = c1: category1 c2: category2 pred: predicate pars: parameter *
17
17
  {
18
+ var result = null;
19
+
18
20
  // Construct rule
19
- var result = new TransformationRule(c1, c2, pred, par1, par2);
21
+ if (pars.length === 1) {
22
+ result = new TransformationRule(c1, c2, pred, pars[0]);
23
+ }
24
+ else {
25
+ if (pars.length === 2) {
26
+ result = new TransformationRule(c1, c2, pred, pars[0], pars[1]);
27
+ }
28
+ else {
29
+ result = new TransformationRule(c1, c2, pred);
30
+ }
31
+ }
20
32
  return(result);
21
33
  }
22
34
 
@@ -29,7 +41,10 @@ predicate = identifier
29
41
  parameter = identifier
30
42
 
31
43
  identifier =
32
- characters: [a-zA-Z_0-9_\-\.,()]+ S_no_eol
44
+ // characters: [a-zA-Z_0-9_\-\.,()]+ S_no_eol
45
+ // https://en.wikipedia.org/wiki/List_of_Unicode_characters
46
+ // Ranges supported: U+0021 - U+007E plus U+00A1 - U+00FF
47
+ characters: [!-~¡-ÿ]+ S_no_eol
33
48
  {
34
49
  var s = "";
35
50
  for (var i = 0; i < characters.length; i++) {
@@ -52,5 +67,5 @@ S =
52
67
  (' ' / '\t' / EOL / Comment)*
53
68
  S_no_eol =
54
69
  (' ' / '\t' / Comment)*
55
- EOI=
70
+ EOI=
56
71
  !.
@@ -20,6 +20,7 @@
20
20
  //var logger = log4js.getLogger();
21
21
 
22
22
  var Predicate = require("./Predicate");
23
+ var Sentence = require('./Sentence');
23
24
 
24
25
  //logger.setLevel('INFO');
25
26
 
@@ -42,11 +43,11 @@ TransformationRule.prototype.key = function() {
42
43
  return(this.literal.toString());
43
44
  };
44
45
 
45
- TransformationRule.prototype.apply = function(tagged_sentence, position) {
46
- if ((tagged_sentence[position][1] === this.old_category) ||
46
+ TransformationRule.prototype.apply = function(sentence, position) {
47
+ if ((sentence.taggedWords[position].tag === this.old_category) ||
47
48
  (this.old_category === category_wild_card)) {
48
- if (this.predicate.evaluate(tagged_sentence, position)) {
49
- tagged_sentence[position][1] = this.new_category;
49
+ if (this.predicate.evaluate(sentence, position)) {
50
+ sentence.taggedWords[position].tag = this.new_category;
50
51
  //logger.debug('TransformationRule.apply: changed category ' +
51
52
  //this.old_category + ' to ' + this.new_category +
52
53
  //' at position ' + position);
@@ -62,13 +63,13 @@ TransformationRule.prototype.apply = function(tagged_sentence, position) {
62
63
  // category to newTag
63
64
  TransformationRule.prototype.isApplicableAt = function(sentence, taggedSentence, i) {
64
65
  //logger.debug("TransformationRule.prototype.isApplicableAt: " + taggedSentence);
65
- var applies = (taggedSentence[i][1] === this.old_category) &&
66
+ var applies = (taggedSentence.taggedWords[i].tag === this.old_category) &&
66
67
  this.predicate.evaluate(taggedSentence, i);
67
68
  //logger.debug("TransformationRule.prototype.isApplicableAt: " + applies);
68
69
 
69
70
  // Set newTag to let the trainer know what the new tag would become
70
71
  if (applies) {
71
- sentence[i].newTag = this.new_category;
72
+ sentence.taggedWords[i].newTag = this.new_category;
72
73
  }
73
74
  return(applies);
74
75
  };
@@ -92,16 +93,14 @@ TransformationRule.prototype.prettyPrint = function() {
92
93
 
93
94
  // Applies the rule the given location (if it applies)
94
95
  TransformationRule.prototype.applyAt = function(sentence, i) {
95
- var taggedSentence = sentence.map(function(token) {
96
- return [token.token, token.tag];
97
- });
96
+ var taggedSentence = sentence.clone();
98
97
 
99
98
  //logger.debug("TransformationRule.prototype.applyAt: input sentence length: " + sentence.length);
100
99
  //logger.debug("TransformationRule.prototype.applyAt: tagged sentence length: " + taggedSentence.length);
101
100
 
102
- this.apply(taggedSentence, i);
101
+ this.apply(sentence, i);
103
102
  // Assign the new tag to the corpus site
104
- sentence[i].testTag = taggedSentence[i][1];
103
+ sentence.taggedWords[i].testTag = taggedSentence.taggedWords[i].tag;
105
104
  };
106
105
 
107
106
  // Calculate the net score of this rule
@@ -58,7 +58,7 @@ function addDocument(text, classification) {
58
58
  }
59
59
 
60
60
  if(typeof text === 'string')
61
- text = this.stemmer.tokenizeAndStem(text);
61
+ text = this.stemmer.tokenizeAndStem(text, this.keepStops);
62
62
 
63
63
  if(text.length === 0) {
64
64
  // ignore empty documents
@@ -82,7 +82,7 @@ function removeDocument(text, classification) {
82
82
  , pos;
83
83
 
84
84
  if (typeof text === 'string') {
85
- text = this.stemmer.tokenizeAndStem(text);
85
+ text = this.stemmer.tokenizeAndStem(text, this.keepStops);
86
86
  }
87
87
 
88
88
  for (var i = 0, ii = docs.length; i < ii; i++) {
@@ -107,7 +107,7 @@ function textToFeatures(observation) {
107
107
  var features = [];
108
108
 
109
109
  if(typeof observation === 'string')
110
- observation = this.stemmer.tokenizeAndStem(observation);
110
+ observation = this.stemmer.tokenizeAndStem(observation, this.keepStops);
111
111
 
112
112
  for(var feature in this.features) {
113
113
  if(observation.indexOf(feature) > -1)
@@ -180,7 +180,7 @@ function trainParallel(numThreads, callback) {
180
180
  for (var i = this.lastAdded; i < totalDocs; i++) {
181
181
  var observation = this.docs[i].text;
182
182
  if (typeof observation === 'string')
183
- observation = this.stemmer.tokenizeAndStem(observation);
183
+ observation = this.stemmer.tokenizeAndStem(observation, this.keepStops);
184
184
  obsDocs.push({
185
185
  index: i,
186
186
  observation: observation
@@ -289,7 +289,7 @@ function trainParallelBatches(options) {
289
289
  for (var i = this.lastAdded; i < totalDocs; i++) {
290
290
  var observation = this.docs[i].text;
291
291
  if (typeof observation === 'string')
292
- observation = this.stemmer.tokenizeAndStem(observation);
292
+ observation = this.stemmer.tokenizeAndStem(observation, this.keepStops);
293
293
  obsDocs.push({
294
294
  index: i,
295
295
  observation: observation
@@ -437,17 +437,24 @@ function load(filename, callback) {
437
437
  });
438
438
  }
439
439
 
440
+ function setOptions(options){
441
+ this.keepStops = (options.keepStops) ? true : false;
442
+ }
443
+
440
444
  Classifier.prototype.addDocument = addDocument;
441
445
  Classifier.prototype.removeDocument = removeDocument;
442
446
  Classifier.prototype.train = train;
443
- Classifier.prototype.trainParallel = trainParallel;
444
- Classifier.prototype.trainParallelBatches = trainParallelBatches;
447
+ if (Threads) {
448
+ Classifier.prototype.trainParallel = trainParallel;
449
+ Classifier.prototype.trainParallelBatches = trainParallelBatches;
450
+ Classifier.prototype.retrainParallel = retrainParallel;
451
+ }
445
452
  Classifier.prototype.retrain = retrain;
446
- Classifier.prototype.retrainParallel = retrainParallel;
447
453
  Classifier.prototype.classify = classify;
448
454
  Classifier.prototype.textToFeatures = textToFeatures;
449
455
  Classifier.prototype.save = save;
450
456
  Classifier.prototype.getClassifications = getClassifications;
457
+ Classifier.prototype.setOptions = setOptions;
451
458
  Classifier.restore = restore;
452
459
  Classifier.load = load;
453
460
 
@@ -0,0 +1,125 @@
1
+ /*
2
+ Classifier class that provides functionality for training and
3
+ classification
4
+ Copyright (C) 2017 Hugo W.L. ter Doest
5
+
6
+ This program is free software: you can redistribute it and/or modify
7
+ it under the terms of the GNU General Public License as published by
8
+ the Free Software Foundation, either version 3 of the License, or
9
+ (at your option) any later version.
10
+
11
+ This program is distributed in the hope that it will be useful,
12
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
13
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14
+ GNU General Public License for more details.
15
+
16
+ You should have received a copy of the GNU General Public License
17
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
18
+ */
19
+
20
+ var fs = require('fs');
21
+
22
+ var Context = require('./Context');
23
+ var Element = require('./Element');
24
+ var Sample = require('./Sample');
25
+ var Scaler = require('./GISScaler');
26
+ var FeatureSet = require('./FeatureSet');
27
+
28
+ function Classifier(features, sample) {
29
+ if (features) {
30
+ this.features = features;
31
+ }
32
+ else {
33
+ this.features = new featureSet();
34
+ }
35
+ this.features = features;
36
+ if (sample) {
37
+ this.sample = sample;
38
+ }
39
+ else {
40
+ this.sample = new Sample();
41
+ }
42
+ }
43
+
44
+ // Loads a classifier from file.
45
+ // Caveat: feature functions are generated from the sample elements. You need
46
+ // to create your own specialisation of the Element class that can generate
47
+ // your own specific feature functions
48
+ Classifier.prototype.load = function(filename, elementClass, callback) {
49
+ fs.readFile(filename, 'utf8', function(err, data) {
50
+
51
+ if(!err) {
52
+ classifierData = JSON.parse(data);
53
+ var sample = new Sample();
54
+ classifierData.sample.elements.forEach(function(elementData) {
55
+ var elt = new elementClass(elementData.a, new Context(elementData.b.data));
56
+ sample.addElement(elt);
57
+ });
58
+ var featureSet = new FeatureSet();
59
+ sample.generateFeatures(featureSet);
60
+ var classifier = new Classifier(featureSet, sample);
61
+ callback(err, classifier);
62
+ }
63
+ else {
64
+ if(callback) {
65
+ callback(err);
66
+ }
67
+ }
68
+ });
69
+ };
70
+
71
+ Classifier.prototype.save = function(filename, callback) {
72
+ var data = JSON.stringify(this, null, 2);
73
+ var classifier = this;
74
+ fs.writeFile(filename, data, 'utf8', function(err) {
75
+ if(callback) {
76
+ callback(err, err ? null : classifier);
77
+ }
78
+ });
79
+ };
80
+
81
+ Classifier.prototype.addElement = function(x) {
82
+ this.sample.addElement(x);
83
+ };
84
+
85
+ Classifier.prototype.addDocument = function(context, classification, elementClass) {
86
+ Classifier.prototype.addElement(new elementClass(classification, context));
87
+ };
88
+
89
+ Classifier.prototype.train = function(maxIterations, minImprovement, approxExpectation) {
90
+ this.scaler = new Scaler(this.features, this.sample);
91
+ this.p = this.scaler.run(maxIterations, minImprovement, approxExpectation);
92
+ };
93
+
94
+ Classifier.prototype.getClassifications = function(b) {
95
+ var scores = [];
96
+ var that = this;
97
+ this.sample.getClasses().forEach(function(a) {
98
+ var x = new Element(a, b);
99
+ scores.push({
100
+ "label": a,
101
+ "value": that.p.calculateAPriori(x)
102
+ });
103
+ });
104
+ return scores;
105
+ };
106
+
107
+ Classifier.prototype.classify = function(b) {
108
+ var scores = this.getClassifications(b);
109
+ // Sort the scores in an array
110
+ scores.sort(function(a, b) {
111
+ return b.value - a.value;
112
+ });
113
+ // Check if the classifier discriminates
114
+ var min = scores[scores.length - 1].value;
115
+ var max = scores[0].value;
116
+ if (min === max) {
117
+ return "";
118
+ }
119
+ else {
120
+ // Return the highest scoring classes
121
+ return scores[0].label;
122
+ }
123
+ };
124
+
125
+ module.exports = Classifier;