natural 0.5.4 → 0.6.1

package/lib/natural/sentiment/tools/README.md

@@ -0,0 +1,31 @@
+# Tools
+This folder contains some tools for manipulating vocabularies for the sentiment analyzer.
+
+## `sentimentXmlParser`
+Transforms ML-Senticon XML files into JSON files. The JSON file contains a vocabulary that maps words to objects as follows:
+```javascript
+"admirable": {
+  "pos": "a",
+  "pol": "1.0",
+  "std": "0.0"
+}
+```
+Property `pol` is the sentiment of the word.
+
+## `XmlParser4PatternData`
+Transforms vocabularies of the [Pattern project](https://www.clips.uantwerpen.be/pages/pattern) to JSON files. The JSON file contains a vocabulary that maps wordt to objects:
+```javascript
+  "aanraden": {
+    "form": "aanraden",
+    "cornetto_id": "",
+    "cornetto_synset_id": "",
+    "wordnet_id": "",
+    "pos": "VB",
+    "sense": "",
+    "polarity": "0.2",
+    "subjectivity": "0.0",
+    "intensity": "1.0",
+    "confidence": "0.9"
+  }
+```
+Property `polarity` is the sentiment of the word.

package/lib/natural/sentiment/tools/XmlParser4PatternData.js

@@ -0,0 +1,31 @@
+const fs = require('fs');
+const util = require('util');
+var parseString = require('xml2js').parseString;
+
+var baseFolder = "/home/hugo/Workspace/pattern/pattern/text/";
+//var inputFile = baseFolder + "nl/nl-sentiment.xml";
+//var outputFile = baseFolder + "nl/nl-sentiment.json";
+//var inputFile = baseFolder + "fr/fr-sentiment.xml";
+//var outputFile = baseFolder + "fr/fr-sentiment.json";
+//var inputFile = baseFolder + "en/en-sentiment.xml";
+//var outputFile = baseFolder + "en/en-sentiment.json";
+var inputFile = baseFolder + "it/it-sentiment.xml";
+var outputFile = baseFolder + "it/pattern-sentiment-it.json";
+
+
+
+var xml = fs.readFileSync(inputFile , 'utf8');
+var list = {};
+
+parseString(xml, function (err, result) {
+  //console.log(JSON.stringify(result, null, 2));
+  console.log(result.sentiment.word.length);
+    for(var i =0; i < result.sentiment.word.length; i++){
+      console.log(result.sentiment.word[i].$.form);
+      list[result.sentiment.word[i].$.form] = result.sentiment.word[i].$;
+    }
+});
+
+console.log(Object.keys(list).length);
+
+fs.writeFileSync(outputFile, JSON.stringify(list, null, 2));

package/lib/natural/sentiment/tools/sentimentXmlParser.js

@@ -0,0 +1,30 @@
+const fs = require('fs');
+const util = require('util');
+var parseString = require('xml2js').parseString;
+
+var xml = fs.readFileSync('senticon.ca.xml', 'utf8');
+
+var list = {};
+
+parseString(xml, function (err, result) {
+    for(var i =0; i < result.senticon.layer.length; i++){
+        for(var j = 0; j < result.senticon.layer[i].positive[0].lemma.length; j++){
+            //console.log(util.inspect(result.senticon.layer[i].negative[0].lemma[j], false, null))
+            list[result.senticon.layer[i].positive[0].lemma[j]._] = result.senticon.layer[i].positive[0].lemma[j].$;
+        }
+        for(var n = 0; n < result.senticon.layer[i].negative[0].lemma.length; n++){
+            list[result.senticon.layer[i].negative[0].lemma[n]._] = result.senticon.layer[i].negative[0].lemma[n].$;
+        }
+    }
+});
+
+var listTrimmed = {};
+
+for(var token in list){
+    listTrimmed[token.replace(/ /gi, "").replace(/_/gi, " ")] = list[token]
+}
+console.log(listTrimmed);
+//console.log(util.inspect(list, false, null))
+console.log(Object.keys(listTrimmed).length);
+
+fs.writeFileSync('senticon_ca.json', JSON.stringify(listTrimmed, null, 2));

package/lib/natural/stemmers/indonesian/base_stemmer_id.js

@@ -0,0 +1,86 @@
+/*
+Copyright (c) 2017, Alif Bhaskoro, Andy Librian, R. Kukuh (Reimplemented from https://github.com/sastrawi/sastrawi)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+var stopwords = require('../../util/stopwords_id');
+var Tokenizer = require('../../tokenizers/aggressive_tokenizer_id');
+
+module.exports = function() {
+    var stemmer = this;
+
+    stemmer.stem = function(token) {
+        return token;
+    };
+
+    stemmer.addStopWord = function(stopWord) {
+        stopwords.words.push(stopWord);
+    };
+
+    stemmer.addStopWords = function(moreStopWords) {
+        stopwords.words = stopwords.words.concat(moreStopWords);
+    };
+
+    stemmer.removeStopWord = function(stopWord) {
+        this.removeStopWords([stopWord])
+    };
+
+    stemmer.removeStopWords = function(moreStopWords) {
+        moreStopWords.forEach(function(stopWord){
+            var idx = stopwords.words.indexOf(stopWord);
+            if (idx >= 0) {
+                stopwords.words.splice(idx, 1);
+            }
+        });
+
+    };
+
+
+    stemmer.tokenizeAndStem = function(text, keepStops) {
+        var stemmedTokens = [];
+        var lowercaseText = text.toLowerCase();
+        var tokens = new Tokenizer().tokenize(lowercaseText);
+
+        if (keepStops) {
+            tokens.forEach(function(token) {
+                stemmedTokens.push(stemmer.stem(token));
+            });
+        }
+
+        else {
+            tokens.forEach(function(token) {
+                if (stopwords.words.indexOf(token) == -1)
+                    stemmedTokens.push(stemmer.stem(token));
+            });
+        }
+
+        return stemmedTokens;
+    };
+
+    stemmer.attach = function() {
+        String.prototype.stem = function() {
+            return stemmer.stem(this);
+        };
+
+        String.prototype.tokenizeAndStem = function(keepStops) {
+            return stemmer.tokenizeAndStem(this, keepStops);
+        };
+    };
+}