natural 8.0.1 → 8.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -41,13 +41,18 @@ function escapeRegExp (string) {
|
|
|
41
41
|
}
|
|
42
42
|
|
|
43
43
|
class SentenceTokenizer extends Tokenizer {
|
|
44
|
-
constructor (abbreviations) {
|
|
44
|
+
constructor (abbreviations, trimSentences) {
|
|
45
45
|
super()
|
|
46
46
|
if (abbreviations) {
|
|
47
47
|
this.abbreviations = abbreviations
|
|
48
48
|
} else {
|
|
49
49
|
this.abbreviations = []
|
|
50
50
|
}
|
|
51
|
+
if (trimSentences === undefined) {
|
|
52
|
+
this.trimSentences = true
|
|
53
|
+
} else {
|
|
54
|
+
this.trimSentences = trimSentences
|
|
55
|
+
}
|
|
51
56
|
this.replacementMap = null
|
|
52
57
|
this.replacementCounter = 0
|
|
53
58
|
}
|
|
@@ -181,7 +186,7 @@ class SentenceTokenizer extends Tokenizer {
|
|
|
181
186
|
const trimmedSentences = this.trim(newSentences)
|
|
182
187
|
DEBUG && console.log('Phase 7: trimming array of empty sentences: ' + JSON.stringify(trimmedSentences))
|
|
183
188
|
|
|
184
|
-
const trimmedSentences2 = trimmedSentences.map(sent => sent.trim())
|
|
189
|
+
const trimmedSentences2 = trimmedSentences.map(sent => this.trimSentences ? sent.trim() : sent)
|
|
185
190
|
DEBUG && console.log('Phase 8: trimming sentences from surrounding whitespace: ' + JSON.stringify(trimmedSentences2))
|
|
186
191
|
DEBUG && console.log('---End of sentence tokenization--------------------------')
|
|
187
192
|
DEBUG && console.log('---Replacement map---------------------------------------')
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "natural",
|
|
3
3
|
"description": "General natural language (tokenizing, stemming (English, Russian, Spanish), part-of-speech tagging, sentiment analysis, classification, inflection, phonetics, tfidf, WordNet, jaro-winkler, Levenshtein distance, Dice's Coefficient) facilities for node.",
|
|
4
|
-
"version": "8.0
|
|
4
|
+
"version": "8.1.0",
|
|
5
5
|
"homepage": "https://github.com/NaturalNode/natural",
|
|
6
6
|
"repository": {
|
|
7
7
|
"type": "git",
|