natural 6.8.1 → 6.9.1

package/lib/natural/stemmers/index.d.ts

@@ -40,6 +40,7 @@ export let PorterStemmerIt: Stemmer
 export let PorterStemmerNl: Stemmer
 export let PorterStemmerNo: Stemmer
 export let PorterStemmerPt: Stemmer
+export let PorterStemmerUk: Stemmer
 export let PorterStemmerRu: Stemmer
 export let PorterStemmerSv: Stemmer
 export let StemmerId: Stemmer

package/lib/natural/stemmers/index.js

@@ -27,6 +27,7 @@ exports.PorterStemmerFa = require('./porter_stemmer_fa')
 exports.PorterStemmerFr = require('./porter_stemmer_fr')
 exports.CarryStemmerFr = require('./Carry')
 exports.PorterStemmerDe = require('./porter_stemmer_de')
+exports.PorterStemmerUk = require('./porter_stemmer_uk')
 exports.PorterStemmerRu = require('./porter_stemmer_ru')
 exports.PorterStemmerEs = require('./porter_stemmer_es')
 exports.PorterStemmerIt = require('./porter_stemmer_it')

package/lib/natural/stemmers/porter_stemmer_uk.js

@@ -0,0 +1,129 @@
+'use strict'
+
+const Stemmer = require('./stemmer_uk')
+
+const PorterStemmer = new Stemmer()
+module.exports = PorterStemmer
+
+function attemptReplacePatterns (token, patterns) {
+  let replacement = null
+  let i = 0; let isReplaced = false
+  while ((i < patterns.length) && !isReplaced) {
+    if (patterns[i][0].test(token)) {
+      replacement = token.replace(patterns[i][0], patterns[i][1])
+      isReplaced = true
+    }
+    i++
+  }
+  return replacement
+}
+
+function perfectiveGerund (token) {
+  const result = attemptReplacePatterns(token, [
+    [/[ая]в(ши|шись)$/g, ''],
+    [/(ив|ивши|ившись)$/g, '']
+  ])
+  return result
+}
+
+function adjectival (token) {
+  let result = adjective(token)
+  if (result != null) {
+    const pariticipleResult = participle(result)
+    result = pariticipleResult || result
+  }
+  return result
+}
+
+function adjective (token) {
+  const result = attemptReplacePatterns(token, [
+    [/(ими|ій|ий|а|е|ова|ове|ів|є|їй|єє|еє|я|ім|ем|им|ім|их|іх|ою|йми|іми|у|ю|ого|ому|ої)$/g, '']
+  ])
+  return result
+}
+
+function participle (token) { // edit
+  const result = attemptReplacePatterns(token, [
+    [/(ий|ого|ому|им|ім|а|ій|у|ою|ій|і|их|йми|их)$/g, '']
+  ])
+  return result
+}
+
+function reflexive (token) {
+  const result = attemptReplacePatterns(token, [
+    [/(с[яьи])$/g, '']
+  ])
+  return result
+}
+
+function verb (token) {
+  const result = attemptReplacePatterns(token, [
+    [/(сь|ся|ив|ать|ять|у|ю|ав|али|учи|ячи|вши|ши|е|ме|ати|яти|є)$/g, '']
+  ])
+  return result
+}
+
+function noun (token) {
+  const result = attemptReplacePatterns(token, [
+    [/(а|ев|ов|е|ями|ами|еи|и|ей|ой|ий|й|иям|ям|ием|ем|ам|ом|о|у|ах|иях|ях|ы|ь|ию|ью|ю|ия|ья|я|і|ові|ї|ею|єю|ою|є|еві|ем|єм|ів|їв|ю)$/g, '']
+  ])
+  return result
+}
+
+function superlative (token) {
+  const result = attemptReplacePatterns(token, [
+    [/(ейш|ейше)$/g, '']
+  ])
+  return result
+}
+
+function derivational (token) {
+  const result = attemptReplacePatterns(token, [
+    [/[^аеиоуюяіїє][аеиоуюяіїє]+[^аеиоуюяіїє]+[аеиоуюяіїє].*(?<=о)сть?$/g, '']
+  ])
+  return result
+}
+
+// perform full stemming algorithm on a single word
+PorterStemmer.stem = function (token) {
+  token = token.toLowerCase()
+  const volwesRegexp = /^(.*?[аеиоуюяіїє])(.*)$/g
+  let RV = volwesRegexp.exec(token)
+  if (!RV || RV.length < 3) {
+    return token
+  }
+  const head = RV[1]
+  RV = RV[2]
+  volwesRegexp.lastIndex = 0
+  const R2 = volwesRegexp.exec(RV)
+  let result = perfectiveGerund(RV)
+  if (result === null) {
+    const resultReflexive = reflexive(RV) || RV
+    result = adjectival(resultReflexive)
+    if (result === null) {
+      result = verb(resultReflexive)
+      if (result === null) {
+        result = noun(resultReflexive)
+        if (result === null) {
+          result = resultReflexive
+        }
+      }
+    }
+  }
+  result = result.replace(/и$/g, '')
+  let derivationalResult = result
+  if (R2 && R2[2]) {
+    derivationalResult = derivational(R2[2])
+    if (derivationalResult != null) {
+      derivationalResult = derivational(result)
+    } else {
+      derivationalResult = result
+    }
+  }
+
+  let superlativeResult = superlative(derivationalResult) || derivationalResult
+
+  superlativeResult = superlativeResult.replace(/(н)н/g, '$1')
+  superlativeResult = superlativeResult.replace(/ь$/g, '')
+  return head + superlativeResult
+}

package/lib/natural/stemmers/stemmer_uk.js

@@ -0,0 +1,28 @@
+'use strict'
+
+const stopwords = require('../util/stopwords_uk')
+const Tokenizer = require('../tokenizers/aggressive_tokenizer_uk')
+
+module.exports = function () {
+  const stemmer = this
+
+  stemmer.stem = function (token) {
+    return token
+  }
+
+  stemmer.tokenizeAndStem = function (text, keepStops) {
+    const stemmedTokens = []
+
+    new Tokenizer().tokenize(text).forEach(function (token) {
+      if (keepStops || stopwords.words.indexOf(token) === -1) {
+        let resultToken = token.toLowerCase()
+        if (resultToken.match(/[а-яґєії0-9]+/gi)) {
+          resultToken = stemmer.stem(resultToken)
+        }
+        stemmedTokens.push(resultToken)
+      }
+    })
+
+    return stemmedTokens
+  }
+}

package/lib/natural/tokenizers/aggressive_tokenizer_uk.js

@@ -0,0 +1,25 @@
+'use strict'
+
+const Tokenizer = require('./tokenizer')
+const util = require('util')
+
+const AggressiveTokenizer = function () {
+  Tokenizer.call(this)
+}
+
+util.inherits(AggressiveTokenizer, Tokenizer)
+
+module.exports = AggressiveTokenizer
+
+AggressiveTokenizer.prototype.withoutEmpty = function (array) {
+  return array.filter(function (a) { return a })
+}
+
+AggressiveTokenizer.prototype.clearText = function (text) {
+  return text.replace(/[^a-zа-яґєії0-9]/gi, ' ').replace(/[\s\n]+/g, ' ').trim()
+}
+
+AggressiveTokenizer.prototype.tokenize = function (text) {
+  // break a string up into an array of tokens by anything non-word
+  return this.withoutEmpty(this.clearText(text).split(' '))
+}

package/lib/natural/tokenizers/sentence_tokenizer.js

@@ -34,7 +34,8 @@ util.inherits(SentenceTokenizer, Tokenizer)
 
 SentenceTokenizer.prototype.tokenize = function (text) {
   // break string up in to sentences based on punctation and quotation marks
-  let tokens = text.match(/(?<=\s+|^)["'‘“'"[({⟨]?(.*?[.?!…]|[^.?!…]+)(\s[.?!…])*["'’”'"\])}⟩]?(?=\s+|$)/g)
+  // let tokens = text.match(/(?<=\s+|^)["'‘“'"[({⟨]?.*?[.?!…](\s[.?!…])*["'’”'"\])}⟩]?(?=\s+|$)/g)
+  let tokens = text.match(/(?<=\s+|^)["'‘“'"[({⟨]?(.*?[.?!…]|.+)(\s[.?!…])*["'’”'"\])}⟩]?(?=\s+|$)/g)
 
   DEBUG && console.log('SentenceTokenizer.tokenize: ' + tokens)
 

package/lib/natural/util/stopwords_uk.js

@@ -0,0 +1,21 @@
+'use strict'
+
+// a list of commonly used words that have little meaning and can be excluded
+// from analysis.
+const words = [
+  'о', 'після', 'всі', 'також', 'і', 'інші', 'все', 'як', 'в', 'бути',
+  'тому', 'був', 'до', 'є', 'між', 'вся', 'але', 'від', 'іди', 'можу',
+  'підійди', 'міг', 'робив', 'роблю', 'кожен', 'для', 'звідки', 'мав',
+  'він', 'має', 'її', 'тут', 'його', 'якщо', 'у', 'воно', 'за',
+  'робити', 'багато', 'може бути', 'самий', 'повинен',
+  'мій', 'ніколи', 'зараз', 'із', 'на', 'тільки', 'або', 'наше', 'поза',
+  'кінець', 'сказав', 'сказала', 'бачив', 'з',
+  'трохи', 'все ще', 'так', 'потім', 'той', 'їх', 'там', 'цей', 'вони', 'те',
+  'через', 'теж', 'під', 'над', 'дуже', 'був', 'добре',
+  'що', 'де', 'який', 'доки', 'хто', 'хотів би',
+  'а', 'б', 'в', 'г', 'ґ', 'д', 'е', 'є', 'ж', 'з', 'и', 'і', 'й', 'ї', 'к',
+  'л', 'м', 'н', 'o', 'п', 'р', 'с', 'т', 'у', 'ф', 'х', 'ц', 'ч', 'ш', 'щ',
+  'ь', 'ю', 'я', '$', '1', '2', '3', '4', '5', '6', '7', '8', '9', '0', '_']
+
+// tell the world about the noise words.
+exports.words = words

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "natural",
   "description": "General natural language (tokenizing, stemming (English, Russian, Spanish), part-of-speech tagging, sentiment analysis, classification, inflection, phonetics, tfidf, WordNet, jaro-winkler, Levenshtein distance, Dice's Coefficient) facilities for node.",
-  "version": "6.8.1",
+  "version": "6.9.1",
   "homepage": "https://github.com/NaturalNode/natural",
   "repository": {
     "type": "git",