natural 7.1.0 → 8.0.1

package/lib/natural/tokenizers/pegjs_grammar_sentence_tokenizer.txt

@@ -1,140 +0,0 @@
-/*
-Copyright (c) 2020, 2022 Hugo W.L. ter Doest
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
-*/
-
-{
-	const knownAbbreviations = require("../util/abbreviations_en").knownAbbreviations
-}
-
-s
-	= sentences: (Sentences Whitespace / QuotedSentences Whitespace)+ {
-		let result = []
-		sentences.forEach(sent0 => {
-			sent0[0].forEach(sent1 => {
-				result.push(sent1)
-			})
-		})
-		return result
-	}
-
-Sentences
-  = sentences: (Sentence Whitespace) + {
-      //return sentences
-      return(
-          sentences.map(sent => {
-          	//if (sent[1]) {
-          	  sent[0].push(sent[1])
-            //}
-            return sent[0].reduce((accu, str) =>  accu + str).trim()
-          })
-      )
-    }
-
-QuotedSentences
-  = open: OpenSymbol sentences: (Sentence Whitespace) + close: CloseSymbol? {
-      //return sentences
-      let result = sentences.map(sent => {
-      	//if (sent[1]) {
-      	  sent[0].push(sent[1])
-        //}
-        return sent[0].reduce((accu, str) =>  accu + str).trim()
-      })
-			result.unshift(open)
-			if (close) {
-      	result.push(close)
-			}
-      return result
-    }
-
-Sentence
-  = seqs: (TokenSeq / QuotedTokenSeq)+ end: EndOfSentence {
-      let res = seqs.reduce((accu, seq) => accu.concat(seq))
-      res.push(end)
-      return res
-    }
-
-TokenSeq
-  =  tokens: (Token Whitespace)+ {
-       let result = tokens.map(pair => pair[0] + pair[1])
-       return result
-     }
-
-QuotedTokenSeq
-  = open: OpenSymbol tokens: (Token Whitespace)+ end: EndOfSentence close: CloseSymbol {
-  	  let result = tokens.map(pair => pair[0] + pair[1])
-      result.unshift(open)
-      result.push(end)
-      result.push(close)
-      return result
-    }
-
-EndOfSentence
-  = [ \t\n\r.?!]* {
-      return text()
-    }
-
-Whitespace
-  = [ \t\n\r]* {
-      return text()
-    }
-
-Token
-	= t: (URI / Email / Number / Abbreviation / Word) {
-      return t
-    }
-
-// An abbreviation may contain periods
-Abbreviation
-    = word: [^ \t\n\r!?\(\[\}\"\`\)\]\}\"\`0-9\@]+ & {
-        const tmp = word.reduce((accu, elt) => accu + elt)
-        return knownAbbreviations.indexOf(tmp) > -1} {
-    	return text()
-    }
-
-Word
-    = [^ \t\n\r!?\.\(\[\}\)\]\}\`\"0-9\@]+ {
-  	  return text()
-    }
-
-Number
-  = [0-9]+ (.[0-9]+)? CloseSymbol ? {
-      return text()
-    }
-
-Email
-  = [a-z]+[@][a-z]+[.][a-z]+ {
-      return text()
-    }
-
-URI
-  = ("http://" / "https://")? [a-z0-9]+[.]([a-z0-9]+[.])?[a-z0-9]+ ([a-z0-9]+[\/])* {
-    return text()
-  }
-
-OpenSymbol
-  = [\(\[\{\"\'\`\‘] {
-      return text()
-    }
-
-CloseSymbol
-  = [\)\]\}\"\'\`\’] {
-      return text()
-    }