natural 6.12.0 → 7.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (108) hide show
  1. package/.env +2 -1
  2. package/.eslintrc.json +13 -0
  3. package/.github/workflows/node.js.yml +3 -6
  4. package/lib/natural/analyzers/SenType.ts +30 -0
  5. package/lib/natural/analyzers/index.d.ts +6 -10
  6. package/lib/natural/brill_pos_tagger/index.d.ts +23 -14
  7. package/lib/natural/brill_pos_tagger/index.js +1 -1
  8. package/lib/natural/classifiers/bayes_classifier.js +1 -1
  9. package/lib/natural/classifiers/classifier.js +11 -34
  10. package/lib/natural/classifiers/classifier_train_parallel.js +7 -5
  11. package/lib/natural/classifiers/index.d.ts +36 -15
  12. package/lib/natural/classifiers/maxent/Classifier.js +5 -3
  13. package/lib/natural/distance/index.d.ts +11 -14
  14. package/lib/natural/inflectors/count_inflector.js +18 -21
  15. package/lib/natural/inflectors/fr/count_inflector.js +18 -19
  16. package/lib/natural/inflectors/index.d.ts +12 -3
  17. package/lib/natural/inflectors/index.js +3 -0
  18. package/lib/natural/ngrams/index.d.ts +5 -5
  19. package/lib/natural/normalizers/index.d.ts +24 -1
  20. package/lib/natural/normalizers/index.js +4 -1
  21. package/lib/natural/normalizers/normalizer_ja.js +109 -72
  22. package/lib/natural/phonetics/dm_soundex.js +52 -51
  23. package/lib/natural/phonetics/double_metaphone.js +397 -399
  24. package/lib/natural/phonetics/index.d.ts +42 -12
  25. package/lib/natural/phonetics/metaphone.js +134 -157
  26. package/lib/natural/phonetics/phonetic.js +5 -29
  27. package/lib/natural/phonetics/soundex.js +45 -54
  28. package/lib/natural/sentiment/index.d.ts +6 -6
  29. package/lib/natural/spellcheck/index.d.ts +2 -0
  30. package/lib/natural/stemmers/index.d.ts +43 -1
  31. package/lib/natural/stemmers/index.js +1 -0
  32. package/lib/natural/stemmers/token.js +131 -132
  33. package/lib/natural/tfidf/index.d.ts +7 -7
  34. package/lib/natural/tfidf/tfidf.js +7 -1
  35. package/lib/natural/tokenizers/index.d.ts +2 -2
  36. package/lib/natural/trie/index.d.ts +1 -1
  37. package/lib/natural/util/index.d.ts +8 -0
  38. package/lib/natural/util/index.js +2 -0
  39. package/lib/natural/util/storage/index.d.ts +39 -0
  40. package/lib/natural/util/storage/index.js +24 -0
  41. package/package.json +22 -9
  42. package/ts_spec/MaxEntClassifier_spec.ts +130 -0
  43. package/ts_spec/SentimentAnalyzer_spec.ts +232 -0
  44. package/ts_spec/WordPunctTokenizer_spec.ts +57 -0
  45. package/ts_spec/aggressive_tokenizer_de_spec.ts +36 -0
  46. package/ts_spec/aggressive_tokenizer_es_spec.ts +39 -0
  47. package/ts_spec/aggressive_tokenizer_fr_spec.ts +111 -0
  48. package/ts_spec/aggressive_tokenizer_hi_spec.ts +70 -0
  49. package/ts_spec/aggressive_tokenizer_nl_spec.ts +39 -0
  50. package/ts_spec/aggressive_tokenizer_pt_spec.ts +48 -0
  51. package/ts_spec/aggressive_tokenizer_spec.ts +57 -0
  52. package/ts_spec/aggressive_tokenizer_sv_spec.ts +52 -0
  53. package/ts_spec/aggressive_tokenizer_vi_spec.ts +48 -0
  54. package/ts_spec/bayes_classifier_spec.ts +165 -0
  55. package/ts_spec/brill_pos_tagger_spec.ts +79 -0
  56. package/ts_spec/brill_pos_trainer_spec.ts +107 -0
  57. package/ts_spec/carry_stemmer_fr_spec.ts +97 -0
  58. package/ts_spec/classifier_spec.ts +104 -0
  59. package/ts_spec/count_inflector_fr_spec.ts +47 -0
  60. package/ts_spec/count_inflector_spec.ts +76 -0
  61. package/ts_spec/damerau_levenshtein_spec.ts +53 -0
  62. package/ts_spec/dice_coefficient_spec.ts +37 -0
  63. package/ts_spec/double_metaphone_spec.ts +739 -0
  64. package/ts_spec/hamming_distance_spec.ts +66 -0
  65. package/ts_spec/jaro-winkler_spec.ts +79 -0
  66. package/ts_spec/lancaster_stemmer_spec.ts +63 -0
  67. package/ts_spec/levenshtein_spec.ts +107 -0
  68. package/ts_spec/logistic_regression_classifier_spec.ts +93 -0
  69. package/ts_spec/longest_path_tree_spec.ts +111 -0
  70. package/ts_spec/metaphone_spec.ts +258 -0
  71. package/ts_spec/ngram_spec.ts +145 -0
  72. package/ts_spec/ngram_zh_spec.ts +124 -0
  73. package/ts_spec/normalizer_ja_spec.ts +165 -0
  74. package/ts_spec/normalizer_no_spec.ts +112 -0
  75. package/ts_spec/normalizer_spec.ts +141 -0
  76. package/ts_spec/noun_inflector_fr_spec.ts +472 -0
  77. package/ts_spec/noun_inflector_ja_spec.ts +116 -0
  78. package/ts_spec/noun_inflector_spec.ts +369 -0
  79. package/ts_spec/orthography_tokenizer_spec.ts +47 -0
  80. package/ts_spec/porter_stemmer_de_spec.ts +53 -0
  81. package/ts_spec/porter_stemmer_es_spec.ts +63 -0
  82. package/ts_spec/porter_stemmer_fr_spec.ts +103 -0
  83. package/ts_spec/porter_stemmer_it_spec.ts +56 -0
  84. package/ts_spec/porter_stemmer_nl_spec.ts +73 -0
  85. package/ts_spec/porter_stemmer_no_spec.ts +148 -0
  86. package/ts_spec/porter_stemmer_pt_spec.ts +50 -0
  87. package/ts_spec/porter_stemmer_ru_spec.ts +67 -0
  88. package/ts_spec/porter_stemmer_spec.ts +202 -0
  89. package/ts_spec/porter_stemmer_sv_spec.ts +41 -0
  90. package/ts_spec/present_verb_inflector_spec.ts +127 -0
  91. package/ts_spec/remove_diacritics_spec.ts +30 -0
  92. package/ts_spec/sentence_analyzer_spec.ts +293 -0
  93. package/ts_spec/sentence_tokenizer_parser_spec.ts +200 -0
  94. package/ts_spec/sentence_tokenizer_spec.ts +189 -0
  95. package/ts_spec/shortest_path_tree_spec.ts +141 -0
  96. package/ts_spec/soundex_spec.ts +162 -0
  97. package/ts_spec/spellcheck_spec.ts +112 -0
  98. package/ts_spec/stemmer_id_spec.ts +374 -0
  99. package/ts_spec/stemmer_ja_spec.ts +69 -0
  100. package/ts_spec/stemmer_token_spec.ts +163 -0
  101. package/ts_spec/tfidf_spec.ts +275 -0
  102. package/ts_spec/tokenizer_case_spec.ts +242 -0
  103. package/ts_spec/tokenizer_ja_spec.ts +63 -0
  104. package/ts_spec/transliterator_ja_spec.ts +105 -0
  105. package/ts_spec/treebank_word_tokenizer_spec.ts +38 -0
  106. package/ts_spec/trie_spec.ts +197 -0
  107. package/tsconfig.json +5 -3
  108. package/example.txt +0 -1
@@ -22,141 +22,140 @@ THE SOFTWARE.
22
22
 
23
23
  'use strict'
24
24
 
25
- module.exports = (function () {
26
- /**
27
- * Stemmer token constructor.
28
- *
29
- * @param {String} string Token string.
30
- */
31
- const Token = function (string) {
32
- this.vowels = ''
33
- this.regions = {}
34
- this.string = string
35
- this.original = string
25
+ /**
26
+ * Stemmer token constructor.
27
+ *
28
+ * @param {String} string Token string.
29
+ */
30
+ function Token (string) {
31
+ this.vowels = ''
32
+ this.regions = {}
33
+ this.string = string
34
+ this.original = string
35
+ }
36
+
37
+ /**
38
+ * Set vowels.
39
+ *
40
+ * @param {String|Array} vowels List of vowels.
41
+ * @return {Token} Token instance.
42
+ */
43
+ Token.prototype.usingVowels = function (vowels) {
44
+ this.vowels = vowels
45
+ return this
46
+ }
47
+
48
+ /**
49
+ * Marks a region by defining its starting index or providing a callback
50
+ * function that does.
51
+ *
52
+ * @param {String} region Region name.
53
+ * @param {Array|Number} args Callback arguments or region start index.
54
+ * @param {Function} callback Function that determines the start index (optional).
55
+ * @param {Object} context Callback context (optional, defaults to this).
56
+ * @return {Token} Token instance.
57
+ */
58
+ Token.prototype.markRegion = function (region, args, callback, context) {
59
+ if (typeof callback === 'function') {
60
+ // this.regions[region] = callback.apply(context || this, [].concat(args))
61
+ this.regions[region] = callback.apply(context || this, [].concat(args))
62
+ } else if (!isNaN(args)) {
63
+ this.regions[region] = args
36
64
  }
37
65
 
38
- /**
39
- * Set vowels.
40
- *
41
- * @param {String|Array} vowels List of vowels.
42
- * @return {Token} Token instance.
43
- */
44
- Token.prototype.usingVowels = function (vowels) {
45
- this.vowels = vowels
46
- return this
47
- }
48
-
49
- /**
50
- * Marks a region by defining its starting index or providing a callback
51
- * function that does.
52
- *
53
- * @param {String} region Region name.
54
- * @param {Array|Number} args Callback arguments or region start index.
55
- * @param {Function} callback Function that determines the start index (optional).
56
- * @param {Object} context Callback context (optional, defaults to this).
57
- * @return {Token} Token instance.
58
- */
59
- Token.prototype.markRegion = function (region, args, callback, context) {
60
- if (typeof callback === 'function') {
61
- this.regions[region] = callback.apply(context || this, [].concat(args))
62
- } else if (!isNaN(args)) {
63
- this.regions[region] = args
66
+ return this
67
+ }
68
+
69
+ /**
70
+ * Replaces all instances of a string with another.
71
+ *
72
+ * @param {String} find String to be replaced.
73
+ * @param {String} replace Replacement string.
74
+ * @return {Token} Token instance.
75
+ */
76
+ Token.prototype.replaceAll = function (find, replace) {
77
+ this.string = this.string.split(find).join(replace)
78
+ return this
79
+ }
80
+
81
+ /**
82
+ * Replaces the token suffix if in a region.
83
+ *
84
+ * @param {String} suffix Suffix to replace.
85
+ * @param {String} replace Replacement string.
86
+ * @param {String} region Region name.
87
+ * @return {Token} Token instance.
88
+ */
89
+ Token.prototype.replaceSuffixInRegion = function (suffix, replace, region) {
90
+ const suffixes = [].concat(suffix)
91
+ for (let i = 0; i < suffixes.length; i++) {
92
+ if (this.hasSuffixInRegion(suffixes[i], region)) {
93
+ this.string = this.string.slice(0, -suffixes[i].length) + replace
94
+ return this
64
95
  }
65
-
66
- return this
67
- }
68
-
69
- /**
70
- * Replaces all instances of a string with another.
71
- *
72
- * @param {String} find String to be replaced.
73
- * @param {String} replace Replacement string.
74
- * @return {Token} Token instance.
75
- */
76
- Token.prototype.replaceAll = function (find, replace) {
77
- this.string = this.string.split(find).join(replace)
78
- return this
79
96
  }
80
-
81
- /**
82
- * Replaces the token suffix if in a region.
83
- *
84
- * @param {String} suffix Suffix to replace.
85
- * @param {String} replace Replacement string.
86
- * @param {String} region Region name.
87
- * @return {Token} Token instance.
88
- */
89
- Token.prototype.replaceSuffixInRegion = function (suffix, replace, region) {
90
- const suffixes = [].concat(suffix)
91
- for (let i = 0; i < suffixes.length; i++) {
92
- if (this.hasSuffixInRegion(suffixes[i], region)) {
93
- this.string = this.string.slice(0, -suffixes[i].length) + replace
94
- return this
95
- }
96
- }
97
- return this
97
+ return this
98
+ }
99
+
100
+ /**
101
+ * Determines whether the token has a vowel at the provided index.
102
+ *
103
+ * @param {Integer} index Character index.
104
+ * @return {Boolean} Whether the token has a vowel at the provided index.
105
+ */
106
+ Token.prototype.hasVowelAtIndex = function (index) {
107
+ return this.vowels.indexOf(this.string[index]) !== -1
108
+ }
109
+
110
+ /**
111
+ * Finds the next vowel in the token.
112
+ *
113
+ * @param {Integer} start Starting index offset.
114
+ * @return {Integer} Vowel index, or the end of the string.
115
+ */
116
+ Token.prototype.nextVowelIndex = function (start) {
117
+ let index = (start >= 0 && start < this.string.length) ? start : this.string.length
118
+ while (index < this.string.length && !this.hasVowelAtIndex(index)) {
119
+ index++
98
120
  }
99
-
100
- /**
101
- * Determines whether the token has a vowel at the provided index.
102
- *
103
- * @param {Integer} index Character index.
104
- * @return {Boolean} Whether the token has a vowel at the provided index.
105
- */
106
- Token.prototype.hasVowelAtIndex = function (index) {
107
- return this.vowels.indexOf(this.string[index]) !== -1
121
+ return index
122
+ }
123
+
124
+ /**
125
+ * Finds the next consonant in the token.
126
+ *
127
+ * @param {Integer} start Starting index offset.
128
+ * @return {Integer} Consonant index, or the end of the string.
129
+ */
130
+ Token.prototype.nextConsonantIndex = function (start) {
131
+ let index = (start >= 0 && start < this.string.length) ? start : this.string.length
132
+ while (index < this.string.length && this.hasVowelAtIndex(index)) {
133
+ index++
108
134
  }
109
-
110
- /**
111
- * Finds the next vowel in the token.
112
- *
113
- * @param {Integer} start Starting index offset.
114
- * @return {Integer} Vowel index, or the end of the string.
115
- */
116
- Token.prototype.nextVowelIndex = function (start) {
117
- let index = (start >= 0 && start < this.string.length) ? start : this.string.length
118
- while (index < this.string.length && !this.hasVowelAtIndex(index)) {
119
- index++
120
- }
121
- return index
122
- }
123
-
124
- /**
125
- * Finds the next consonant in the token.
126
- *
127
- * @param {Integer} start Starting index offset.
128
- * @return {Integer} Consonant index, or the end of the string.
129
- */
130
- Token.prototype.nextConsonantIndex = function (start) {
131
- let index = (start >= 0 && start < this.string.length) ? start : this.string.length
132
- while (index < this.string.length && this.hasVowelAtIndex(index)) {
133
- index++
134
- }
135
- return index
136
- }
137
-
138
- /**
139
- * Determines whether the token has the provided suffix.
140
- * @param {String} suffix Suffix to match.
141
- * @return {Boolean} Whether the token string ends in suffix.
142
- */
143
- Token.prototype.hasSuffix = function (suffix) {
144
- return this.string.slice(-suffix.length) === suffix
145
- }
146
-
147
- /**
148
- * Determines whether the token has the provided suffix within the specified
149
- * region.
150
- *
151
- * @param {String} suffix Suffix to match.
152
- * @param {String} region Region name.
153
- * @return {Boolean} Whether the token string ends in suffix.
154
- */
155
- Token.prototype.hasSuffixInRegion = function (suffix, region) {
156
- const regionStart = this.regions[region] || 0
157
- const suffixStart = this.string.length - suffix.length
158
- return this.hasSuffix(suffix) && suffixStart >= regionStart
159
- }
160
-
161
- return Token
162
- })()
135
+ return index
136
+ }
137
+
138
+ /**
139
+ * Determines whether the token has the provided suffix.
140
+ * @param {String} suffix Suffix to match.
141
+ * @return {Boolean} Whether the token string ends in suffix.
142
+ */
143
+ Token.prototype.hasSuffix = function (suffix) {
144
+ return this.string.slice(-suffix.length) === suffix
145
+ }
146
+
147
+ /**
148
+ * Determines whether the token has the provided suffix within the specified
149
+ * region.
150
+ *
151
+ * @param {String} suffix Suffix to match.
152
+ * @param {String} region Region name.
153
+ * @return {Boolean} Whether the token string ends in suffix.
154
+ */
155
+ Token.prototype.hasSuffixInRegion = function (suffix, region) {
156
+ const regionStart = this.regions[region] || 0
157
+ const suffixStart = this.string.length - suffix.length
158
+ return this.hasSuffix(suffix) && suffixStart >= regionStart
159
+ }
160
+
161
+ module.exports = Token
@@ -23,9 +23,9 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
23
23
  THE SOFTWARE.
24
24
  */
25
25
 
26
- import { Tokenizer } from '../tokenizers'
26
+ import type { Tokenizer } from '../tokenizers'
27
27
 
28
- declare type TfIdfCallback = (i: number, measure: number, key?: string) => void
28
+ declare type TfIdfCallback = (i: number, measure: number, key?: string | Record<string, any>) => void
29
29
 
30
30
  declare interface TfIdfTerm {
31
31
  term: string
@@ -34,15 +34,15 @@ declare interface TfIdfTerm {
34
34
  tfidf: number
35
35
  }
36
36
 
37
- declare type TfIdfDocument = { __key: string | undefined } | { [key: string]: number | undefined }
37
+ declare type TfIdfDocument = Record<string, number>
38
38
 
39
39
  export class TfIdf {
40
- documents: TfIdfDocument | undefined
41
- _idfCache: { [key: string]: number } | undefined
40
+ documents: TfIdfDocument[]
41
+ _idfCache: Record<string, number>
42
42
 
43
- constructor (deserialized?: { documents: TfIdfDocument[] })
43
+ constructor (deserialized?: Record<string, unknown>)
44
44
  idf (term: string, force?: boolean): number
45
- addDocument (document: string | string[], key?: string, restoreCache?: boolean): void
45
+ addDocument (document: string | string[] | Record<string, string>, key?: Record<string, any> | any, restoreCache?: boolean): void
46
46
  addFileSync (path: string, encoding?: string, key?: string, restoreCache?: boolean): void
47
47
  tfidf (terms: string | string[], d: number): number
48
48
  tfidfs (terms: string | string[], callback?: TfIdfCallback): number[]
@@ -26,6 +26,8 @@ let tokenizer = new Tokenizer()
26
26
  let stopwords = require('../util/stopwords').words
27
27
  const fs = require('fs')
28
28
 
29
+ // Returns a frequency map of word to frequency
30
+ // Key is the document key and stored in the map that is returned as __keys
29
31
  function buildDocument (text, key) {
30
32
  let stopOut
31
33
 
@@ -87,6 +89,8 @@ class TfIdf {
87
89
  return document[term] ? document[term] : 0
88
90
  }
89
91
 
92
+ // Returns the inverse document frequency of the term
93
+ // If force is true the cache will be invalidated and recomputed
90
94
  idf (term, force) {
91
95
  // Lookup the term in the New term-IDF caching,
92
96
  // this will cut search times down exponentially on large document sets.
@@ -95,10 +99,12 @@ class TfIdf {
95
99
  return this._idfCache[term]
96
100
  }
97
101
 
102
+ // Count the number of documents that contain the term
98
103
  const docsWithTerm = this.documents.reduce(function (count, document) {
99
104
  return count + (documentHasTerm(term, document) ? 1 : 0)
100
105
  }, 0)
101
106
 
107
+ // Compute the inverse document frequency
102
108
  const idf = 1 + Math.log((this.documents.length) / (1 + docsWithTerm))
103
109
 
104
110
  // Add the idf to the term cache and return it
@@ -169,7 +175,7 @@ class TfIdf {
169
175
  if (this.documents[d]) {
170
176
  if (term !== '__key') {
171
177
  terms.push({
172
- term: term,
178
+ term,
173
179
  tf: TfIdf.tf(term, _this.documents[d]),
174
180
  idf: _this.idf(term),
175
181
  tfidf: _this.tfidf(term, d)
@@ -92,7 +92,7 @@ export class AggressiveTokenizer extends Tokenizer {
92
92
  }
93
93
 
94
94
  export class CaseTokenizer extends Tokenizer {
95
- tokenize (text: string): string[]
95
+ tokenize (text: string, preserveApostrophe?: boolean): string[]
96
96
  }
97
97
 
98
98
  declare interface RegexTokenizerOptions {
@@ -103,7 +103,7 @@ declare interface RegexTokenizerOptions {
103
103
 
104
104
  export class RegexpTokenizer extends Tokenizer {
105
105
  constructor (opts?: RegexTokenizerOptions)
106
- tokenize (text: string): string[] | null
106
+ tokenize (text: string): string[]
107
107
  }
108
108
 
109
109
  declare interface OrthographyTokenizerOptions extends RegexTokenizerOptions {
@@ -27,7 +27,7 @@ export class Trie {
27
27
  addString (string: string): boolean
28
28
  addStrings (list: string[]): void
29
29
  contains (string: string): boolean
30
- findPrefix (search: string): string[]
30
+ findPrefix (search: string): Array<string | null>
31
31
  findMatchesOnPath (search: string): string[]
32
32
  keysWithPrefix (prefix: string): string[]
33
33
  getSize (): number
@@ -70,3 +70,11 @@ export class LongestPathTree {
70
70
  hasPathTo (vertex: number): boolean
71
71
  pathTo (vertex: number): number[]
72
72
  }
73
+
74
+ export class Topological {
75
+ constructor (digraph: EdgeWeightedDigraph)
76
+ isDAG (): boolean
77
+ order (): number[]
78
+ }
79
+
80
+ export * from './storage'
@@ -27,3 +27,5 @@ exports.ShortestPathTree = require('./shortest_path_tree')
27
27
  exports.LongestPathTree = require('./longest_path_tree')
28
28
  exports.DirectedEdge = require('./directed_edge')
29
29
  exports.EdgeWeightedDigraph = require('./edge_weighted_digraph')
30
+ exports.Topological = require('./topological')
31
+ exports.StorageBackend = require('./storage')
@@ -0,0 +1,39 @@
1
+ /*
2
+ Copyright (c) 2024, Hugo W.L. ter Doest
3
+
4
+ Permission is hereby granted, free of charge, to any person obtaining a copy
5
+ of this software and associated documentation files (the "Software"), to deal
6
+ in the Software without restriction, including without limitation the rights
7
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8
+ copies of the Software, and to permit persons to whom the Software is
9
+ furnished to do so, subject to the following conditions:
10
+
11
+ The above copyright notice and this permission notice shall be included in
12
+ all copies or substantial portions of the Software.
13
+
14
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20
+ THE SOFTWARE.
21
+ */
22
+
23
+ export declare enum STORAGE_TYPES {
24
+ POSTGRES = 'POSTGRES',
25
+ REDIS = 'REDIS',
26
+ MONGODB = 'MONGODB',
27
+ MEMCACHED = 'MEMCACHED',
28
+ FILE = 'FILE'
29
+ }
30
+
31
+ export class StorageBackend {
32
+ private readonly storageType: STORAGE_TYPES
33
+ private readonly client: any
34
+
35
+ constructor (type: STORAGE_TYPES)
36
+ setStorageType (storageType: STORAGE_TYPES): void
37
+ store (key: string, value: any): Promise<string>
38
+ retrieve (key: string): Promise<any>
39
+ }
@@ -0,0 +1,24 @@
1
+
2
+ /*
3
+ Copyright (c) 2024, Hugo W.L. ter Doest
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in
13
+ all copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ THE SOFTWARE.
22
+ */
23
+
24
+ exports.StorageBackend = require('./StorageBackend')
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "natural",
3
3
  "description": "General natural language (tokenizing, stemming (English, Russian, Spanish), part-of-speech tagging, sentiment analysis, classification, inflection, phonetics, tfidf, WordNet, jaro-winkler, Levenshtein distance, Dice's Coefficient) facilities for node.",
4
- "version": "6.12.0",
4
+ "version": "7.0.0",
5
5
  "homepage": "https://github.com/NaturalNode/natural",
6
6
  "repository": {
7
7
  "type": "git",
@@ -11,10 +11,12 @@
11
11
  "node": ">=0.4.10"
12
12
  },
13
13
  "dependencies": {
14
+ "@types/jasmine": "^5.1.4",
14
15
  "afinn-165": "^1.0.2",
15
16
  "afinn-165-financialmarketnews": "^3.0.0",
16
17
  "apparatus": "^0.0.10",
17
18
  "dotenv": "^16.4.5",
19
+ "lint": "^0.8.19",
18
20
  "memjs": "^1.3.2",
19
21
  "mongoose": "^8.2.0",
20
22
  "pg": "^8.11.3",
@@ -28,20 +30,30 @@
28
30
  },
29
31
  "devDependencies": {
30
32
  "@types/node": "^18.11.18",
33
+ "@typescript-eslint/eslint-plugin": "^6.21.0",
31
34
  "browserfs": "^1.4.3",
35
+ "cross-env": "^7.0.3",
36
+ "eslint": "^8.57.0",
37
+ "eslint-config-standard-with-typescript": "^43.0.1",
38
+ "eslint-plugin-import": "^2.29.1",
39
+ "eslint-plugin-n": "^15.7.0",
40
+ "eslint-plugin-promise": "^6.1.1",
32
41
  "gulp": "^4.0.2",
33
42
  "gulp-jasmine": "^4.0.0",
34
43
  "gulp-jasmine-browser": "^4.1.0",
35
44
  "jasmine": "^3.3.1",
36
45
  "jasmine-core": "^3.3.0",
46
+ "ncp": "^2.0.0",
37
47
  "nyc": "^15.1.0",
38
48
  "pegjs": "^0.10.0",
39
49
  "proxyquire": "^1.8.0",
40
50
  "rimraf": "^2.6.3",
41
51
  "sinon": "^1.12.2",
52
+ "source-map-support": "^0.5.21",
42
53
  "standard": "^16.0.4",
54
+ "ts-node": "^10.9.2",
43
55
  "ts-standard": "^12.0.2",
44
- "typescript": "^4.9.3",
56
+ "typescript": "^5.4.2",
45
57
  "uubench": "^0.0.1",
46
58
  "webpack": "^4.29.0",
47
59
  "webpack-stream": "^5.2.1"
@@ -63,13 +75,14 @@
63
75
  },
64
76
  "scripts": {
65
77
  "benchmark": "node benchmarks",
66
- "clean": "rimraf *~ #* *classifier.json",
67
- "test": "NODE_PATH=. node ./node_modules/jasmine/bin/jasmine.js --random=false spec/*_spec.js",
68
- "coverage": "nyc --reporter=lcov npm run test && nyc npm run test_io",
69
- "test_io": "NODE_PATH=. node ./node_modules/jasmine/bin/jasmine.js --random=false io_spec/*_spec.js",
70
- "test_browser": "NODE_PATH=. node ./node_modules/gulp/bin/gulp.js",
71
- "lint": "eslint . --ext .ts",
72
- "lint-ts": "ts-standard **/*.ts"
78
+ "build": "tsc && ncp spec/test_data dist",
79
+ "clean": "rimraf *~ #* *classifier.json dist io_spec/tmp/*.json coverage",
80
+ "test": "cross-env NODE_PATH=. jasmine --random=false spec/*_spec.js",
81
+ "test_io": "jasmine --random=false io_spec/*_spec.js",
82
+ "test_ts": "cross-env NODE_PATH=.:./dist jasmine --random=false dist/ts_spec/*_spec.js",
83
+ "coverage": "nyc --reporter=lcov npm run test && nyc --reporter=lcov npm run test_io && nyc --reporter=lcov npm run test_ts",
84
+ "test_browser": "cross-env NODE_PATH=. node ./node_modules/gulp/bin/gulp.js",
85
+ "lint": "eslint . --ext .ts"
73
86
  },
74
87
  "license": "MIT",
75
88
  "author": "Chris Umbel <chris@chrisumbel.com>",
@@ -0,0 +1,130 @@
1
+ /*
2
+ Unit test of Classifier
3
+ Copyright (C) 2018 Hugo W.L. ter Doest
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in
13
+ all copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ THE SOFTWARE.
22
+ */
23
+
24
+ 'use strict'
25
+
26
+ import {
27
+ Context,
28
+ FeatureSet,
29
+ Sample,
30
+ MaxEntClassifier as Classifier,
31
+ SEElement
32
+ } from 'lib/natural'
33
+
34
+ import type { Element } from 'lib/natural'
35
+
36
+ const classifierFilename = 'classifier.json'
37
+ const minImprovement = 0.01
38
+ const nrIterations = 20
39
+
40
+ let sample: Sample
41
+ let featureSet: FeatureSet
42
+ let classifier: Classifier
43
+
44
+ const DEBUG = false
45
+
46
+ describe('The MaxEnt module', function () {
47
+ it('The Sample class creates a sample', function () {
48
+ sample = new Sample()
49
+ sample.addElement(new SEElement('x', new Context('0')))
50
+ sample.addElement(new SEElement('x', new Context('0')))
51
+ sample.addElement(new SEElement('x', new Context('0')))
52
+ sample.addElement(new SEElement('y', new Context('0')))
53
+ sample.addElement(new SEElement('y', new Context('0')))
54
+ sample.addElement(new SEElement('y', new Context('0')))
55
+
56
+ sample.addElement(new SEElement('x', new Context('1')))
57
+ sample.addElement(new SEElement('y', new Context('1')))
58
+ sample.addElement(new SEElement('y', new Context('1')))
59
+ sample.addElement(new SEElement('y', new Context('1')))
60
+
61
+ expect(sample.size()).toBe(10)
62
+ })
63
+
64
+ it('The FeatureSet class creates a feature set', function () {
65
+ featureSet = new FeatureSet()
66
+ sample.generateFeatures(featureSet)
67
+
68
+ expect(featureSet.size()).toBe(2)
69
+ })
70
+
71
+ it('The Classifier class creates a classifier', function () {
72
+ // Create a classifier
73
+ classifier = new Classifier(featureSet, sample)
74
+
75
+ expect(classifier).toBeDefined()
76
+ })
77
+
78
+ it('Classifier does not need a correction feature', function () {
79
+
80
+ })
81
+
82
+ it('The classifier stops training after a specified number or iterations ' +
83
+ 'or when the minimum improvement in likelihood is reached', function () {
84
+ classifier.train(nrIterations, minImprovement)
85
+
86
+ expect(classifier.scaler.iteration).toBeLessThan(nrIterations + 1)
87
+ if (classifier.scaler.iteration < nrIterations) {
88
+ expect(classifier.scaler.improvement).toBeLessThan(minImprovement)
89
+ }
90
+ })
91
+
92
+ it('Save classifer to a file', function (done) {
93
+ classifier.save(classifierFilename, function (err, c) {
94
+ if (err !== null) {
95
+ console.log(err)
96
+ } else {
97
+ DEBUG && console.log('Classifier saved to ' + classifierFilename)
98
+ }
99
+ done()
100
+ })
101
+ })
102
+
103
+ let newClassifier: Classifier
104
+ it('Load classifer', function (done) {
105
+ classifier.load(classifierFilename, SEElement as unknown as Element, function (err, c) {
106
+ if (err !== null) {
107
+ console.log(err)
108
+ } else if (c !== undefined) {
109
+ DEBUG && console.log('Classifier loaded from ' + classifierFilename)
110
+ newClassifier = c
111
+ }
112
+ done()
113
+ })
114
+ if (newClassifier !== undefined) {
115
+ classifier = newClassifier
116
+ }
117
+ })
118
+
119
+ it('The classifier classifies events', function () {
120
+ let context = new Context('0')
121
+ DEBUG && console.log('Classes plus scores ' + JSON.stringify(classifier.getClassifications(context)))
122
+ let classification = classifier.classify(context)
123
+ expect(classification).toBe('x')
124
+
125
+ context = new Context('1')
126
+ DEBUG && console.log('Classes plus scores ' + JSON.stringify(classifier.getClassifications(context)))
127
+ classification = classifier.classify(context)
128
+ expect(classification).toBe('y')
129
+ })
130
+ })