natural 6.12.0 → 7.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. package/.env +2 -1
  2. package/.eslintrc.json +13 -0
  3. package/.github/workflows/node.js.yml +3 -6
  4. package/lib/natural/analyzers/SenType.ts +30 -0
  5. package/lib/natural/analyzers/index.d.ts +6 -10
  6. package/lib/natural/brill_pos_tagger/index.d.ts +23 -14
  7. package/lib/natural/brill_pos_tagger/index.js +1 -1
  8. package/lib/natural/classifiers/bayes_classifier.js +1 -1
  9. package/lib/natural/classifiers/classifier.js +11 -34
  10. package/lib/natural/classifiers/classifier_train_parallel.js +7 -5
  11. package/lib/natural/classifiers/index.d.ts +36 -15
  12. package/lib/natural/classifiers/maxent/Classifier.js +5 -3
  13. package/lib/natural/distance/index.d.ts +11 -14
  14. package/lib/natural/inflectors/count_inflector.js +18 -21
  15. package/lib/natural/inflectors/fr/count_inflector.js +18 -19
  16. package/lib/natural/inflectors/index.d.ts +12 -3
  17. package/lib/natural/inflectors/index.js +3 -0
  18. package/lib/natural/ngrams/index.d.ts +5 -5
  19. package/lib/natural/normalizers/index.d.ts +24 -1
  20. package/lib/natural/normalizers/index.js +4 -1
  21. package/lib/natural/normalizers/normalizer_ja.js +109 -72
  22. package/lib/natural/phonetics/dm_soundex.js +52 -51
  23. package/lib/natural/phonetics/double_metaphone.js +397 -399
  24. package/lib/natural/phonetics/index.d.ts +42 -12
  25. package/lib/natural/phonetics/metaphone.js +134 -157
  26. package/lib/natural/phonetics/phonetic.js +5 -29
  27. package/lib/natural/phonetics/soundex.js +45 -54
  28. package/lib/natural/sentiment/index.d.ts +6 -6
  29. package/lib/natural/spellcheck/index.d.ts +2 -0
  30. package/lib/natural/stemmers/index.d.ts +43 -1
  31. package/lib/natural/stemmers/index.js +1 -0
  32. package/lib/natural/stemmers/token.js +131 -132
  33. package/lib/natural/tfidf/index.d.ts +7 -7
  34. package/lib/natural/tfidf/tfidf.js +7 -1
  35. package/lib/natural/tokenizers/index.d.ts +6 -2
  36. package/lib/natural/tokenizers/index.js +1 -0
  37. package/lib/natural/trie/index.d.ts +1 -1
  38. package/lib/natural/util/index.d.ts +8 -0
  39. package/lib/natural/util/index.js +2 -0
  40. package/lib/natural/util/storage/index.d.ts +39 -0
  41. package/lib/natural/util/storage/index.js +24 -0
  42. package/package.json +22 -9
  43. package/ts_spec/MaxEntClassifier_spec.ts +130 -0
  44. package/ts_spec/SentimentAnalyzer_spec.ts +232 -0
  45. package/ts_spec/WordPunctTokenizer_spec.ts +57 -0
  46. package/ts_spec/aggressive_tokenizer_de_spec.ts +36 -0
  47. package/ts_spec/aggressive_tokenizer_es_spec.ts +39 -0
  48. package/ts_spec/aggressive_tokenizer_fr_spec.ts +111 -0
  49. package/ts_spec/aggressive_tokenizer_hi_spec.ts +70 -0
  50. package/ts_spec/aggressive_tokenizer_nl_spec.ts +39 -0
  51. package/ts_spec/aggressive_tokenizer_pt_spec.ts +48 -0
  52. package/ts_spec/aggressive_tokenizer_spec.ts +57 -0
  53. package/ts_spec/aggressive_tokenizer_sv_spec.ts +52 -0
  54. package/ts_spec/aggressive_tokenizer_vi_spec.ts +48 -0
  55. package/ts_spec/bayes_classifier_spec.ts +165 -0
  56. package/ts_spec/brill_pos_tagger_spec.ts +79 -0
  57. package/ts_spec/brill_pos_trainer_spec.ts +107 -0
  58. package/ts_spec/carry_stemmer_fr_spec.ts +97 -0
  59. package/ts_spec/classifier_spec.ts +104 -0
  60. package/ts_spec/count_inflector_fr_spec.ts +47 -0
  61. package/ts_spec/count_inflector_spec.ts +76 -0
  62. package/ts_spec/damerau_levenshtein_spec.ts +53 -0
  63. package/ts_spec/dice_coefficient_spec.ts +37 -0
  64. package/ts_spec/double_metaphone_spec.ts +739 -0
  65. package/ts_spec/hamming_distance_spec.ts +66 -0
  66. package/ts_spec/jaro-winkler_spec.ts +79 -0
  67. package/ts_spec/lancaster_stemmer_spec.ts +63 -0
  68. package/ts_spec/levenshtein_spec.ts +107 -0
  69. package/ts_spec/logistic_regression_classifier_spec.ts +93 -0
  70. package/ts_spec/longest_path_tree_spec.ts +111 -0
  71. package/ts_spec/metaphone_spec.ts +258 -0
  72. package/ts_spec/ngram_spec.ts +145 -0
  73. package/ts_spec/ngram_zh_spec.ts +124 -0
  74. package/ts_spec/normalizer_ja_spec.ts +165 -0
  75. package/ts_spec/normalizer_no_spec.ts +112 -0
  76. package/ts_spec/normalizer_spec.ts +141 -0
  77. package/ts_spec/noun_inflector_fr_spec.ts +472 -0
  78. package/ts_spec/noun_inflector_ja_spec.ts +116 -0
  79. package/ts_spec/noun_inflector_spec.ts +369 -0
  80. package/ts_spec/orthography_tokenizer_spec.ts +47 -0
  81. package/ts_spec/porter_stemmer_de_spec.ts +53 -0
  82. package/ts_spec/porter_stemmer_es_spec.ts +63 -0
  83. package/ts_spec/porter_stemmer_fr_spec.ts +103 -0
  84. package/ts_spec/porter_stemmer_it_spec.ts +56 -0
  85. package/ts_spec/porter_stemmer_nl_spec.ts +73 -0
  86. package/ts_spec/porter_stemmer_no_spec.ts +148 -0
  87. package/ts_spec/porter_stemmer_pt_spec.ts +50 -0
  88. package/ts_spec/porter_stemmer_ru_spec.ts +67 -0
  89. package/ts_spec/porter_stemmer_spec.ts +202 -0
  90. package/ts_spec/porter_stemmer_sv_spec.ts +41 -0
  91. package/ts_spec/present_verb_inflector_spec.ts +127 -0
  92. package/ts_spec/remove_diacritics_spec.ts +30 -0
  93. package/ts_spec/sentence_analyzer_spec.ts +293 -0
  94. package/ts_spec/sentence_tokenizer_parser_spec.ts +200 -0
  95. package/ts_spec/sentence_tokenizer_spec.ts +189 -0
  96. package/ts_spec/shortest_path_tree_spec.ts +141 -0
  97. package/ts_spec/soundex_spec.ts +162 -0
  98. package/ts_spec/spellcheck_spec.ts +112 -0
  99. package/ts_spec/stemmer_id_spec.ts +374 -0
  100. package/ts_spec/stemmer_ja_spec.ts +69 -0
  101. package/ts_spec/stemmer_token_spec.ts +163 -0
  102. package/ts_spec/tfidf_spec.ts +275 -0
  103. package/ts_spec/tokenizer_case_spec.ts +242 -0
  104. package/ts_spec/tokenizer_ja_spec.ts +63 -0
  105. package/ts_spec/transliterator_ja_spec.ts +105 -0
  106. package/ts_spec/treebank_word_tokenizer_spec.ts +38 -0
  107. package/ts_spec/trie_spec.ts +197 -0
  108. package/tsconfig.json +5 -3
  109. package/example.txt +0 -1
package/.env CHANGED
@@ -1,3 +1,4 @@
1
+ NODE_PATH=.
1
2
  # Storage backend you want to use
2
3
  NATURAL_STORAGE=PG
3
4
  # Settings for Postgres
@@ -20,4 +21,4 @@ MONGO_PORT=27017
20
21
  MONGO_DATABASE=naturaldb
21
22
  MONGO_OBJECTMODEL=NaturalObjectModel
22
23
  # Path to the folder writing files
23
- FS_PATH=.
24
+ FS_PATH=./io_spec/tmp
package/.eslintrc.json ADDED
@@ -0,0 +1,13 @@
1
+ {
2
+ "env": {
3
+ "browser": true,
4
+ "commonjs": true,
5
+ "es2021": true
6
+ },
7
+ "extends": "standard-with-typescript",
8
+ "parserOptions": {
9
+ "ecmaVersion": "latest"
10
+ },
11
+ "rules": {
12
+ }
13
+ }
@@ -60,12 +60,9 @@ jobs:
60
60
  - name: Clean install
61
61
  run: npm ci
62
62
 
63
- - name: Run tests
64
- run: npm run test
65
-
66
- - name: Run IO tests
67
- run: npm run test_io
68
-
63
+ - name: Build
64
+ run: npm run build
65
+
69
66
  - name: Run Istanbul coverage
70
67
  run: npm run coverage
71
68
 
@@ -0,0 +1,30 @@
1
+ /*
2
+ Copyright (c) 2024,
3
+ Hugo W.L. ter Doest <https://github.com/Hugo-ter-Doest>
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in
13
+ all copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ THE SOFTWARE.
22
+ */
23
+
24
+ export const enum SenType {
25
+ Unknown = 'UNKNOWN',
26
+ Command = 'COMMAND',
27
+ Interrogative = 'INTERROGATIVE',
28
+ Exclamatory = 'EXCLAMATORY',
29
+ Declarative = 'DECLARATIVE'
30
+ }
@@ -21,15 +21,19 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
21
  THE SOFTWARE.
22
22
  */
23
23
 
24
+ import type { SenType } from './SenType'
25
+
24
26
  declare interface TaggedWord {
25
27
  token: string
26
28
  pos: string
27
29
  pp?: boolean
28
30
  spos?: string
29
31
  added?: boolean
32
+ [key: string]: any
30
33
  }
31
34
 
32
- declare type PunctuationFunction = () => string[] | ''
35
+ declare type PunctuationMapping = Array<Record<string, string>> | ''
36
+ declare type PunctuationFunction = () => PunctuationMapping
33
37
 
34
38
  declare interface TaggedSentence {
35
39
  tags: TaggedWord[]
@@ -38,14 +42,6 @@ declare interface TaggedSentence {
38
42
 
39
43
  declare type CallbackFunction = (obj: SentenceAnalyzer) => void
40
44
 
41
- declare enum SenType {
42
- Unknown = 'UNKNOWN',
43
- Command = 'COMMAND',
44
- Interrogative = 'INTERROGATIVE',
45
- Exclamatory = 'EXCLAMATORY',
46
- Declarative = 'DECLARATIVE',
47
- }
48
-
49
45
  export class SentenceAnalyzer {
50
46
  posObj: TaggedSentence
51
47
  senType: SenType | null
@@ -57,5 +53,5 @@ export class SentenceAnalyzer {
57
53
  predicateToString (): string
58
54
  implicitYou (): boolean
59
55
  toString (): string
60
- type (cbf: CallbackFunction): SenType
56
+ type (cbf: CallbackFunction): string
61
57
  }
@@ -23,7 +23,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
23
23
  THE SOFTWARE.
24
24
  */
25
25
 
26
- import { Feature } from '../classifiers'
26
+ import type { Feature } from '../classifiers'
27
27
 
28
28
  declare interface RuleTemplatesItem {
29
29
  function: (sentence: Sentence, i: number, parameter1?: string, parameter2?: string) => boolean
@@ -33,9 +33,8 @@ declare interface RuleTemplatesItem {
33
33
  parameter2Values?: (sentence: Sentence, i: number) => string[]
34
34
  }
35
35
 
36
- export interface RuleTemplates {
37
- [key: string]: RuleTemplatesItem | undefined
38
- }
36
+ export type RuleTemplates = Record<string, RuleTemplatesItem>
37
+ export let ruleTemplates: RuleTemplates
39
38
 
40
39
  export class RuleTemplate {
41
40
  constructor (templateName: string, metadata: RuleTemplatesItem)
@@ -82,7 +81,7 @@ export class RuleSet {
82
81
 
83
82
  export class Lexicon {
84
83
  constructor (language: string, defaultCategory: string, defaultCategoryCapitalised?: string)
85
- lexicon: { [key: string]: string[] | undefined }
84
+ lexicon: Record<string, string[] | undefined>
86
85
  defaultCategory: string
87
86
  defaultCategoryCapitalised: string | undefined
88
87
  parseLexicon (data: string): void
@@ -96,11 +95,11 @@ export class Lexicon {
96
95
  }
97
96
 
98
97
  declare class Corpus {
99
- constructor (data: string | Corpus, typeOfCorpus: number, SentenceClass: typeof Sentence)
98
+ constructor (data: string | TaggedCorpus, typeOfCorpus: number, SentenceClass: typeof Sentence)
100
99
  private readonly wordCount: number
101
100
  private readonly sentences: Sentence[]
102
- private readonly tagFrequencies: { [key: string]: string[] | undefined }
103
- private readonly posTags: { [key: string]: string[] | undefined }
101
+ private readonly tagFrequencies: Record<string, Record<string, number> | undefined>
102
+ private readonly posTags: Record<string, boolean | undefined>
104
103
  parseBrownCorpus (data: string, SentenceClass: typeof Sentence): void
105
104
  getTags (): string[]
106
105
  splitInTrainAndTest (percentageTrain: number): [Corpus, Corpus]
@@ -119,6 +118,15 @@ declare interface BrillPOSTaggedWord {
119
118
  tag: string
120
119
  }
121
120
 
121
+ declare interface BrillPOSTaggedSentence {
122
+ taggedWords: BrillPOSTaggedWord[]
123
+ }
124
+
125
+ declare interface TaggedCorpus {
126
+ wordCount: number
127
+ sentences: BrillPOSTaggedSentence[]
128
+ }
129
+
122
130
  export class Sentence {
123
131
  constructor (data?: string[])
124
132
  taggedWords: BrillPOSTaggedWord[]
@@ -136,9 +144,6 @@ export class BrillPOSTagger {
136
144
  }
137
145
 
138
146
  export class BrillPOSTester {
139
- constructor (lexicon: Lexicon, ruleSet: RuleSet)
140
- private readonly lexicon: Lexicon
141
- private readonly ruleSet: RuleSet
142
147
  test (corpus: Corpus, tagger: BrillPOSTagger): [number, number]
143
148
  }
144
149
 
@@ -148,8 +153,8 @@ export class BrillPOSTrainer {
148
153
  private readonly corpus: Corpus
149
154
  private readonly templates: RuleTemplates
150
155
  private readonly positiveRules: RuleSet
151
- private readonly mapRuleToSites: { [key: string]: { [key: number ]: { [key: number ]: boolean | undefined } | undefined } | undefined }
152
- private readonly mapSiteToRules: { [key: number]: { [key: number ]: { [key: string ]: TransformationRule | undefined } | undefined } | undefined }
156
+ private readonly mapRuleToSites: Record<string, Record<number, Record< number, boolean | undefined> | undefined> | undefined>
157
+ private readonly mapSiteToRules: Record<number, Record<number, Record<string, boolean | undefined> | undefined> | undefined>
153
158
  private selectHighRule (): TransformationRule
154
159
  private mapRuleToSite (rule: TransformationRule, i: number, j: number): void
155
160
  private mapSiteToRule (i: number, j: number, rule: TransformationRule): void
@@ -163,6 +168,10 @@ export class BrillPOSTrainer {
163
168
  private scanForPositiveRules (): void
164
169
  private scanForSites (): void
165
170
  private neighbourhood (i: number, j: number): Array<[number, number]>
166
- train (corpus: Corpus, templates: RuleTemplates, lexicon: Lexicon): RuleSet
171
+ train (corpus: Corpus, templates: RuleTemplate[], lexicon: Lexicon): RuleSet
167
172
  printRulesWithScores (): string
168
173
  }
174
+
175
+ export interface TagResults {
176
+ results: string[][][]
177
+ }
@@ -27,7 +27,7 @@ exports.BrillPOSTrainer = require('./lib/Brill_POS_Trainer')
27
27
  exports.BrillPOSTester = require('./lib/Brill_POS_Tester')
28
28
  exports.Lexicon = require('./lib/Lexicon')
29
29
  exports.RuleSet = require('./lib/RuleSet')
30
- exports.RuleTemplates = require('./lib/RuleTemplates')
30
+ exports.ruleTemplates = require('./lib/RuleTemplates')
31
31
  exports.RuleTemplate = require('./lib/RuleTemplate')
32
32
  exports.Corpus = require('./lib/Corpus')
33
33
  exports.Sentence = require('./lib/Sentence')
@@ -44,7 +44,7 @@ class BayesClassifier extends Classifier {
44
44
  }
45
45
 
46
46
  static load (filename, stemmer, callback) {
47
- Classifier.load(filename, function (err, classifier) {
47
+ Classifier.load(filename, stemmer, function (err, classifier) {
48
48
  if (err) {
49
49
  return callback(err)
50
50
  } else {
@@ -21,6 +21,7 @@ THE SOFTWARE.
21
21
  */
22
22
 
23
23
  'use strict'
24
+ const fs = require('fs')
24
25
 
25
26
  const EventEmitter = require('events')
26
27
  const PorterStemmer = require('../stemmers/porter_stemmer')
@@ -34,6 +35,12 @@ class Classifier extends EventEmitter {
34
35
  this.features = {}
35
36
  this.stemmer = stemmer || PorterStemmer
36
37
  this.lastAdded = 0
38
+
39
+ // Add methods for parallel training
40
+ this.Threads = parallelTrainer.Threads
41
+ this.trainParallel = parallelTrainer.trainParallel
42
+ this.retrainParallel = parallelTrainer.retrainParallel
43
+ this.trainParallelBatches = parallelTrainer.trainParallelBatches
37
44
  }
38
45
 
39
46
  addDocument (text, classification) {
@@ -54,7 +61,7 @@ class Classifier extends EventEmitter {
54
61
 
55
62
  this.docs.push({
56
63
  label: classification,
57
- text: text
64
+ text
58
65
  })
59
66
 
60
67
  for (let i = 0; i < text.length; i++) {
@@ -114,8 +121,8 @@ class Classifier extends EventEmitter {
114
121
  this.emit('trainedWithDocument', { index: i, total: totalDocs, doc: this.docs[i] })
115
122
  this.lastAdded++
116
123
  }
117
- this.emit('doneTraining', true)
118
124
  this.classifier.train()
125
+ this.emit('doneTraining', true)
119
126
  }
120
127
 
121
128
  retrain () {
@@ -139,7 +146,6 @@ class Classifier extends EventEmitter {
139
146
 
140
147
  save (filename, callback) {
141
148
  const data = JSON.stringify(this)
142
- const fs = require('fs')
143
149
  const classifier = this
144
150
  fs.writeFile(filename, data, 'utf8', function (err) {
145
151
  if (callback) {
@@ -148,9 +154,7 @@ class Classifier extends EventEmitter {
148
154
  })
149
155
  }
150
156
 
151
- static load (filename, callback) {
152
- const fs = require('fs')
153
-
157
+ static load (filename, stemmer, callback) {
154
158
  if (!callback) {
155
159
  return
156
160
  }
@@ -159,7 +163,7 @@ class Classifier extends EventEmitter {
159
163
  callback(err, null)
160
164
  } else {
161
165
  const classifier = JSON.parse(data)
162
- callback(err, classifier)
166
+ callback(err, Classifier.restore(classifier, stemmer))
163
167
  }
164
168
  })
165
169
  }
@@ -177,33 +181,6 @@ class Classifier extends EventEmitter {
177
181
  setOptions (options) {
178
182
  this.keepStops = !!(options.keepStops)
179
183
  }
180
-
181
- ClassifiertrainParallel (numThreads, callback) {
182
- if (parallelTrainer.Threads) {
183
- return parallelTrainer.trainParallel(numThreads, callback)
184
- } else {
185
- this.emit('No threads available')
186
- return this.train()
187
- }
188
- }
189
-
190
- trainParallelBatches (options) {
191
- if (parallelTrainer.Threads) {
192
- return parallelTrainer.trainParallelBatches(options)
193
- } else {
194
- this.emit('No threads available')
195
- return this.train()
196
- }
197
- }
198
-
199
- retrainParallel (numThreads, callback) {
200
- if (parallelTrainer.Threads) {
201
- return parallelTrainer.trainParallel(numThreads, callback)
202
- } else {
203
- this.emit('No threads available')
204
- return this.retrain()
205
- }
206
- }
207
184
  }
208
185
 
209
186
  module.exports = Classifier
@@ -5,8 +5,8 @@ let Threads = null
5
5
  try {
6
6
  Threads = require('webworker-threads')
7
7
  } catch (e) {
8
- // Since webworker-threads are optional, only thow if the module is found
9
- if (e.code !== 'MODULE_NOT_FOUND') throw e
8
+ // Silently set Threads to null
9
+ Threads = null
10
10
  }
11
11
 
12
12
  function checkThreadSupport () {
@@ -27,7 +27,7 @@ function docsToFeatures (docs) {
27
27
 
28
28
  parsedDocs.push({
29
29
  index: docs[i].index,
30
- features: features
30
+ features
31
31
  })
32
32
  }
33
33
 
@@ -44,7 +44,7 @@ function docsToObs (docs, lastAdded, stemmer, keepStops) {
44
44
  }
45
45
  obsDocs.push({
46
46
  index: i,
47
- observation: observation
47
+ observation
48
48
  })
49
49
  }
50
50
  return obsDocs
@@ -181,7 +181,7 @@ function trainParallelBatches (options) {
181
181
  if (typeof observation === 'string') { observation = this.stemmer.tokenizeAndStem(observation, this.keepStops) }
182
182
  obsDocs.push({
183
183
  index: i,
184
- observation: observation
184
+ observation
185
185
  })
186
186
  }
187
187
 
@@ -266,6 +266,8 @@ function trainParallelBatches (options) {
266
266
  }
267
267
 
268
268
  function retrainParallel (numThreads, callback) {
269
+ checkThreadSupport()
270
+
269
271
  this.classifier = new (this.classifier.constructor)()
270
272
  this.lastAdded = 0
271
273
  this.trainParallel(numThreads, callback)
@@ -1,6 +1,7 @@
1
1
  import events from 'events'
2
- import { Corpus, Sentence } from '../brill_pos_tagger'
3
- import { Stemmer } from '../stemmers'
2
+ import type { Corpus, Sentence } from '../brill_pos_tagger'
3
+ import type { Stemmer } from '../stemmers'
4
+ import type { StorageBackend } from '../util'
4
5
 
5
6
  // Start apparatus declarations
6
7
 
@@ -19,7 +20,8 @@ declare interface ApparatusClassification {
19
20
  value: number
20
21
  }
21
22
 
22
- declare type ApparatusObservation = number[] | { [key: string]: number | string | boolean | undefined }
23
+ // Observations are either array of numbers or a sparse vector in the form of a map
24
+ declare type ApparatusObservation = number[] | Record<string, number | string | boolean>
23
25
 
24
26
  declare class ApparatusClassifier {
25
27
  addExample (observation: ApparatusObservation, label: string): void
@@ -31,8 +33,8 @@ declare class ApparatusClassifier {
31
33
  // TODO: Not needed for natural repository, but could be moved to
32
34
  // aparatus repository. Temporarily leaving here for copypasta.
33
35
  declare class ApparatusBayesClassifier extends ApparatusClassifier {
34
- classFeatures: { [key: string]: { [key: number]: number | undefined } | undefined }
35
- classTotals: { [key: string]: number | undefined }
36
+ classFeatures: Record<string, Record<number, number>>
37
+ classTotals: Record<string, number>
36
38
  totalExamples: number
37
39
  smoothing: number
38
40
 
@@ -45,8 +47,8 @@ declare class ApparatusBayesClassifier extends ApparatusClassifier {
45
47
  // TODO: Not needed for natural repository, but could be moved to
46
48
  // aparatus repository. Temporarily leaving here for copypasta.
47
49
  declare class ApparatusLogisticRegressionClassifier extends ApparatusClassifier {
48
- examples: { [key: string]: number[] | undefined }
49
- // TODO: These appear used
50
+ examples: Record<string, ApparatusObservation>
51
+ // TODO: These appear unused
50
52
  // features: any[]
51
53
  // featurePositions: { [key: string]: any | undefined }
52
54
  maxFeaturePosition: number
@@ -71,11 +73,12 @@ declare interface ClassifierOptions {
71
73
  }
72
74
 
73
75
  declare type ClassifierCallback = (err: NodeJS.ErrnoException | null, classifier?: ClassifierBase) => void
76
+ declare type ParallelTrainerCallback = (err: NodeJS.ErrnoException | null) => void
74
77
 
75
78
  declare class ClassifierBase extends events.EventEmitter {
76
79
  classifier: ApparatusClassifier
77
80
  docs: ClassifierDoc[]
78
- features: { [key: string]: number | undefined }
81
+ features: Record<string, number>
79
82
  stemmer: Stemmer
80
83
  lastAdded: number
81
84
 
@@ -89,14 +92,26 @@ declare class ClassifierBase extends events.EventEmitter {
89
92
  classify (observation: string | string[]): string
90
93
  setOptions (options: ClassifierOptions): void
91
94
  save (filename: string, callback?: ClassifierCallback): void
95
+ static load (filename: string, stemmer: Stemmer | null | undefined, callback: ClassifierCallback): void
96
+ saveTo (storage: StorageBackend): string
97
+ static loadFrom (storage: StorageBackend): ClassifierBase
98
+
99
+ Threads: any
100
+ trainParallel (numThreads: number, callback: ParallelTrainerCallback): void
101
+ trainParallelBatches (options: { numThreads: number, batchSize: number }): void
102
+ retrainParallel (numThreads: number, callback: ParallelTrainerCallback): void
92
103
  }
93
104
 
94
105
  declare type BayesClassifierCallback = (err: NodeJS.ErrnoException | null, classifier?: BayesClassifier) => void
95
106
 
96
107
  export class BayesClassifier extends ClassifierBase {
108
+ classifier: ApparatusBayesClassifier
109
+
97
110
  constructor (stemmer?: Stemmer, smoothing?: number)
98
111
  static load (filename: string, stemmer: Stemmer | null | undefined, callback: BayesClassifierCallback): void
99
- static restore (classifier: BayesClassifier, stemmer?: Stemmer): BayesClassifier
112
+ static restore (classifier: Record<string, unknown>, stemmer?: Stemmer): BayesClassifier
113
+ saveTo (storage: StorageBackend): string
114
+ static loadFrom (storage: StorageBackend): ClassifierBase
100
115
  }
101
116
 
102
117
  declare type LogisticRegressionClassifierCallback = (err: NodeJS.ErrnoException | null, classifier?: LogisticRegressionClassifier) => void
@@ -104,19 +119,22 @@ declare type LogisticRegressionClassifierCallback = (err: NodeJS.ErrnoException
104
119
  export class LogisticRegressionClassifier extends ClassifierBase {
105
120
  constructor (stemmer?: Stemmer)
106
121
  static load (filename: string, stemmer: Stemmer | null | undefined, callback: LogisticRegressionClassifierCallback): void
107
- static restore (classifier: LogisticRegressionClassifier, stemmer?: Stemmer): LogisticRegressionClassifier
122
+ static restore (classifier: Record<string, unknown>, stemmer?: Stemmer): LogisticRegressionClassifier
123
+ saveTo (storage: StorageBackend): string
124
+ static loadFrom (storage: StorageBackend): ClassifierBase
108
125
  }
109
126
 
110
- declare type MaxEntClassifierCallback = (err: NodeJS.ErrnoException | null, classifier?: MaxEntClassifier | null) => void
127
+ declare type MaxEntClassifierCallback = (err: NodeJS.ErrnoException | null, classifier?: MaxEntClassifier) => void
111
128
 
112
129
  export class MaxEntClassifier {
113
130
  sample: Sample
114
131
  features: FeatureSet
132
+ scaler: GISScaler
115
133
 
116
134
  constructor (features: FeatureSet, sample: Sample)
117
135
  addElement (x: Element): void
118
136
  addDocument (context: Context, classification: string, elementClass: Element): void
119
- train (maxIterations: number, minImprovement: number, unused: any): void
137
+ train (maxIterations: number, minImprovement: number): void
120
138
  getClassifications (b: Context): ApparatusClassification[]
121
139
  classify (b: Context): string
122
140
  // These are not static like in other Classifier classes
@@ -160,7 +178,7 @@ export class Feature {
160
178
 
161
179
  export class FeatureSet {
162
180
  features: Feature[]
163
- map: { [key: string]: boolean | undefined }
181
+ map: Record<string, boolean>
164
182
 
165
183
  addFeature (feature: Feature): boolean
166
184
  featureExists (feature: Feature): boolean
@@ -172,8 +190,8 @@ export class FeatureSet {
172
190
  declare type SampleCallback = (err: NodeJS.ErrnoException | null, sample?: Sample | null) => void
173
191
 
174
192
  export class Sample {
175
- frequencyOfContext: { [key: string]: number | undefined }
176
- frequency: { [key: string]: number | undefined }
193
+ frequencyOfContext: Record<string, number>
194
+ frequency: Record<string, number>
177
195
  classes: string[]
178
196
 
179
197
  constructor (elements?: Element[])
@@ -215,6 +233,9 @@ export class POSElement extends Element {
215
233
  }
216
234
 
217
235
  export class GISScaler {
236
+ iteration: number
237
+ improvement: number
238
+
218
239
  constructor (featureSet: FeatureSet, sample: Sample)
219
240
  calculateMaxSumOfFeatures (): boolean
220
241
  addCorrectionFeature (): void
@@ -32,6 +32,8 @@ const Sample = require('./Sample')
32
32
  const Scaler = require('./GISScaler')
33
33
  const FeatureSet = require('./FeatureSet')
34
34
 
35
+ const DEBUG = false
36
+
35
37
  class Classifier {
36
38
  constructor (features, sample) {
37
39
  if (features) {
@@ -77,7 +79,7 @@ class Classifier {
77
79
  const classifier = this
78
80
  fs.writeFile(filename, data, 'utf8', function (err) {
79
81
  if (callback) {
80
- console.log('Saved classifier to ' + filename)
82
+ DEBUG && console.log('Saved classifier to ' + filename)
81
83
  callback(err, err ? null : classifier)
82
84
  }
83
85
  })
@@ -91,9 +93,9 @@ class Classifier {
91
93
  Classifier.prototype.addElement(new ElementClass(classification, context))
92
94
  }
93
95
 
94
- train (maxIterations, minImprovement, approxExpectation) {
96
+ train (maxIterations, minImprovement) {
95
97
  this.scaler = new Scaler(this.features, this.sample)
96
- this.p = this.scaler.run(maxIterations, minImprovement, approxExpectation)
98
+ this.p = this.scaler.run(maxIterations, minImprovement)
97
99
  }
98
100
 
99
101
  getClassifications (b) {
@@ -1,6 +1,8 @@
1
-
2
1
  /*
3
- Copyright (c) 2022, Dylan R. E. Moonfire <https://github.com/dmoonfire>, Emily Marigold Klassen <https://github.com/forivall>, Hugo W.L. ter Doest
2
+ Copyright (c) 2023,
3
+ Dylan R. E. Moonfire <https://github.com/dmoonfire>,
4
+ Emily Marigold Klassen <https://github.com/forivall>,
5
+ Hugo W.L. ter Doest
4
6
 
5
7
  Permission is hereby granted, free of charge, to any person obtaining a copy
6
8
  of this software and associated documentation files (the "Software"), to deal
@@ -21,15 +23,13 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
23
  THE SOFTWARE.
22
24
  */
23
25
 
24
- // Based on type definitions on Definitely Typed
25
-
26
26
  declare interface JaroWinklerOptions {
27
27
  dj?: number
28
28
  /** @default false */
29
29
  ignoreCase?: boolean
30
30
  }
31
31
 
32
- export function JaroWinklerDistance (s1: string, s2: string, options: JaroWinklerOptions): number
32
+ export function JaroWinklerDistance (s1: string, s2: string, options?: JaroWinklerOptions): number
33
33
 
34
34
  declare interface DamerauLevenshteinDistanceOptions {
35
35
  /** @default 1 */
@@ -41,10 +41,7 @@ declare interface DamerauLevenshteinDistanceOptions {
41
41
  /** @default 1 */
42
42
  transposition_cost?: number
43
43
  /** @default false */
44
- search?: boolean
45
- /** @default false */
46
44
  restricted?: boolean
47
- damerau?: boolean
48
45
  }
49
46
 
50
47
  declare interface SubstringDistanceResult {
@@ -67,23 +64,23 @@ export function LevenshteinDistance (
67
64
  * Valid edit operations are:
68
65
  * - transposition, insertion, deletion, and substitution
69
66
  */
70
- export function DamerauLevenshteinDistance (
67
+ export function LevenshteinDistanceSearch (
71
68
  source: string,
72
69
  target: string,
73
- options: DamerauLevenshteinDistanceOptions & { search: true }
70
+ options?: DamerauLevenshteinDistanceOptions
74
71
  ): SubstringDistanceResult
75
72
 
76
73
  export function DamerauLevenshteinDistance (
77
74
  source: string,
78
75
  target: string,
79
- options?: DamerauLevenshteinDistanceOptions & { search?: false }
76
+ options?: DamerauLevenshteinDistanceOptions
80
77
  ): number
81
78
 
82
- export function DamerauLevenshteinDistance (
79
+ export function DamerauLevenshteinDistanceSearch (
83
80
  source: string,
84
81
  target: string,
85
- options: DamerauLevenshteinDistanceOptions & { search: boolean }
86
- ): number | SubstringDistanceResult
82
+ options?: DamerauLevenshteinDistanceOptions
83
+ ): SubstringDistanceResult
87
84
 
88
85
  export function DiceCoefficient (str1: string, str2: string): number
89
86
 
@@ -22,30 +22,27 @@ THE SOFTWARE.
22
22
 
23
23
  'use strict'
24
24
 
25
- function nthForm (i) {
26
- const teenth = (i % 100)
27
-
28
- if (teenth > 10 && teenth < 14) { return 'th' } else {
29
- switch (i % 10) {
30
- case 1:
31
- return 'st'
32
- case 2:
33
- return 'nd'
34
- case 3:
35
- return 'rd'
36
- default:
37
- return 'th'
25
+ class CountInflector {
26
+ nthForm (i) {
27
+ const teenth = (i % 100)
28
+
29
+ if (teenth > 10 && teenth < 14) { return 'th' } else {
30
+ switch (i % 10) {
31
+ case 1:
32
+ return 'st'
33
+ case 2:
34
+ return 'nd'
35
+ case 3:
36
+ return 'rd'
37
+ default:
38
+ return 'th'
39
+ }
38
40
  }
39
41
  }
40
- }
41
-
42
- function nth (i) {
43
- return i.toString() + nthForm(i)
44
- }
45
42
 
46
- const CountInflector = function () {
43
+ nth (i) {
44
+ return i.toString() + this.nthForm(i)
45
+ }
47
46
  }
48
47
 
49
- CountInflector.nth = nth
50
-
51
48
  module.exports = CountInflector