natural 5.0.5 → 5.1.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/codeql-analysis.yml +70 -0
- package/CONTRIBUTING.md +30 -6
- package/README.md +0 -1
- package/SECURITY.md +23 -0
- package/lib/natural/classifiers/maxent/Classifier.js +18 -13
- package/lib/natural/classifiers/maxent/Context.js +21 -16
- package/lib/natural/classifiers/maxent/Distribution.js +20 -15
- package/lib/natural/classifiers/maxent/Element.js +17 -12
- package/lib/natural/classifiers/maxent/Feature.js +18 -13
- package/lib/natural/classifiers/maxent/FeatureSet.js +17 -12
- package/lib/natural/classifiers/maxent/GISScaler.js +20 -15
- package/lib/natural/classifiers/maxent/POS/ME_Corpus.js +20 -15
- package/lib/natural/classifiers/maxent/POS/ME_Sentence.js +17 -12
- package/lib/natural/classifiers/maxent/POS/POS_Element.js +20 -15
- package/lib/natural/classifiers/maxent/Sample.js +20 -15
- package/lib/natural/classifiers/maxent/SimpleExample/SE_Element.js +20 -15
- package/lib/natural/distance/dice_coefficient.js +1 -1
- package/lib/natural/tokenizers/aggressive_tokenizer_fr.js +1 -1
- package/lib/natural/tokenizers/aggressive_tokenizer_nl.js +1 -1
- package/package.json +2 -2
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
# For most projects, this workflow file will not need changing; you simply need
|
|
2
|
+
# to commit it to your repository.
|
|
3
|
+
#
|
|
4
|
+
# You may wish to alter this file to override the set of languages analyzed,
|
|
5
|
+
# or to provide custom queries or build logic.
|
|
6
|
+
#
|
|
7
|
+
# ******** NOTE ********
|
|
8
|
+
# We have attempted to detect the languages in your repository. Please check
|
|
9
|
+
# the `language` matrix defined below to confirm you have the correct set of
|
|
10
|
+
# supported CodeQL languages.
|
|
11
|
+
#
|
|
12
|
+
name: "CodeQL"
|
|
13
|
+
|
|
14
|
+
on:
|
|
15
|
+
push:
|
|
16
|
+
branches: [ master ]
|
|
17
|
+
pull_request:
|
|
18
|
+
# The branches below must be a subset of the branches above
|
|
19
|
+
branches: [ master ]
|
|
20
|
+
schedule:
|
|
21
|
+
- cron: '38 9 * * 4'
|
|
22
|
+
|
|
23
|
+
jobs:
|
|
24
|
+
analyze:
|
|
25
|
+
name: Analyze
|
|
26
|
+
runs-on: ubuntu-latest
|
|
27
|
+
permissions:
|
|
28
|
+
actions: read
|
|
29
|
+
contents: read
|
|
30
|
+
security-events: write
|
|
31
|
+
|
|
32
|
+
strategy:
|
|
33
|
+
fail-fast: false
|
|
34
|
+
matrix:
|
|
35
|
+
language: [ 'javascript' ]
|
|
36
|
+
# CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
|
|
37
|
+
# Learn more about CodeQL language support at https://git.io/codeql-language-support
|
|
38
|
+
|
|
39
|
+
steps:
|
|
40
|
+
- name: Checkout repository
|
|
41
|
+
uses: actions/checkout@v2
|
|
42
|
+
|
|
43
|
+
# Initializes the CodeQL tools for scanning.
|
|
44
|
+
- name: Initialize CodeQL
|
|
45
|
+
uses: github/codeql-action/init@v1
|
|
46
|
+
with:
|
|
47
|
+
languages: ${{ matrix.language }}
|
|
48
|
+
# If you wish to specify custom queries, you can do so here or in a config file.
|
|
49
|
+
# By default, queries listed here will override any specified in a config file.
|
|
50
|
+
# Prefix the list here with "+" to use these queries and those in the config file.
|
|
51
|
+
# queries: ./path/to/local/query, your-org/your-repo/queries@main
|
|
52
|
+
|
|
53
|
+
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
|
|
54
|
+
# If this step fails, then you should remove it and run the build manually (see below)
|
|
55
|
+
- name: Autobuild
|
|
56
|
+
uses: github/codeql-action/autobuild@v1
|
|
57
|
+
|
|
58
|
+
# ℹ️ Command-line programs to run using the OS shell.
|
|
59
|
+
# 📚 https://git.io/JvXDl
|
|
60
|
+
|
|
61
|
+
# ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
|
|
62
|
+
# and modify them (or add more) to build your code if your project
|
|
63
|
+
# uses a compiled language
|
|
64
|
+
|
|
65
|
+
#- run: |
|
|
66
|
+
# make bootstrap
|
|
67
|
+
# make release
|
|
68
|
+
|
|
69
|
+
- name: Perform CodeQL Analysis
|
|
70
|
+
uses: github/codeql-action/analyze@v1
|
package/CONTRIBUTING.md
CHANGED
|
@@ -1,9 +1,33 @@
|
|
|
1
|
-
|
|
1
|
+
# Contributing
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
When contributing to this repository, please explain your changes or additions in an issue or in the pull request.
|
|
4
4
|
|
|
5
|
-
|
|
6
|
-
* `npm test` or
|
|
7
|
-
* `make test`
|
|
5
|
+
When developing, please:
|
|
8
6
|
|
|
9
|
-
|
|
7
|
+
* Comply with Standard JS. This is tested when you commit to natural.
|
|
8
|
+
* Prevent code duplication. Use jscpd to test this. This is tested when you commit to natural.
|
|
9
|
+
* Write unit tests for jasmine. Make sure your unit tests pass.
|
|
10
|
+
* For compatibility with browsers, do not use the file system fs. If you need to read files, use JSON and require.
|
|
11
|
+
* Write documentation and commit this to the gh-pages branch.
|
|
12
|
+
* The current configuration of the unit tests requires the following environment variable to be set:
|
|
13
|
+
```
|
|
14
|
+
export NODE_PATH=.
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
# CD/CD pipeline
|
|
18
|
+
The repository uses Github Actions for testing and publishing.
|
|
19
|
+
|
|
20
|
+
Currently, there three workflows:
|
|
21
|
+
|
|
22
|
+
* Node CI: tests natural on the four most recent releases of Nodejs. Code coverage is analysed with Istanbul. Results are forwarded to Coveralls. In the README a badge shows the coverage.
|
|
23
|
+
* Nodejs package: The natural package is automatically published to NPM when a release is created (tagged).
|
|
24
|
+
* Lint Code Base: the repository is linted using the Github super linter which includes jslint and jscpd.
|
|
25
|
+
* CodeQL for detecting security vulnerabilities.
|
|
26
|
+
Github workflows can be found in the folder ./.github/workflows.
|
|
27
|
+
|
|
28
|
+
# Versioning
|
|
29
|
+
We apply semantic versioning. Given natural’s version number MAJOR.MINOR.PATCH, increment the:
|
|
30
|
+
|
|
31
|
+
* MAJOR version when you make incompatible API changes,
|
|
32
|
+
* MINOR version when you add functionality in a backwards compatible manner, and
|
|
33
|
+
* PATCH version when you make backwards compatible bug fixes.
|
package/README.md
CHANGED
|
@@ -6,7 +6,6 @@
|
|
|
6
6
|
[](https://github.com/marketplace/actions/super-linter)
|
|
7
7
|
[](https://coveralls.io/github/NaturalNode/natural?branch=master)
|
|
8
8
|
[](https://bestpractices.coreinfrastructure.org/projects/4651)
|
|
9
|
-
[](https://slack.bri.im)
|
|
10
9
|
|
|
11
10
|
"Natural" is a general natural language facility for nodejs. It offers a broad range of functionalities for natural language processing. Documentation can be found [here on Github Pages](http://naturalnode.github.io/natural/).
|
|
12
11
|
|
package/SECURITY.md
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# Security Policy
|
|
2
|
+
|
|
3
|
+
##
|
|
4
|
+
|
|
5
|
+
We run dependabot and Github's CodeQL analysis to detect security problems.
|
|
6
|
+
|
|
7
|
+
## Supported Versions
|
|
8
|
+
|
|
9
|
+
Use this section to tell people about which versions of your project are
|
|
10
|
+
currently being supported with security updates.
|
|
11
|
+
|
|
12
|
+
| Version | Supported |
|
|
13
|
+
| ------- | ------------------ |
|
|
14
|
+
| 5.1.10 | :white_check_mark: |
|
|
15
|
+
| 5.0.x | :x: |
|
|
16
|
+
| 4.0.x | :x: |
|
|
17
|
+
| < 4.0 | :x: |
|
|
18
|
+
|
|
19
|
+
## Reporting a Vulnerability
|
|
20
|
+
|
|
21
|
+
If you find a vulnerability please report a security issue in the Issues section of the repository. If you know how to fix it, create a pull request.
|
|
22
|
+
|
|
23
|
+
We will analyse the issue, and repair it a.s.a.p (best effort).
|
|
@@ -1,20 +1,25 @@
|
|
|
1
1
|
/*
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
2
|
+
Classifier class that provides functionality for training and
|
|
3
|
+
classification
|
|
4
|
+
Copyright (C) 2017 Hugo W.L. ter Doest
|
|
5
5
|
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
6
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
7
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
8
|
+
in the Software without restriction, including without limitation the rights
|
|
9
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
10
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
11
|
+
furnished to do so, subject to the following conditions:
|
|
10
12
|
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
14
|
-
GNU General Public License for more details.
|
|
13
|
+
The above copyright notice and this permission notice shall be included in
|
|
14
|
+
all copies or substantial portions of the Software.
|
|
15
15
|
|
|
16
|
-
|
|
17
|
-
|
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
17
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
18
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
19
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
20
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
21
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
22
|
+
THE SOFTWARE.
|
|
18
23
|
*/
|
|
19
24
|
|
|
20
25
|
'use strict'
|
|
@@ -1,24 +1,29 @@
|
|
|
1
1
|
/*
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
2
|
+
Context class
|
|
3
|
+
Copyright (C) 2017 Hugo W.L. ter Doest
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
|
13
|
+
all copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
21
|
+
THE SOFTWARE.
|
|
17
22
|
*/
|
|
18
23
|
|
|
19
24
|
'use strict'
|
|
20
25
|
|
|
21
|
-
const stringify = require('
|
|
26
|
+
const stringify = require('safe-stable-stringify')
|
|
22
27
|
|
|
23
28
|
function Context (data) {
|
|
24
29
|
this.data = data
|
|
@@ -1,19 +1,24 @@
|
|
|
1
1
|
/*
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
2
|
+
Distribution class for probability distributions
|
|
3
|
+
Copyright (C) 2017 Hugo W.L. ter Doest
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
|
13
|
+
all copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
21
|
+
THE SOFTWARE.
|
|
17
22
|
*/
|
|
18
23
|
|
|
19
24
|
'use strict'
|
|
@@ -1,19 +1,24 @@
|
|
|
1
1
|
/*
|
|
2
|
-
|
|
3
|
-
|
|
2
|
+
Element class for elements in the event space
|
|
3
|
+
Copyright (C) 2017 Hugo W.L. ter Doest
|
|
4
4
|
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
9
11
|
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
13
|
-
GNU General Public License for more details.
|
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
|
13
|
+
all copies or substantial portions of the Software.
|
|
14
14
|
|
|
15
|
-
|
|
16
|
-
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
21
|
+
THE SOFTWARE.
|
|
17
22
|
*/
|
|
18
23
|
|
|
19
24
|
'use strict'
|
|
@@ -1,20 +1,25 @@
|
|
|
1
1
|
/*
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
2
|
+
Feature class for features that fire (or don't) on combinations of context
|
|
3
|
+
and class
|
|
4
|
+
Copyright (C) 2017 Hugo W.L. ter Doest
|
|
5
5
|
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
6
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
7
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
8
|
+
in the Software without restriction, including without limitation the rights
|
|
9
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
10
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
11
|
+
furnished to do so, subject to the following conditions:
|
|
10
12
|
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
14
|
-
GNU General Public License for more details.
|
|
13
|
+
The above copyright notice and this permission notice shall be included in
|
|
14
|
+
all copies or substantial portions of the Software.
|
|
15
15
|
|
|
16
|
-
|
|
17
|
-
|
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
17
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
18
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
19
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
20
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
21
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
22
|
+
THE SOFTWARE.
|
|
18
23
|
*/
|
|
19
24
|
|
|
20
25
|
'use strict'
|
|
@@ -1,19 +1,24 @@
|
|
|
1
1
|
/*
|
|
2
|
-
|
|
3
|
-
|
|
2
|
+
Feature set class for administrating a set of unique feature
|
|
3
|
+
Copyright (C) 2017 Hugo W.L. ter Doest
|
|
4
4
|
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
9
11
|
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
13
|
-
GNU General Public License for more details.
|
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
|
13
|
+
all copies or substantial portions of the Software.
|
|
14
14
|
|
|
15
|
-
|
|
16
|
-
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
21
|
+
THE SOFTWARE.
|
|
17
22
|
*/
|
|
18
23
|
|
|
19
24
|
'use strict'
|
|
@@ -1,19 +1,24 @@
|
|
|
1
1
|
/*
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
2
|
+
GISScaler class that finds parameters of features
|
|
3
|
+
Copyright (C) 2017 Hugo W.L. ter Doest
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
|
13
|
+
all copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
21
|
+
THE SOFTWARE.
|
|
17
22
|
*/
|
|
18
23
|
|
|
19
24
|
'use strict'
|
|
@@ -1,19 +1,24 @@
|
|
|
1
1
|
/*
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
2
|
+
Corpus class specific for MaxEnt modeling
|
|
3
|
+
Copyright (C) 2018 Hugo W.L. ter Doest
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
|
13
|
+
all copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
21
|
+
THE SOFTWARE.
|
|
17
22
|
*/
|
|
18
23
|
|
|
19
24
|
'use strict'
|
|
@@ -1,19 +1,24 @@
|
|
|
1
1
|
/*
|
|
2
|
-
|
|
3
|
-
|
|
2
|
+
Sentence class specific for MaxEnt modeling
|
|
3
|
+
Copyright (C) 2019 Hugo W.L. ter Doest
|
|
4
4
|
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
9
11
|
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
13
|
-
GNU General Public License for more details.
|
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
|
13
|
+
all copies or substantial portions of the Software.
|
|
14
14
|
|
|
15
|
-
|
|
16
|
-
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
21
|
+
THE SOFTWARE.
|
|
17
22
|
*/
|
|
18
23
|
|
|
19
24
|
'use strict'
|
|
@@ -1,19 +1,24 @@
|
|
|
1
1
|
/*
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
2
|
+
Element class for POS tagging
|
|
3
|
+
Copyright (C) 2018 Hugo W.L. ter Doest
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
|
13
|
+
all copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
21
|
+
THE SOFTWARE.
|
|
17
22
|
*/
|
|
18
23
|
|
|
19
24
|
'use strict'
|
|
@@ -1,19 +1,24 @@
|
|
|
1
1
|
/*
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
2
|
+
Sample space of observed events
|
|
3
|
+
Copyright (C) 2018 Hugo W.L. ter Doest
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
|
13
|
+
all copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
21
|
+
THE SOFTWARE.
|
|
17
22
|
*/
|
|
18
23
|
|
|
19
24
|
'use strict'
|
|
@@ -1,19 +1,24 @@
|
|
|
1
1
|
/*
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
2
|
+
Simple Example Element class
|
|
3
|
+
Copyright (C) 2018 Hugo W.L. ter Doest
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
|
13
|
+
all copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
21
|
+
THE SOFTWARE.
|
|
17
22
|
*/
|
|
18
23
|
|
|
19
24
|
'use strict'
|
|
@@ -51,7 +51,7 @@ function intersect (set1, set2) {
|
|
|
51
51
|
function sanitize (str) {
|
|
52
52
|
// Turn characters to lower string, remove space at the beginning and end,
|
|
53
53
|
// replace multiple spaces in the middle by single spaces
|
|
54
|
-
return str.toLowerCase().replace(
|
|
54
|
+
return str.toLowerCase().replace(/\s+/g, ' ').replace(/^\s+|\s+$/g, '')
|
|
55
55
|
}
|
|
56
56
|
|
|
57
57
|
function diceCoefficient (str1, str2) {
|
|
@@ -34,5 +34,5 @@ module.exports = AggressiveTokenizer
|
|
|
34
34
|
|
|
35
35
|
AggressiveTokenizer.prototype.tokenize = function (text) {
|
|
36
36
|
// break a string up into an array of tokens by anything non-word
|
|
37
|
-
return this.trim(text.split(/[^a-z0-9
|
|
37
|
+
return this.trim(text.split(/[^a-z0-9äâàéèëêïîöôùüûœç-]+/i))
|
|
38
38
|
}
|
|
@@ -34,5 +34,5 @@ module.exports = AggressiveTokenizer
|
|
|
34
34
|
|
|
35
35
|
AggressiveTokenizer.prototype.tokenize = function (text) {
|
|
36
36
|
// break a string up into an array of tokens by anything non-word
|
|
37
|
-
return this.trim(text.split(/[^a-zA-Z0-9_']+/))
|
|
37
|
+
return this.trim(text.split(/[^a-zA-Z0-9_'-]+/))
|
|
38
38
|
}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "natural",
|
|
3
3
|
"description": "General natural language (tokenizing, stemming (English, Russian, Spanish), part-of-speech tagging, sentiment analysis, classification, inflection, phonetics, tfidf, WordNet, jaro-winkler, Levenshtein distance, Dice's Coefficient) facilities for node.",
|
|
4
|
-
"version": "5.
|
|
4
|
+
"version": "5.1.11",
|
|
5
5
|
"homepage": "https://github.com/NaturalNode/natural",
|
|
6
6
|
"repository": {
|
|
7
7
|
"type": "git",
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
"dependencies": {
|
|
14
14
|
"afinn-165": "^1.0.2",
|
|
15
15
|
"apparatus": "^0.0.10",
|
|
16
|
-
"
|
|
16
|
+
"safe-stable-stringify": "^2.2.0",
|
|
17
17
|
"sylvester": "^0.0.12",
|
|
18
18
|
"underscore": "^1.9.1",
|
|
19
19
|
"wordnet-db": "^3.1.11"
|