@concordance-wiki/nlp 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (121) hide show
  1. package/LICENSE +674 -0
  2. package/README.md +7 -0
  3. package/dist/dictionary/build.d.ts +19 -0
  4. package/dist/dictionary/build.d.ts.map +1 -0
  5. package/dist/dictionary/build.js +72 -0
  6. package/dist/dictionary/build.js.map +1 -0
  7. package/dist/dictionary/stopwords.d.ts +16 -0
  8. package/dist/dictionary/stopwords.d.ts.map +1 -0
  9. package/dist/dictionary/stopwords.js +27 -0
  10. package/dist/dictionary/stopwords.js.map +1 -0
  11. package/dist/dictionary/types.d.ts +34 -0
  12. package/dist/dictionary/types.d.ts.map +1 -0
  13. package/dist/dictionary/types.js +2 -0
  14. package/dist/dictionary/types.js.map +1 -0
  15. package/dist/index.d.ts +26 -0
  16. package/dist/index.d.ts.map +1 -0
  17. package/dist/index.js +24 -0
  18. package/dist/index.js.map +1 -0
  19. package/dist/keywords/confidence.d.ts +56 -0
  20. package/dist/keywords/confidence.d.ts.map +1 -0
  21. package/dist/keywords/confidence.js +59 -0
  22. package/dist/keywords/confidence.js.map +1 -0
  23. package/dist/keywords/defined.d.ts +26 -0
  24. package/dist/keywords/defined.d.ts.map +1 -0
  25. package/dist/keywords/defined.js +33 -0
  26. package/dist/keywords/defined.js.map +1 -0
  27. package/dist/keywords/findings.d.ts +13 -0
  28. package/dist/keywords/findings.d.ts.map +1 -0
  29. package/dist/keywords/findings.js +35 -0
  30. package/dist/keywords/findings.js.map +1 -0
  31. package/dist/keywords/ngrams.d.ts +47 -0
  32. package/dist/keywords/ngrams.d.ts.map +1 -0
  33. package/dist/keywords/ngrams.js +64 -0
  34. package/dist/keywords/ngrams.js.map +1 -0
  35. package/dist/keywords/options.d.ts +22 -0
  36. package/dist/keywords/options.d.ts.map +1 -0
  37. package/dist/keywords/options.js +22 -0
  38. package/dist/keywords/options.js.map +1 -0
  39. package/dist/keywords/publish.d.ts +50 -0
  40. package/dist/keywords/publish.d.ts.map +1 -0
  41. package/dist/keywords/publish.js +96 -0
  42. package/dist/keywords/publish.js.map +1 -0
  43. package/dist/keywords/score.d.ts +60 -0
  44. package/dist/keywords/score.d.ts.map +1 -0
  45. package/dist/keywords/score.js +0 -0
  46. package/dist/keywords/score.js.map +1 -0
  47. package/dist/keywords/similar.d.ts +19 -0
  48. package/dist/keywords/similar.d.ts.map +1 -0
  49. package/dist/keywords/similar.js +52 -0
  50. package/dist/keywords/similar.js.map +1 -0
  51. package/dist/locale/load-pack.d.ts +11 -0
  52. package/dist/locale/load-pack.d.ts.map +1 -0
  53. package/dist/locale/load-pack.js +82 -0
  54. package/dist/locale/load-pack.js.map +1 -0
  55. package/dist/locale/normalize.d.ts +3 -0
  56. package/dist/locale/normalize.d.ts.map +1 -0
  57. package/dist/locale/normalize.js +14 -0
  58. package/dist/locale/normalize.js.map +1 -0
  59. package/dist/locale/pack.d.ts +34 -0
  60. package/dist/locale/pack.d.ts.map +1 -0
  61. package/dist/locale/pack.js +2 -0
  62. package/dist/locale/pack.js.map +1 -0
  63. package/dist/locale/registry.d.ts +16 -0
  64. package/dist/locale/registry.d.ts.map +1 -0
  65. package/dist/locale/registry.js +35 -0
  66. package/dist/locale/registry.js.map +1 -0
  67. package/dist/locale/stopwords.d.ts +3 -0
  68. package/dist/locale/stopwords.d.ts.map +1 -0
  69. package/dist/locale/stopwords.js +17 -0
  70. package/dist/locale/stopwords.js.map +1 -0
  71. package/dist/locale/suffixes.d.ts +7 -0
  72. package/dist/locale/suffixes.d.ts.map +1 -0
  73. package/dist/locale/suffixes.js +21 -0
  74. package/dist/locale/suffixes.js.map +1 -0
  75. package/dist/locale/tag.d.ts +4 -0
  76. package/dist/locale/tag.d.ts.map +1 -0
  77. package/dist/locale/tag.js +5 -0
  78. package/dist/locale/tag.js.map +1 -0
  79. package/dist/scan/automaton.d.ts +46 -0
  80. package/dist/scan/automaton.d.ts.map +1 -0
  81. package/dist/scan/automaton.js +92 -0
  82. package/dist/scan/automaton.js.map +1 -0
  83. package/dist/scan/context.d.ts +20 -0
  84. package/dist/scan/context.d.ts.map +1 -0
  85. package/dist/scan/context.js +26 -0
  86. package/dist/scan/context.js.map +1 -0
  87. package/dist/scan/occurrences.d.ts +64 -0
  88. package/dist/scan/occurrences.d.ts.map +1 -0
  89. package/dist/scan/occurrences.js +109 -0
  90. package/dist/scan/occurrences.js.map +1 -0
  91. package/dist/scan/tokens.d.ts +12 -0
  92. package/dist/scan/tokens.d.ts.map +1 -0
  93. package/dist/scan/tokens.js +34 -0
  94. package/dist/scan/tokens.js.map +1 -0
  95. package/dist/text/boundaries.d.ts +12 -0
  96. package/dist/text/boundaries.d.ts.map +1 -0
  97. package/dist/text/boundaries.js +13 -0
  98. package/dist/text/boundaries.js.map +1 -0
  99. package/dist/text/comparison-form.d.ts +9 -0
  100. package/dist/text/comparison-form.d.ts.map +1 -0
  101. package/dist/text/comparison-form.js +23 -0
  102. package/dist/text/comparison-form.js.map +1 -0
  103. package/dist/text/context.d.ts +6 -0
  104. package/dist/text/context.d.ts.map +1 -0
  105. package/dist/text/context.js +12 -0
  106. package/dist/text/context.js.map +1 -0
  107. package/dist/text/search-tokens.d.ts +10 -0
  108. package/dist/text/search-tokens.d.ts.map +1 -0
  109. package/dist/text/search-tokens.js +29 -0
  110. package/dist/text/search-tokens.js.map +1 -0
  111. package/dist/text/singular.d.ts +7 -0
  112. package/dist/text/singular.d.ts.map +1 -0
  113. package/dist/text/singular.js +12 -0
  114. package/dist/text/singular.js.map +1 -0
  115. package/locales/en/pack.yaml +18 -0
  116. package/locales/en/stopwords.txt +536 -0
  117. package/locales/en/suffixes.txt +7 -0
  118. package/locales/fr/pack.yaml +15 -0
  119. package/locales/fr/stopwords.txt +1021 -0
  120. package/locales/fr/suffixes.txt +12 -0
  121. package/package.json +44 -0
@@ -0,0 +1,12 @@
1
+ # Endings of the inflected forms of French: a candidate expression whose word ends with one
2
+ # is more often a conjugated verb or an adverb of the prose than a term, and its confidence
3
+ # takes a soft penalty, never an elimination ("document", "paiement" are terms). One suffix
4
+ # per line, the leading hyphen optional; compared on the comparison form of the words, after
5
+ # the plural rules of the pack: an ending in -s ("-ons") would never match, "faisons" being
6
+ # compared as "faison", so none is listed.
7
+ -ment
8
+ -ait
9
+ -aient
10
+ -ez
11
+ -era
12
+ -erait
package/package.json ADDED
@@ -0,0 +1,44 @@
1
+ {
2
+ "name": "@concordance-wiki/nlp",
3
+ "version": "0.1.0",
4
+ "description": "Normalisation, Aho-Corasick, n-grams, C-value, MinHash and the language packs.",
5
+ "license": "GPL-3.0-or-later",
6
+ "repository": {
7
+ "type": "git",
8
+ "url": "git+https://github.com/concordance-wiki/concordance.git",
9
+ "directory": "packages/nlp"
10
+ },
11
+ "homepage": "https://github.com/concordance-wiki/concordance/tree/main/packages/nlp#readme",
12
+ "bugs": "https://github.com/concordance-wiki/concordance/issues",
13
+ "type": "module",
14
+ "sideEffects": false,
15
+ "engines": {
16
+ "node": ">=22"
17
+ },
18
+ "publishConfig": {
19
+ "access": "public"
20
+ },
21
+ "exports": {
22
+ ".": {
23
+ "types": "./dist/index.d.ts",
24
+ "default": "./dist/index.js"
25
+ }
26
+ },
27
+ "files": [
28
+ "dist",
29
+ "locales",
30
+ "README.md",
31
+ "LICENSE"
32
+ ],
33
+ "scripts": {
34
+ "typecheck": "tsc -b tsconfig.json"
35
+ },
36
+ "dependencies": {
37
+ "@concordance-wiki/core": "workspace:*",
38
+ "ajv": "8.17.1",
39
+ "yaml": "2.8.1"
40
+ },
41
+ "devDependencies": {
42
+ "@concordance-wiki/ingest": "workspace:*"
43
+ }
44
+ }