mittens 0.3.0 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +8 -0
- data/Gemfile +2 -1
- data/README.md +2 -2
- data/Rakefile +12 -5
- data/ext/mittens/extconf.rb +3 -1
- data/lib/mittens/version.rb +1 -1
- data/vendor/snowball/.github/workflows/ci.yml +144 -17
- data/vendor/snowball/.github/workflows/coverage.yml +117 -0
- data/vendor/snowball/.github/workflows/runtime-tests.yml +26 -0
- data/vendor/snowball/.gitignore +33 -7
- data/vendor/snowball/CONTRIBUTING.rst +7 -0
- data/vendor/snowball/COPYING +1 -1
- data/vendor/snowball/GNUmakefile +736 -298
- data/vendor/snowball/NEWS +1394 -7
- data/vendor/snowball/README.rst +8 -8
- data/vendor/snowball/ada/generate/generate.adb +5 -5
- data/vendor/snowball/ada/src/stemmer.adb +177 -188
- data/vendor/snowball/ada/src/stemmer.ads +86 -89
- data/vendor/snowball/ada/src/stemwords.adb +21 -5
- data/vendor/snowball/algorithms/czech.sbl +255 -0
- data/vendor/snowball/algorithms/danish.sbl +22 -10
- data/vendor/snowball/algorithms/english.sbl +7 -4
- data/vendor/snowball/algorithms/esperanto.sbl +6 -7
- data/vendor/snowball/algorithms/estonian.sbl +9 -4
- data/vendor/snowball/algorithms/finnish.sbl +33 -21
- data/vendor/snowball/algorithms/german.sbl +5 -0
- data/vendor/snowball/algorithms/indonesian.sbl +115 -96
- data/vendor/snowball/algorithms/italian.sbl +26 -1
- data/vendor/snowball/algorithms/lithuanian.sbl +13 -13
- data/vendor/snowball/algorithms/norwegian.sbl +17 -6
- data/vendor/snowball/algorithms/persian.sbl +387 -0
- data/vendor/snowball/algorithms/polish.sbl +177 -0
- data/vendor/snowball/algorithms/sesotho.sbl +115 -0
- data/vendor/snowball/algorithms/turkish.sbl +4 -4
- data/vendor/snowball/compiler/analyser.c +2002 -711
- data/vendor/snowball/compiler/driver.c +460 -298
- data/vendor/snowball/compiler/generator.c +439 -1965
- data/vendor/snowball/compiler/generator_ada.c +605 -430
- data/vendor/snowball/compiler/generator_c.c +2227 -0
- data/vendor/snowball/compiler/generator_csharp.c +365 -274
- data/vendor/snowball/compiler/generator_dart.c +1476 -0
- data/vendor/snowball/compiler/generator_go.c +348 -270
- data/vendor/snowball/compiler/generator_java.c +330 -233
- data/vendor/snowball/compiler/generator_js.c +534 -385
- data/vendor/snowball/compiler/generator_pascal.c +361 -320
- data/vendor/snowball/compiler/generator_php.c +1445 -0
- data/vendor/snowball/compiler/generator_python.c +399 -299
- data/vendor/snowball/compiler/generator_rust.c +370 -261
- data/vendor/snowball/compiler/generator_zig.c +1474 -0
- data/vendor/snowball/compiler/header.h +153 -86
- data/vendor/snowball/compiler/space.c +121 -63
- data/vendor/snowball/compiler/tokeniser.c +286 -182
- data/vendor/snowball/compiler/tokens.h +71 -0
- data/vendor/snowball/csharp/Snowball/Among.cs +14 -26
- data/vendor/snowball/csharp/Snowball/AssemblyInfo.cs +3 -3
- data/vendor/snowball/csharp/Snowball/Stemmer.cs +41 -45
- data/vendor/snowball/csharp/Stemwords/Program.cs +20 -14
- data/vendor/snowball/cxx/.gitignore +4 -0
- data/vendor/snowball/cxx/generate_algorithms.pl +87 -0
- data/vendor/snowball/cxx/stemmer.h +12 -0
- data/vendor/snowball/cxx/stemwords.cxx +176 -0
- data/vendor/snowball/cxx/utilities.cxx +2 -0
- data/vendor/snowball/dart/.gitignore +4 -0
- data/vendor/snowball/dart/analysis_options.yaml +1 -0
- data/vendor/snowball/dart/example/test_app.dart +65 -0
- data/vendor/snowball/dart/generate_algorithms.pl +21 -0
- data/vendor/snowball/dart/lib/snowball.dart +18 -0
- data/vendor/snowball/dart/lib/src/snowball.dart +339 -0
- data/vendor/snowball/dart/pubspec.yaml +17 -0
- data/vendor/snowball/doc/libstemmer_dart_README +37 -0
- data/vendor/snowball/doc/libstemmer_js_README +2 -2
- data/vendor/snowball/doc/libstemmer_php_README +38 -0
- data/vendor/snowball/doc/libstemmer_python_README +3 -3
- data/vendor/snowball/examples/stemwords.c +18 -3
- data/vendor/snowball/go/README.md +12 -4
- data/vendor/snowball/go/env.go +12 -14
- data/vendor/snowball/go/stemwords/main.go +2 -1
- data/vendor/snowball/java/org/tartarus/snowball/CharArraySequence.java +36 -0
- data/vendor/snowball/java/org/tartarus/snowball/SnowballProgram.java +17 -34
- data/vendor/snowball/java/org/tartarus/snowball/SnowballStemmer.java +0 -2
- data/vendor/snowball/javascript/base-stemmer.js +219 -252
- data/vendor/snowball/javascript/stemwords.js +86 -58
- data/vendor/snowball/libstemmer/mkmodules.pl +1 -1
- data/vendor/snowball/libstemmer/modules.txt +4 -0
- data/vendor/snowball/pascal/SnowballProgram.pas +18 -31
- data/vendor/snowball/php/base-stemmer.php +453 -0
- data/vendor/snowball/php/stemwords.php +25 -0
- data/vendor/snowball/python/create_init.py +16 -21
- data/vendor/snowball/python/pyproject.toml +3 -0
- data/vendor/snowball/python/setup.py +5 -7
- data/vendor/snowball/python/snowballstemmer/among.py +1 -2
- data/vendor/snowball/python/snowballstemmer/basestemmer.py +24 -33
- data/vendor/snowball/python/stemwords.py +72 -63
- data/vendor/snowball/runtime/api.c +11 -42
- data/vendor/snowball/runtime/api.h +11 -9
- data/vendor/snowball/runtime/snowball_runtime.h +110 -0
- data/vendor/snowball/runtime/utilities.c +282 -106
- data/vendor/snowball/rust/src/main.rs +2 -2
- data/vendor/snowball/rust/src/snowball/snowball_env.rs +7 -9
- data/vendor/snowball/tests/compilertest +62 -0
- data/vendor/snowball/tests/errors/ae-errors.sbl +53 -0
- data/vendor/snowball/tests/errors/ae-errors.stderr +8 -0
- data/vendor/snowball/tests/errors/bad-dollar.sbl +14 -0
- data/vendor/snowball/tests/errors/bad-dollar.stderr +4 -0
- data/vendor/snowball/tests/errors/bad-grouping-definition.sbl +9 -0
- data/vendor/snowball/tests/errors/bad-grouping-definition.stderr +6 -0
- data/vendor/snowball/tests/errors/missing-bra.sbl +6 -0
- data/vendor/snowball/tests/errors/missing-bra.stderr +3 -0
- data/vendor/snowball/tests/errors/missing-command.sbl +9 -0
- data/vendor/snowball/tests/errors/missing-command.stderr +1 -0
- data/vendor/snowball/tests/errors/missing-ket-backwardmode.sbl +3 -0
- data/vendor/snowball/tests/errors/missing-ket-backwardmode.stderr +1 -0
- data/vendor/snowball/tests/errors/missing-ket.sbl +7 -0
- data/vendor/snowball/tests/errors/missing-ket.stderr +6 -0
- data/vendor/snowball/tests/errors/notdefined.sbl +4 -0
- data/vendor/snowball/tests/errors/notdefined.stderr +2 -0
- data/vendor/snowball/tests/errors/string-omitted.sbl +5 -0
- data/vendor/snowball/tests/errors/string-omitted.stderr +1 -0
- data/vendor/snowball/tests/errors/undeclared.sbl +20 -0
- data/vendor/snowball/tests/errors/undeclared.stderr +9 -0
- data/vendor/snowball/tests/errors/unexpected-token.sbl +24 -0
- data/vendor/snowball/tests/errors/unexpected-token.stderr +17 -0
- data/vendor/snowball/tests/errors/wrongdirection.sbl +14 -0
- data/vendor/snowball/tests/errors/wrongdirection.stderr +1 -0
- data/vendor/snowball/tests/runtime/among.sbl +19 -0
- data/vendor/snowball/tests/runtime/arithmeticexpr.sbl +86 -0
- data/vendor/snowball/tests/runtime/attachinsert.sbl +47 -0
- data/vendor/snowball/tests/runtime/booleans.sbl +19 -0
- data/vendor/snowball/tests/runtime/externals.sbl +22 -0
- data/vendor/snowball/tests/runtime/hop.sbl +12 -0
- data/vendor/snowball/tests/runtime/integertests.sbl +60 -0
- data/vendor/snowball/tests/runtime/intlimits.sbl +25 -0
- data/vendor/snowball/tests/runtime/naming.sbl +20 -0
- data/vendor/snowball/tests/runtime/not.sbl +30 -0
- data/vendor/snowball/tests/runtime/or.sbl +17 -0
- data/vendor/snowball/tests/runtime/repeat.sbl +16 -0
- data/vendor/snowball/tests/runtime/setlimit.sbl +24 -0
- data/vendor/snowball/tests/runtime/sizelen.sbl +56 -0
- data/vendor/snowball/tests/runtime/slice.sbl +27 -0
- data/vendor/snowball/tests/runtime/stringdollar.sbl +39 -0
- data/vendor/snowball/tests/runtime/strings.sbl +66 -0
- data/vendor/snowball/tests/runtime/test.sbl +19 -0
- data/vendor/snowball/tests/stemtest.c +41 -1
- data/vendor/snowball/tests/syntax/canon.sbl +32 -0
- data/vendor/snowball/tests/syntax/canon.stderr +0 -0
- data/vendor/snowball/tests/syntax/canon.syntax +57 -0
- data/vendor/snowball/tests/syntax/emptyprogram.sbl +2 -0
- data/vendor/snowball/tests/syntax/emptyprogram.syntax +0 -0
- data/vendor/snowball/tests/syntax/groupings.sbl +11 -0
- data/vendor/snowball/tests/syntax/inline.sbl +144 -0
- data/vendor/snowball/tests/syntax/inline.stderr +0 -0
- data/vendor/snowball/tests/syntax/inline.syntax +121 -0
- data/vendor/snowball/tests/syntax/legacy.sbl +17 -0
- data/vendor/snowball/tests/syntax/legacy.stderr +5 -0
- data/vendor/snowball/tests/syntax/legacy.syntax +19 -0
- data/vendor/snowball/tests/syntax/loops.sbl +14 -0
- data/vendor/snowball/tests/syntax/loops.stderr +9 -0
- data/vendor/snowball/tests/syntax/loops.syntax +29 -0
- data/vendor/snowball/tests/syntax/noops.sbl +76 -0
- data/vendor/snowball/tests/syntax/noops.stderr +27 -0
- data/vendor/snowball/tests/syntax/noops.syntax +135 -0
- data/vendor/snowball/tests/syntax/simplifyae.sbl +21 -0
- data/vendor/snowball/tests/syntax/simplifyae.stderr +0 -0
- data/vendor/snowball/tests/syntax/simplifyae.syntax +39 -0
- data/vendor/snowball/tests/syntax/unused.sbl +18 -0
- data/vendor/snowball/tests/syntax/unused.stderr +7 -0
- data/vendor/snowball/tests/syntax/unused.syntax +3 -0
- data/vendor/snowball/zig/env.zig +599 -0
- data/vendor/snowball/zig/generate_algorithms.pl +19 -0
- data/vendor/snowball/zig/stemwords.zig +123 -0
- metadata +102 -4
- data/vendor/snowball/compiler/syswords.h +0 -86
- data/vendor/snowball/runtime/header.h +0 -62
|
@@ -1,288 +1,277 @@
|
|
|
1
1
|
// @ts-check
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
3
|
+
export default class BaseStemmer {
|
|
4
|
+
constructor() {
|
|
5
|
+
/** @protected */
|
|
6
|
+
this.current = '';
|
|
7
|
+
this.c = 0;
|
|
8
|
+
this.limit = 0;
|
|
9
|
+
this.limit_backward = 0;
|
|
10
|
+
this.bra = 0;
|
|
11
|
+
this.ket = 0;
|
|
12
|
+
this.af = 0;
|
|
13
|
+
}
|
|
12
14
|
|
|
13
15
|
/**
|
|
14
16
|
* @param {string} value
|
|
15
17
|
*/
|
|
16
|
-
|
|
18
|
+
setCurrent(value) {
|
|
17
19
|
this.current = value;
|
|
18
|
-
this.
|
|
20
|
+
this.c = 0;
|
|
19
21
|
this.limit = this.current.length;
|
|
20
22
|
this.limit_backward = 0;
|
|
21
|
-
this.bra = this.
|
|
23
|
+
this.bra = this.c;
|
|
22
24
|
this.ket = this.limit;
|
|
23
|
-
}
|
|
25
|
+
}
|
|
24
26
|
|
|
25
27
|
/**
|
|
26
28
|
* @return {string}
|
|
27
29
|
*/
|
|
28
|
-
|
|
30
|
+
getCurrent() {
|
|
29
31
|
return this.current;
|
|
30
|
-
}
|
|
32
|
+
}
|
|
31
33
|
|
|
32
34
|
/**
|
|
33
35
|
* @param {BaseStemmer} other
|
|
34
36
|
*/
|
|
35
|
-
|
|
37
|
+
copy_from(other) {
|
|
36
38
|
/** @protected */
|
|
37
39
|
this.current = other.current;
|
|
38
|
-
this.
|
|
40
|
+
this.c = other.c;
|
|
39
41
|
this.limit = other.limit;
|
|
40
42
|
this.limit_backward = other.limit_backward;
|
|
41
43
|
this.bra = other.bra;
|
|
42
44
|
this.ket = other.ket;
|
|
43
|
-
}
|
|
45
|
+
}
|
|
44
46
|
|
|
45
47
|
/**
|
|
46
|
-
* @param {number
|
|
48
|
+
* @param {Array<number>} s
|
|
47
49
|
* @param {number} min
|
|
48
50
|
* @param {number} max
|
|
49
51
|
* @return {boolean}
|
|
50
52
|
*/
|
|
51
|
-
|
|
53
|
+
in_grouping(s, min, max) {
|
|
52
54
|
/** @protected */
|
|
53
|
-
if (this.
|
|
54
|
-
|
|
55
|
+
if (this.c >= this.limit) return false;
|
|
56
|
+
let ch = this.current.charCodeAt(this.c);
|
|
55
57
|
if (ch > max || ch < min) return false;
|
|
56
58
|
ch -= min;
|
|
57
|
-
if ((s[ch >>> 3] & (0x1 << (ch & 0x7)))
|
|
58
|
-
this.
|
|
59
|
+
if ((s[ch >>> 3] & (0x1 << (ch & 0x7))) === 0) return false;
|
|
60
|
+
this.c++;
|
|
59
61
|
return true;
|
|
60
|
-
}
|
|
62
|
+
}
|
|
61
63
|
|
|
62
64
|
/**
|
|
63
|
-
* @param {number
|
|
65
|
+
* @param {Array<number>} s
|
|
64
66
|
* @param {number} min
|
|
65
67
|
* @param {number} max
|
|
66
68
|
* @return {boolean}
|
|
67
69
|
*/
|
|
68
|
-
|
|
70
|
+
go_in_grouping(s, min, max) {
|
|
69
71
|
/** @protected */
|
|
70
|
-
while (this.
|
|
71
|
-
|
|
72
|
+
while (this.c < this.limit) {
|
|
73
|
+
let ch = this.current.charCodeAt(this.c);
|
|
72
74
|
if (ch > max || ch < min)
|
|
73
75
|
return true;
|
|
74
76
|
ch -= min;
|
|
75
|
-
if ((s[ch >>> 3] & (0x1 << (ch & 0x7)))
|
|
77
|
+
if ((s[ch >>> 3] & (0x1 << (ch & 0x7))) === 0)
|
|
76
78
|
return true;
|
|
77
|
-
this.
|
|
79
|
+
this.c++;
|
|
78
80
|
}
|
|
79
81
|
return false;
|
|
80
|
-
}
|
|
82
|
+
}
|
|
81
83
|
|
|
82
84
|
/**
|
|
83
|
-
* @param {number
|
|
85
|
+
* @param {Array<number>} s
|
|
84
86
|
* @param {number} min
|
|
85
87
|
* @param {number} max
|
|
86
88
|
* @return {boolean}
|
|
87
89
|
*/
|
|
88
|
-
|
|
90
|
+
in_grouping_b(s, min, max) {
|
|
89
91
|
/** @protected */
|
|
90
|
-
if (this.
|
|
91
|
-
|
|
92
|
+
if (this.c <= this.limit_backward) return false;
|
|
93
|
+
let ch = this.current.charCodeAt(this.c - 1);
|
|
92
94
|
if (ch > max || ch < min) return false;
|
|
93
95
|
ch -= min;
|
|
94
|
-
if ((s[ch >>> 3] & (0x1 << (ch & 0x7)))
|
|
95
|
-
this.
|
|
96
|
+
if ((s[ch >>> 3] & (0x1 << (ch & 0x7))) === 0) return false;
|
|
97
|
+
this.c--;
|
|
96
98
|
return true;
|
|
97
|
-
}
|
|
99
|
+
}
|
|
98
100
|
|
|
99
101
|
/**
|
|
100
|
-
* @param {number
|
|
102
|
+
* @param {Array<number>} s
|
|
101
103
|
* @param {number} min
|
|
102
104
|
* @param {number} max
|
|
103
105
|
* @return {boolean}
|
|
104
106
|
*/
|
|
105
|
-
|
|
107
|
+
go_in_grouping_b(s, min, max) {
|
|
106
108
|
/** @protected */
|
|
107
|
-
while (this.
|
|
108
|
-
|
|
109
|
+
while (this.c > this.limit_backward) {
|
|
110
|
+
let ch = this.current.charCodeAt(this.c - 1);
|
|
109
111
|
if (ch > max || ch < min) return true;
|
|
110
112
|
ch -= min;
|
|
111
|
-
if ((s[ch >>> 3] & (0x1 << (ch & 0x7)))
|
|
112
|
-
this.
|
|
113
|
+
if ((s[ch >>> 3] & (0x1 << (ch & 0x7))) === 0) return true;
|
|
114
|
+
this.c--;
|
|
113
115
|
}
|
|
114
116
|
return false;
|
|
115
|
-
}
|
|
117
|
+
}
|
|
116
118
|
|
|
117
119
|
/**
|
|
118
|
-
* @param {number
|
|
120
|
+
* @param {Array<number>} s
|
|
119
121
|
* @param {number} min
|
|
120
122
|
* @param {number} max
|
|
121
123
|
* @return {boolean}
|
|
122
124
|
*/
|
|
123
|
-
|
|
125
|
+
out_grouping(s, min, max) {
|
|
124
126
|
/** @protected */
|
|
125
|
-
if (this.
|
|
126
|
-
|
|
127
|
+
if (this.c >= this.limit) return false;
|
|
128
|
+
let ch = this.current.charCodeAt(this.c);
|
|
127
129
|
if (ch > max || ch < min) {
|
|
128
|
-
this.
|
|
130
|
+
this.c++;
|
|
129
131
|
return true;
|
|
130
132
|
}
|
|
131
133
|
ch -= min;
|
|
132
|
-
if ((s[ch >>> 3] & (0X1 << (ch & 0x7)))
|
|
133
|
-
this.
|
|
134
|
+
if ((s[ch >>> 3] & (0X1 << (ch & 0x7))) === 0) {
|
|
135
|
+
this.c++;
|
|
134
136
|
return true;
|
|
135
137
|
}
|
|
136
138
|
return false;
|
|
137
|
-
}
|
|
139
|
+
}
|
|
138
140
|
|
|
139
141
|
/**
|
|
140
|
-
* @param {number
|
|
142
|
+
* @param {Array<number>} s
|
|
141
143
|
* @param {number} min
|
|
142
144
|
* @param {number} max
|
|
143
145
|
* @return {boolean}
|
|
144
146
|
*/
|
|
145
|
-
|
|
147
|
+
go_out_grouping(s, min, max) {
|
|
146
148
|
/** @protected */
|
|
147
|
-
while (this.
|
|
148
|
-
|
|
149
|
+
while (this.c < this.limit) {
|
|
150
|
+
let ch = this.current.charCodeAt(this.c);
|
|
149
151
|
if (ch <= max && ch >= min) {
|
|
150
152
|
ch -= min;
|
|
151
|
-
if ((s[ch >>> 3] & (0X1 << (ch & 0x7)))
|
|
153
|
+
if ((s[ch >>> 3] & (0X1 << (ch & 0x7))) !== 0) {
|
|
152
154
|
return true;
|
|
153
155
|
}
|
|
154
156
|
}
|
|
155
|
-
this.
|
|
157
|
+
this.c++;
|
|
156
158
|
}
|
|
157
159
|
return false;
|
|
158
|
-
}
|
|
160
|
+
}
|
|
159
161
|
|
|
160
162
|
/**
|
|
161
|
-
* @param {number
|
|
163
|
+
* @param {Array<number>} s
|
|
162
164
|
* @param {number} min
|
|
163
165
|
* @param {number} max
|
|
164
166
|
* @return {boolean}
|
|
165
167
|
*/
|
|
166
|
-
|
|
168
|
+
out_grouping_b(s, min, max) {
|
|
167
169
|
/** @protected */
|
|
168
|
-
if (this.
|
|
169
|
-
|
|
170
|
+
if (this.c <= this.limit_backward) return false;
|
|
171
|
+
let ch = this.current.charCodeAt(this.c - 1);
|
|
170
172
|
if (ch > max || ch < min) {
|
|
171
|
-
this.
|
|
173
|
+
this.c--;
|
|
172
174
|
return true;
|
|
173
175
|
}
|
|
174
176
|
ch -= min;
|
|
175
|
-
if ((s[ch >>> 3] & (0x1 << (ch & 0x7)))
|
|
176
|
-
this.
|
|
177
|
+
if ((s[ch >>> 3] & (0x1 << (ch & 0x7))) === 0) {
|
|
178
|
+
this.c--;
|
|
177
179
|
return true;
|
|
178
180
|
}
|
|
179
181
|
return false;
|
|
180
|
-
}
|
|
182
|
+
}
|
|
181
183
|
|
|
182
184
|
/**
|
|
183
|
-
* @param {number
|
|
185
|
+
* @param {Array<number>} s
|
|
184
186
|
* @param {number} min
|
|
185
187
|
* @param {number} max
|
|
186
188
|
* @return {boolean}
|
|
187
189
|
*/
|
|
188
|
-
|
|
190
|
+
go_out_grouping_b(s, min, max) {
|
|
189
191
|
/** @protected */
|
|
190
|
-
while (this.
|
|
191
|
-
|
|
192
|
+
while (this.c > this.limit_backward) {
|
|
193
|
+
let ch = this.current.charCodeAt(this.c - 1);
|
|
192
194
|
if (ch <= max && ch >= min) {
|
|
193
195
|
ch -= min;
|
|
194
|
-
if ((s[ch >>> 3] & (0x1 << (ch & 0x7)))
|
|
196
|
+
if ((s[ch >>> 3] & (0x1 << (ch & 0x7))) !== 0) {
|
|
195
197
|
return true;
|
|
196
198
|
}
|
|
197
199
|
}
|
|
198
|
-
this.
|
|
200
|
+
this.c--;
|
|
199
201
|
}
|
|
200
202
|
return false;
|
|
201
|
-
}
|
|
203
|
+
}
|
|
202
204
|
|
|
203
205
|
/**
|
|
204
206
|
* @param {string} s
|
|
205
207
|
* @return {boolean}
|
|
206
208
|
*/
|
|
207
|
-
|
|
208
|
-
{
|
|
209
|
+
eq_s(s) {
|
|
209
210
|
/** @protected */
|
|
210
|
-
if (this.limit - this.
|
|
211
|
-
if (this.current.
|
|
212
|
-
|
|
213
|
-
return false;
|
|
214
|
-
}
|
|
215
|
-
this.cursor += s.length;
|
|
211
|
+
if (this.limit - this.c < s.length) return false;
|
|
212
|
+
if (!this.current.startsWith(s, this.c)) return false;
|
|
213
|
+
this.c += s.length;
|
|
216
214
|
return true;
|
|
217
|
-
}
|
|
215
|
+
}
|
|
218
216
|
|
|
219
217
|
/**
|
|
220
218
|
* @param {string} s
|
|
221
219
|
* @return {boolean}
|
|
222
220
|
*/
|
|
223
|
-
|
|
224
|
-
{
|
|
221
|
+
eq_s_b(s) {
|
|
225
222
|
/** @protected */
|
|
226
|
-
if (this.
|
|
227
|
-
if (this.current.
|
|
228
|
-
|
|
229
|
-
return false;
|
|
230
|
-
}
|
|
231
|
-
this.cursor -= s.length;
|
|
223
|
+
if (this.c - this.limit_backward < s.length) return false;
|
|
224
|
+
if (!this.current.endsWith(s, this.c)) return false;
|
|
225
|
+
this.c -= s.length;
|
|
232
226
|
return true;
|
|
233
|
-
}
|
|
227
|
+
}
|
|
234
228
|
|
|
235
229
|
/**
|
|
236
|
-
* @param {
|
|
230
|
+
* @param {Array<Array<string|number>>} v
|
|
231
|
+
* @param {?function(): boolean} call_among_func
|
|
237
232
|
* @return {number}
|
|
238
233
|
*/
|
|
239
|
-
|
|
240
|
-
{
|
|
234
|
+
find_among(v, call_among_func) {
|
|
241
235
|
/** @protected */
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
while (true)
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
for (i2 = common; i2 < w[0].length; i2++)
|
|
262
|
-
|
|
263
|
-
if (c + common == l)
|
|
264
|
-
{
|
|
236
|
+
let i = 0;
|
|
237
|
+
let j = v.length;
|
|
238
|
+
|
|
239
|
+
const c = this.c;
|
|
240
|
+
const l = this.limit;
|
|
241
|
+
|
|
242
|
+
let common_i = 0;
|
|
243
|
+
let common_j = 0;
|
|
244
|
+
|
|
245
|
+
let first_key_inspected = false;
|
|
246
|
+
|
|
247
|
+
while (true) {
|
|
248
|
+
const k = i + ((j - i) >>> 1);
|
|
249
|
+
let diff = 0;
|
|
250
|
+
let common = common_i < common_j ? common_i : common_j; // smaller
|
|
251
|
+
// w[0]: string, w[1]: result, w[2]: substring_i (optional), w[3]: function (optional)
|
|
252
|
+
const w = v[k];
|
|
253
|
+
let i2;
|
|
254
|
+
// @ts-expect-error: w[0] always string.
|
|
255
|
+
for (i2 = common; i2 < w[0].length; i2++) {
|
|
256
|
+
if (c + common === l) {
|
|
265
257
|
diff = -1;
|
|
266
258
|
break;
|
|
267
259
|
}
|
|
260
|
+
// @ts-expect-error: w[0] always string.
|
|
268
261
|
diff = this.current.charCodeAt(c + common) - w[0].charCodeAt(i2);
|
|
269
|
-
if (diff
|
|
262
|
+
if (diff !== 0) break;
|
|
270
263
|
common++;
|
|
271
264
|
}
|
|
272
|
-
if (diff < 0)
|
|
273
|
-
{
|
|
265
|
+
if (diff < 0) {
|
|
274
266
|
j = k;
|
|
275
267
|
common_j = common;
|
|
276
|
-
}
|
|
277
|
-
else
|
|
278
|
-
{
|
|
268
|
+
} else {
|
|
279
269
|
i = k;
|
|
280
270
|
common_i = common;
|
|
281
271
|
}
|
|
282
|
-
if (j - i <= 1)
|
|
283
|
-
{
|
|
272
|
+
if (j - i <= 1) {
|
|
284
273
|
if (i > 0) break; // v->s has been inspected
|
|
285
|
-
if (j
|
|
274
|
+
if (j === i) break; // only one item in v
|
|
286
275
|
|
|
287
276
|
// - but now we need to go round once more to get
|
|
288
277
|
// v->s inspected. This looks messy, but is actually
|
|
@@ -292,90 +281,102 @@ const BaseStemmer = function() {
|
|
|
292
281
|
first_key_inspected = true;
|
|
293
282
|
}
|
|
294
283
|
}
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
{
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
284
|
+
while (true) {
|
|
285
|
+
const w = v[i];
|
|
286
|
+
// @ts-expect-error: w[0] always string.
|
|
287
|
+
if (common_i >= w[0].length) {
|
|
288
|
+
// @ts-expect-error: w[0] always string.
|
|
289
|
+
this.c = c + w[0].length;
|
|
290
|
+
// @ts-expect-error: w[1] always number.
|
|
291
|
+
if (w.length < 4) return w[1];
|
|
292
|
+
// @ts-expect-error: w[3] always number.
|
|
293
|
+
this.af = w[3];
|
|
294
|
+
// @ts-expect-error: call_among_func never null here.
|
|
295
|
+
if (call_among_func.call(this)) {
|
|
296
|
+
// @ts-expect-error: w[0] always string.
|
|
297
|
+
this.c = c + w[0].length;
|
|
298
|
+
// @ts-expect-error: w[3] always number.
|
|
299
|
+
return w[1];
|
|
300
|
+
}
|
|
304
301
|
}
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
302
|
+
// Tests for undefined (if w.length < 2) or 0.
|
|
303
|
+
if (!w[2]) return 0;
|
|
304
|
+
// @ts-expect-error: w[2] always number.
|
|
305
|
+
i -= w[2];
|
|
306
|
+
}
|
|
307
|
+
}
|
|
309
308
|
|
|
310
309
|
// find_among_b is for backwards processing. Same comments apply
|
|
311
310
|
/**
|
|
312
|
-
* @param {
|
|
313
|
-
* @
|
|
311
|
+
* @param {Array<Array<string|number>>} v
|
|
312
|
+
* @param {?function(): boolean} call_among_func
|
|
314
313
|
*/
|
|
315
|
-
|
|
316
|
-
{
|
|
314
|
+
find_among_b(v, call_among_func) {
|
|
317
315
|
/** @protected */
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
while (true)
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
for (i2 = w[0].length - 1 - common; i2 >= 0; i2--)
|
|
337
|
-
|
|
338
|
-
if (c - common == lb)
|
|
339
|
-
{
|
|
316
|
+
let i = 0;
|
|
317
|
+
let j = v.length
|
|
318
|
+
|
|
319
|
+
const c = this.c;
|
|
320
|
+
const lb = this.limit_backward;
|
|
321
|
+
|
|
322
|
+
let common_i = 0;
|
|
323
|
+
let common_j = 0;
|
|
324
|
+
|
|
325
|
+
let first_key_inspected = false;
|
|
326
|
+
|
|
327
|
+
while (true) {
|
|
328
|
+
const k = i + ((j - i) >> 1);
|
|
329
|
+
let diff = 0;
|
|
330
|
+
let common = common_i < common_j ? common_i : common_j;
|
|
331
|
+
const w = v[k];
|
|
332
|
+
let i2;
|
|
333
|
+
// @ts-expect-error: w[0] always string.
|
|
334
|
+
for (i2 = w[0].length - 1 - common; i2 >= 0; i2--) {
|
|
335
|
+
if (c - common === lb) {
|
|
340
336
|
diff = -1;
|
|
341
337
|
break;
|
|
342
338
|
}
|
|
339
|
+
// @ts-expect-error: w[0] always string.
|
|
343
340
|
diff = this.current.charCodeAt(c - 1 - common) - w[0].charCodeAt(i2);
|
|
344
|
-
if (diff
|
|
341
|
+
if (diff !== 0) break;
|
|
345
342
|
common++;
|
|
346
343
|
}
|
|
347
|
-
if (diff < 0)
|
|
348
|
-
{
|
|
344
|
+
if (diff < 0) {
|
|
349
345
|
j = k;
|
|
350
346
|
common_j = common;
|
|
351
|
-
}
|
|
352
|
-
else
|
|
353
|
-
{
|
|
347
|
+
} else {
|
|
354
348
|
i = k;
|
|
355
349
|
common_i = common;
|
|
356
350
|
}
|
|
357
|
-
if (j - i <= 1)
|
|
358
|
-
{
|
|
351
|
+
if (j - i <= 1) {
|
|
359
352
|
if (i > 0) break;
|
|
360
|
-
if (j
|
|
353
|
+
if (j === i) break;
|
|
361
354
|
if (first_key_inspected) break;
|
|
362
355
|
first_key_inspected = true;
|
|
363
356
|
}
|
|
364
357
|
}
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
{
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
358
|
+
while (true) {
|
|
359
|
+
const w = v[i];
|
|
360
|
+
// @ts-expect-error: w[0] always string.
|
|
361
|
+
if (common_i >= w[0].length) {
|
|
362
|
+
// @ts-expect-error: w[0] always string.
|
|
363
|
+
this.c = c - w[0].length;
|
|
364
|
+
if (w.length < 4) return w[1];
|
|
365
|
+
// @ts-expect-error: w[3] always number.
|
|
366
|
+
this.af = w[3];
|
|
367
|
+
// @ts-expect-error: call_among_func never null here.
|
|
368
|
+
if (call_among_func.call(this)) {
|
|
369
|
+
// @ts-expect-error: w[0] always string.
|
|
370
|
+
this.c = c - w[0].length;
|
|
371
|
+
return w[1];
|
|
372
|
+
}
|
|
374
373
|
}
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
374
|
+
// Tests for undefined (if w.length < 2) or 0.
|
|
375
|
+
if (!w[2]) return 0;
|
|
376
|
+
// @ts-expect-error: w[2] always number.
|
|
377
|
+
i -= w[2];
|
|
378
|
+
}
|
|
379
|
+
}
|
|
379
380
|
|
|
380
381
|
/* to replace chars between c_bra and c_ket in this.current by the
|
|
381
382
|
* chars in s.
|
|
@@ -386,93 +387,59 @@ const BaseStemmer = function() {
|
|
|
386
387
|
* @param {string} s
|
|
387
388
|
* @return {number}
|
|
388
389
|
*/
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
/** @protected */
|
|
392
|
-
var adjustment = s.length - (c_ket - c_bra);
|
|
390
|
+
#replace_s(c_bra, c_ket, s) {
|
|
391
|
+
const adjustment = s.length - (c_ket - c_bra);
|
|
393
392
|
this.current = this.current.slice(0, c_bra) + s + this.current.slice(c_ket);
|
|
394
393
|
this.limit += adjustment;
|
|
395
|
-
if (this.
|
|
396
|
-
else if (this.
|
|
394
|
+
if (this.c >= c_ket) this.c += adjustment;
|
|
395
|
+
else if (this.c > c_bra) this.c = c_bra;
|
|
397
396
|
return adjustment;
|
|
398
|
-
}
|
|
397
|
+
}
|
|
399
398
|
|
|
400
399
|
/**
|
|
401
|
-
* @return {boolean}
|
|
402
400
|
*/
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
this.limit > this.current.length)
|
|
410
|
-
{
|
|
411
|
-
return false;
|
|
412
|
-
}
|
|
413
|
-
return true;
|
|
414
|
-
};
|
|
401
|
+
#slice_check() {
|
|
402
|
+
console.assert(this.bra >= 0);
|
|
403
|
+
console.assert(this.bra <= this.ket);
|
|
404
|
+
console.assert(this.ket <= this.limit);
|
|
405
|
+
console.assert(this.limit <= this.current.length);
|
|
406
|
+
}
|
|
415
407
|
|
|
416
408
|
/**
|
|
417
|
-
* @param {
|
|
418
|
-
* @return {boolean}
|
|
409
|
+
* @param {string} s
|
|
419
410
|
*/
|
|
420
|
-
|
|
421
|
-
{
|
|
411
|
+
slice_from(s) {
|
|
422
412
|
/** @protected */
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
result = true;
|
|
428
|
-
}
|
|
429
|
-
return result;
|
|
430
|
-
};
|
|
413
|
+
this.#slice_check();
|
|
414
|
+
this.#replace_s(this.bra, this.ket, s);
|
|
415
|
+
this.ket = this.bra + s.length;
|
|
416
|
+
}
|
|
431
417
|
|
|
432
418
|
/**
|
|
433
|
-
* @return {boolean}
|
|
434
419
|
*/
|
|
435
|
-
|
|
436
|
-
{
|
|
420
|
+
slice_del() {
|
|
437
421
|
/** @protected */
|
|
438
|
-
|
|
439
|
-
}
|
|
422
|
+
this.slice_from("");
|
|
423
|
+
}
|
|
440
424
|
|
|
441
425
|
/**
|
|
442
426
|
* @param {number} c_bra
|
|
443
427
|
* @param {number} c_ket
|
|
444
428
|
* @param {string} s
|
|
445
429
|
*/
|
|
446
|
-
|
|
447
|
-
{
|
|
430
|
+
insert(c_bra, c_ket, s) {
|
|
448
431
|
/** @protected */
|
|
449
|
-
|
|
432
|
+
const adjustment = this.#replace_s(c_bra, c_ket, s);
|
|
450
433
|
if (c_bra <= this.bra) this.bra += adjustment;
|
|
451
434
|
if (c_bra <= this.ket) this.ket += adjustment;
|
|
452
|
-
}
|
|
435
|
+
}
|
|
453
436
|
|
|
454
437
|
/**
|
|
455
438
|
* @return {string}
|
|
456
439
|
*/
|
|
457
|
-
|
|
458
|
-
{
|
|
440
|
+
slice_to() {
|
|
459
441
|
/** @protected */
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
}
|
|
465
|
-
return result;
|
|
466
|
-
};
|
|
467
|
-
|
|
468
|
-
/**
|
|
469
|
-
* @return {string}
|
|
470
|
-
*/
|
|
471
|
-
this.assign_to = function()
|
|
472
|
-
{
|
|
473
|
-
/** @protected */
|
|
474
|
-
return this.current.slice(0, this.limit);
|
|
475
|
-
};
|
|
476
|
-
};
|
|
477
|
-
|
|
478
|
-
if (typeof module === 'object' && module.exports) module.exports = BaseStemmer;
|
|
442
|
+
this.#slice_check();
|
|
443
|
+
return this.current.slice(this.bra, this.ket);
|
|
444
|
+
}
|
|
445
|
+
}
|