mittens 0.3.0 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +8 -0
- data/Gemfile +2 -1
- data/README.md +2 -2
- data/Rakefile +12 -5
- data/ext/mittens/extconf.rb +3 -1
- data/lib/mittens/version.rb +1 -1
- data/vendor/snowball/.github/workflows/ci.yml +144 -17
- data/vendor/snowball/.github/workflows/coverage.yml +117 -0
- data/vendor/snowball/.github/workflows/runtime-tests.yml +26 -0
- data/vendor/snowball/.gitignore +33 -7
- data/vendor/snowball/CONTRIBUTING.rst +7 -0
- data/vendor/snowball/COPYING +1 -1
- data/vendor/snowball/GNUmakefile +736 -298
- data/vendor/snowball/NEWS +1394 -7
- data/vendor/snowball/README.rst +8 -8
- data/vendor/snowball/ada/generate/generate.adb +5 -5
- data/vendor/snowball/ada/src/stemmer.adb +177 -188
- data/vendor/snowball/ada/src/stemmer.ads +86 -89
- data/vendor/snowball/ada/src/stemwords.adb +21 -5
- data/vendor/snowball/algorithms/czech.sbl +255 -0
- data/vendor/snowball/algorithms/danish.sbl +22 -10
- data/vendor/snowball/algorithms/english.sbl +7 -4
- data/vendor/snowball/algorithms/esperanto.sbl +6 -7
- data/vendor/snowball/algorithms/estonian.sbl +9 -4
- data/vendor/snowball/algorithms/finnish.sbl +33 -21
- data/vendor/snowball/algorithms/german.sbl +5 -0
- data/vendor/snowball/algorithms/indonesian.sbl +115 -96
- data/vendor/snowball/algorithms/italian.sbl +26 -1
- data/vendor/snowball/algorithms/lithuanian.sbl +13 -13
- data/vendor/snowball/algorithms/norwegian.sbl +17 -6
- data/vendor/snowball/algorithms/persian.sbl +387 -0
- data/vendor/snowball/algorithms/polish.sbl +177 -0
- data/vendor/snowball/algorithms/sesotho.sbl +115 -0
- data/vendor/snowball/algorithms/turkish.sbl +4 -4
- data/vendor/snowball/compiler/analyser.c +2002 -711
- data/vendor/snowball/compiler/driver.c +460 -298
- data/vendor/snowball/compiler/generator.c +439 -1965
- data/vendor/snowball/compiler/generator_ada.c +605 -430
- data/vendor/snowball/compiler/generator_c.c +2227 -0
- data/vendor/snowball/compiler/generator_csharp.c +365 -274
- data/vendor/snowball/compiler/generator_dart.c +1476 -0
- data/vendor/snowball/compiler/generator_go.c +348 -270
- data/vendor/snowball/compiler/generator_java.c +330 -233
- data/vendor/snowball/compiler/generator_js.c +534 -385
- data/vendor/snowball/compiler/generator_pascal.c +361 -320
- data/vendor/snowball/compiler/generator_php.c +1445 -0
- data/vendor/snowball/compiler/generator_python.c +399 -299
- data/vendor/snowball/compiler/generator_rust.c +370 -261
- data/vendor/snowball/compiler/generator_zig.c +1474 -0
- data/vendor/snowball/compiler/header.h +153 -86
- data/vendor/snowball/compiler/space.c +121 -63
- data/vendor/snowball/compiler/tokeniser.c +286 -182
- data/vendor/snowball/compiler/tokens.h +71 -0
- data/vendor/snowball/csharp/Snowball/Among.cs +14 -26
- data/vendor/snowball/csharp/Snowball/AssemblyInfo.cs +3 -3
- data/vendor/snowball/csharp/Snowball/Stemmer.cs +41 -45
- data/vendor/snowball/csharp/Stemwords/Program.cs +20 -14
- data/vendor/snowball/cxx/.gitignore +4 -0
- data/vendor/snowball/cxx/generate_algorithms.pl +87 -0
- data/vendor/snowball/cxx/stemmer.h +12 -0
- data/vendor/snowball/cxx/stemwords.cxx +176 -0
- data/vendor/snowball/cxx/utilities.cxx +2 -0
- data/vendor/snowball/dart/.gitignore +4 -0
- data/vendor/snowball/dart/analysis_options.yaml +1 -0
- data/vendor/snowball/dart/example/test_app.dart +65 -0
- data/vendor/snowball/dart/generate_algorithms.pl +21 -0
- data/vendor/snowball/dart/lib/snowball.dart +18 -0
- data/vendor/snowball/dart/lib/src/snowball.dart +339 -0
- data/vendor/snowball/dart/pubspec.yaml +17 -0
- data/vendor/snowball/doc/libstemmer_dart_README +37 -0
- data/vendor/snowball/doc/libstemmer_js_README +2 -2
- data/vendor/snowball/doc/libstemmer_php_README +38 -0
- data/vendor/snowball/doc/libstemmer_python_README +3 -3
- data/vendor/snowball/examples/stemwords.c +18 -3
- data/vendor/snowball/go/README.md +12 -4
- data/vendor/snowball/go/env.go +12 -14
- data/vendor/snowball/go/stemwords/main.go +2 -1
- data/vendor/snowball/java/org/tartarus/snowball/CharArraySequence.java +36 -0
- data/vendor/snowball/java/org/tartarus/snowball/SnowballProgram.java +17 -34
- data/vendor/snowball/java/org/tartarus/snowball/SnowballStemmer.java +0 -2
- data/vendor/snowball/javascript/base-stemmer.js +219 -252
- data/vendor/snowball/javascript/stemwords.js +86 -58
- data/vendor/snowball/libstemmer/mkmodules.pl +1 -1
- data/vendor/snowball/libstemmer/modules.txt +4 -0
- data/vendor/snowball/pascal/SnowballProgram.pas +18 -31
- data/vendor/snowball/php/base-stemmer.php +453 -0
- data/vendor/snowball/php/stemwords.php +25 -0
- data/vendor/snowball/python/create_init.py +16 -21
- data/vendor/snowball/python/pyproject.toml +3 -0
- data/vendor/snowball/python/setup.py +5 -7
- data/vendor/snowball/python/snowballstemmer/among.py +1 -2
- data/vendor/snowball/python/snowballstemmer/basestemmer.py +24 -33
- data/vendor/snowball/python/stemwords.py +72 -63
- data/vendor/snowball/runtime/api.c +11 -42
- data/vendor/snowball/runtime/api.h +11 -9
- data/vendor/snowball/runtime/snowball_runtime.h +110 -0
- data/vendor/snowball/runtime/utilities.c +282 -106
- data/vendor/snowball/rust/src/main.rs +2 -2
- data/vendor/snowball/rust/src/snowball/snowball_env.rs +7 -9
- data/vendor/snowball/tests/compilertest +62 -0
- data/vendor/snowball/tests/errors/ae-errors.sbl +53 -0
- data/vendor/snowball/tests/errors/ae-errors.stderr +8 -0
- data/vendor/snowball/tests/errors/bad-dollar.sbl +14 -0
- data/vendor/snowball/tests/errors/bad-dollar.stderr +4 -0
- data/vendor/snowball/tests/errors/bad-grouping-definition.sbl +9 -0
- data/vendor/snowball/tests/errors/bad-grouping-definition.stderr +6 -0
- data/vendor/snowball/tests/errors/missing-bra.sbl +6 -0
- data/vendor/snowball/tests/errors/missing-bra.stderr +3 -0
- data/vendor/snowball/tests/errors/missing-command.sbl +9 -0
- data/vendor/snowball/tests/errors/missing-command.stderr +1 -0
- data/vendor/snowball/tests/errors/missing-ket-backwardmode.sbl +3 -0
- data/vendor/snowball/tests/errors/missing-ket-backwardmode.stderr +1 -0
- data/vendor/snowball/tests/errors/missing-ket.sbl +7 -0
- data/vendor/snowball/tests/errors/missing-ket.stderr +6 -0
- data/vendor/snowball/tests/errors/notdefined.sbl +4 -0
- data/vendor/snowball/tests/errors/notdefined.stderr +2 -0
- data/vendor/snowball/tests/errors/string-omitted.sbl +5 -0
- data/vendor/snowball/tests/errors/string-omitted.stderr +1 -0
- data/vendor/snowball/tests/errors/undeclared.sbl +20 -0
- data/vendor/snowball/tests/errors/undeclared.stderr +9 -0
- data/vendor/snowball/tests/errors/unexpected-token.sbl +24 -0
- data/vendor/snowball/tests/errors/unexpected-token.stderr +17 -0
- data/vendor/snowball/tests/errors/wrongdirection.sbl +14 -0
- data/vendor/snowball/tests/errors/wrongdirection.stderr +1 -0
- data/vendor/snowball/tests/runtime/among.sbl +19 -0
- data/vendor/snowball/tests/runtime/arithmeticexpr.sbl +86 -0
- data/vendor/snowball/tests/runtime/attachinsert.sbl +47 -0
- data/vendor/snowball/tests/runtime/booleans.sbl +19 -0
- data/vendor/snowball/tests/runtime/externals.sbl +22 -0
- data/vendor/snowball/tests/runtime/hop.sbl +12 -0
- data/vendor/snowball/tests/runtime/integertests.sbl +60 -0
- data/vendor/snowball/tests/runtime/intlimits.sbl +25 -0
- data/vendor/snowball/tests/runtime/naming.sbl +20 -0
- data/vendor/snowball/tests/runtime/not.sbl +30 -0
- data/vendor/snowball/tests/runtime/or.sbl +17 -0
- data/vendor/snowball/tests/runtime/repeat.sbl +16 -0
- data/vendor/snowball/tests/runtime/setlimit.sbl +24 -0
- data/vendor/snowball/tests/runtime/sizelen.sbl +56 -0
- data/vendor/snowball/tests/runtime/slice.sbl +27 -0
- data/vendor/snowball/tests/runtime/stringdollar.sbl +39 -0
- data/vendor/snowball/tests/runtime/strings.sbl +66 -0
- data/vendor/snowball/tests/runtime/test.sbl +19 -0
- data/vendor/snowball/tests/stemtest.c +41 -1
- data/vendor/snowball/tests/syntax/canon.sbl +32 -0
- data/vendor/snowball/tests/syntax/canon.stderr +0 -0
- data/vendor/snowball/tests/syntax/canon.syntax +57 -0
- data/vendor/snowball/tests/syntax/emptyprogram.sbl +2 -0
- data/vendor/snowball/tests/syntax/emptyprogram.syntax +0 -0
- data/vendor/snowball/tests/syntax/groupings.sbl +11 -0
- data/vendor/snowball/tests/syntax/inline.sbl +144 -0
- data/vendor/snowball/tests/syntax/inline.stderr +0 -0
- data/vendor/snowball/tests/syntax/inline.syntax +121 -0
- data/vendor/snowball/tests/syntax/legacy.sbl +17 -0
- data/vendor/snowball/tests/syntax/legacy.stderr +5 -0
- data/vendor/snowball/tests/syntax/legacy.syntax +19 -0
- data/vendor/snowball/tests/syntax/loops.sbl +14 -0
- data/vendor/snowball/tests/syntax/loops.stderr +9 -0
- data/vendor/snowball/tests/syntax/loops.syntax +29 -0
- data/vendor/snowball/tests/syntax/noops.sbl +76 -0
- data/vendor/snowball/tests/syntax/noops.stderr +27 -0
- data/vendor/snowball/tests/syntax/noops.syntax +135 -0
- data/vendor/snowball/tests/syntax/simplifyae.sbl +21 -0
- data/vendor/snowball/tests/syntax/simplifyae.stderr +0 -0
- data/vendor/snowball/tests/syntax/simplifyae.syntax +39 -0
- data/vendor/snowball/tests/syntax/unused.sbl +18 -0
- data/vendor/snowball/tests/syntax/unused.stderr +7 -0
- data/vendor/snowball/tests/syntax/unused.syntax +3 -0
- data/vendor/snowball/zig/env.zig +599 -0
- data/vendor/snowball/zig/generate_algorithms.pl +19 -0
- data/vendor/snowball/zig/stemwords.zig +123 -0
- metadata +102 -4
- data/vendor/snowball/compiler/syswords.h +0 -86
- data/vendor/snowball/runtime/header.h +0 -62
|
@@ -33,68 +33,64 @@ package body Stemmer with SPARK_Mode is
|
|
|
33
33
|
subtype Byte is Interfaces.Unsigned_8;
|
|
34
34
|
use type Interfaces.Unsigned_8;
|
|
35
35
|
|
|
36
|
-
procedure Stem_Word (
|
|
37
|
-
Word
|
|
38
|
-
Result
|
|
36
|
+
procedure Stem_Word (Z : in out Context_Type'Class;
|
|
37
|
+
Word : in String;
|
|
38
|
+
Result : out Boolean) is
|
|
39
39
|
begin
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
Stemmer.Stem (
|
|
40
|
+
Z.P (1 .. Word'Length) := Word;
|
|
41
|
+
Z.Len := Word'Length;
|
|
42
|
+
Z.C := 0;
|
|
43
|
+
Z.L := Word'Length;
|
|
44
|
+
Z.Lb := 0;
|
|
45
|
+
Stemmer.Stem (Z, Result);
|
|
46
46
|
end Stem_Word;
|
|
47
47
|
|
|
48
|
-
function Get_Result (
|
|
48
|
+
function Get_Result (Z : in Context_Type'Class) return String is
|
|
49
49
|
begin
|
|
50
|
-
return
|
|
50
|
+
return Z.P (1 .. Z.Len);
|
|
51
51
|
end Get_Result;
|
|
52
52
|
|
|
53
|
-
|
|
54
|
-
|
|
53
|
+
procedure Eq_S (Z : in out Context_Type'Class;
|
|
54
|
+
S : in String;
|
|
55
|
+
Len : in Char_Index;
|
|
56
|
+
Result : out Boolean) is
|
|
55
57
|
begin
|
|
56
|
-
if
|
|
57
|
-
|
|
58
|
+
if Z.L - Z.C < Len then
|
|
59
|
+
Result := False;
|
|
60
|
+
return;
|
|
58
61
|
end if;
|
|
59
|
-
if
|
|
60
|
-
|
|
62
|
+
if Z.P (Z.C + 1 .. Z.C + Len) /= S (S'First .. Len) then
|
|
63
|
+
Result := False;
|
|
64
|
+
return;
|
|
61
65
|
end if;
|
|
62
|
-
|
|
66
|
+
Z.C := Z.C + Len;
|
|
67
|
+
Result := True;
|
|
63
68
|
end Eq_S;
|
|
64
69
|
|
|
65
|
-
|
|
66
|
-
|
|
70
|
+
procedure Eq_S_Backward (Z : in out Context_Type'Class;
|
|
71
|
+
S : in String;
|
|
72
|
+
Len : in Char_Index;
|
|
73
|
+
Result : out Boolean) is
|
|
67
74
|
begin
|
|
68
|
-
if
|
|
69
|
-
|
|
75
|
+
if Z.C - Z.Lb < Len then
|
|
76
|
+
Result := False;
|
|
77
|
+
return;
|
|
70
78
|
end if;
|
|
71
|
-
if
|
|
72
|
-
|
|
79
|
+
if Z.P (Z.C + 1 - Len .. Z.C) /= S (S'First .. Len) then
|
|
80
|
+
Result := False;
|
|
81
|
+
return;
|
|
73
82
|
end if;
|
|
74
|
-
|
|
83
|
+
Z.C := Z.C - Len;
|
|
84
|
+
Result := True;
|
|
75
85
|
end Eq_S_Backward;
|
|
76
86
|
|
|
77
|
-
function Length_Utf8 (
|
|
87
|
+
function Length_Utf8 (S : in String;
|
|
88
|
+
Len : in Char_Index) return Natural is
|
|
78
89
|
Count : Natural := 0;
|
|
79
90
|
Pos : Positive := 1;
|
|
80
91
|
Val : Byte;
|
|
81
92
|
begin
|
|
82
|
-
while Pos <=
|
|
83
|
-
Val := Character'Pos (Context.P (Pos));
|
|
84
|
-
Pos := Pos + 1;
|
|
85
|
-
if Val >= 16#C0# or Val < 16#80# then
|
|
86
|
-
Count := Count + 1;
|
|
87
|
-
end if;
|
|
88
|
-
end loop;
|
|
89
|
-
return Count;
|
|
90
|
-
end Length_Utf8;
|
|
91
|
-
|
|
92
|
-
function Length_Utf8 (S : in String) return Natural is
|
|
93
|
-
Count : Natural := 0;
|
|
94
|
-
Pos : Positive := 1;
|
|
95
|
-
Val : Byte;
|
|
96
|
-
begin
|
|
97
|
-
while Pos <= S'Length loop
|
|
93
|
+
while Pos <= Len loop
|
|
98
94
|
Val := Character'Pos (S (Pos));
|
|
99
95
|
Pos := Pos + 1;
|
|
100
96
|
if Val >= 16#C0# or Val < 16#80# then
|
|
@@ -104,12 +100,12 @@ package body Stemmer with SPARK_Mode is
|
|
|
104
100
|
return Count;
|
|
105
101
|
end Length_Utf8;
|
|
106
102
|
|
|
107
|
-
function Check_Among (
|
|
108
|
-
Pos
|
|
109
|
-
Shift
|
|
110
|
-
Mask
|
|
103
|
+
function Check_Among (Z : in Context_Type'Class;
|
|
104
|
+
Pos : in Char_Index;
|
|
105
|
+
Shift : in Natural;
|
|
106
|
+
Mask : in Mask_Type) return Boolean is
|
|
111
107
|
use Interfaces;
|
|
112
|
-
Val : constant Byte := Character'Pos (
|
|
108
|
+
Val : constant Byte := Character'Pos (Z.P (Pos + 1));
|
|
113
109
|
begin
|
|
114
110
|
if Natural (Shift_Right (Val, 5)) /= Shift then
|
|
115
111
|
return True;
|
|
@@ -117,7 +113,7 @@ package body Stemmer with SPARK_Mode is
|
|
|
117
113
|
return (Shift_Right (Unsigned_64 (Mask), Natural (Val and 16#1f#)) and 1) = 0;
|
|
118
114
|
end Check_Among;
|
|
119
115
|
|
|
120
|
-
procedure Find_Among (
|
|
116
|
+
procedure Find_Among (Z : in out Context_Type'Class;
|
|
121
117
|
Amongs : in Among_Array_Type;
|
|
122
118
|
Pattern : in String;
|
|
123
119
|
Execute : access procedure
|
|
@@ -130,8 +126,8 @@ package body Stemmer with SPARK_Mode is
|
|
|
130
126
|
Common_I : Natural := 0;
|
|
131
127
|
Common_J : Natural := 0;
|
|
132
128
|
First_Key_Inspected : Boolean := False;
|
|
133
|
-
C : constant Natural :=
|
|
134
|
-
L : constant Integer :=
|
|
129
|
+
C : constant Natural := Z.C;
|
|
130
|
+
L : constant Integer := Z.L;
|
|
135
131
|
begin
|
|
136
132
|
loop
|
|
137
133
|
declare
|
|
@@ -145,7 +141,7 @@ package body Stemmer with SPARK_Mode is
|
|
|
145
141
|
Diff := -1;
|
|
146
142
|
exit;
|
|
147
143
|
end if;
|
|
148
|
-
Diff := Character'Pos (
|
|
144
|
+
Diff := Character'Pos (Z.P (C + Common + 1))
|
|
149
145
|
- Character'Pos (Pattern (I2));
|
|
150
146
|
exit when Diff /= 0;
|
|
151
147
|
Common := Common + 1;
|
|
@@ -171,14 +167,14 @@ package body Stemmer with SPARK_Mode is
|
|
|
171
167
|
Status : Boolean;
|
|
172
168
|
begin
|
|
173
169
|
if Common_I >= Len then
|
|
174
|
-
|
|
170
|
+
Z.C := C + Len;
|
|
175
171
|
if W.Operation = 0 then
|
|
176
172
|
Result := W.Result;
|
|
177
173
|
return;
|
|
178
174
|
end if;
|
|
179
|
-
Execute (
|
|
180
|
-
Context.C := C + Len;
|
|
175
|
+
Execute (Z, W.Operation, Status);
|
|
181
176
|
if Status then
|
|
177
|
+
Z.C := C + Len;
|
|
182
178
|
Result := W.Result;
|
|
183
179
|
return;
|
|
184
180
|
end if;
|
|
@@ -190,7 +186,7 @@ package body Stemmer with SPARK_Mode is
|
|
|
190
186
|
Result := 0;
|
|
191
187
|
end Find_Among;
|
|
192
188
|
|
|
193
|
-
procedure Find_Among_Backward (
|
|
189
|
+
procedure Find_Among_Backward (Z : in out Context_Type'Class;
|
|
194
190
|
Amongs : in Among_Array_Type;
|
|
195
191
|
Pattern : in String;
|
|
196
192
|
Execute : access procedure
|
|
@@ -203,8 +199,8 @@ package body Stemmer with SPARK_Mode is
|
|
|
203
199
|
Common_I : Natural := 0;
|
|
204
200
|
Common_J : Natural := 0;
|
|
205
201
|
First_Key_Inspected : Boolean := False;
|
|
206
|
-
C : constant Integer :=
|
|
207
|
-
Lb : constant Integer :=
|
|
202
|
+
C : constant Integer := Z.C;
|
|
203
|
+
Lb : constant Integer := Z.Lb;
|
|
208
204
|
begin
|
|
209
205
|
loop
|
|
210
206
|
declare
|
|
@@ -218,7 +214,7 @@ package body Stemmer with SPARK_Mode is
|
|
|
218
214
|
Diff := -1;
|
|
219
215
|
exit;
|
|
220
216
|
end if;
|
|
221
|
-
Diff := Character'Pos (
|
|
217
|
+
Diff := Character'Pos (Z.P (C - Common))
|
|
222
218
|
- Character'Pos (Pattern (I2));
|
|
223
219
|
exit when Diff /= 0;
|
|
224
220
|
Common := Common + 1;
|
|
@@ -244,14 +240,14 @@ package body Stemmer with SPARK_Mode is
|
|
|
244
240
|
Status : Boolean;
|
|
245
241
|
begin
|
|
246
242
|
if Common_I >= Len then
|
|
247
|
-
|
|
243
|
+
Z.C := C - Len;
|
|
248
244
|
if W.Operation = 0 then
|
|
249
245
|
Result := W.Result;
|
|
250
246
|
return;
|
|
251
247
|
end if;
|
|
252
|
-
Execute (
|
|
253
|
-
Context.C := C - Len;
|
|
248
|
+
Execute (Z, W.Operation, Status);
|
|
254
249
|
if Status then
|
|
250
|
+
Z.C := C - Len;
|
|
255
251
|
Result := W.Result;
|
|
256
252
|
return;
|
|
257
253
|
end if;
|
|
@@ -263,18 +259,18 @@ package body Stemmer with SPARK_Mode is
|
|
|
263
259
|
Result := 0;
|
|
264
260
|
end Find_Among_Backward;
|
|
265
261
|
|
|
266
|
-
function Skip_Utf8 (
|
|
267
|
-
Pos : Char_Index :=
|
|
262
|
+
function Skip_Utf8 (Z : in Context_Type'Class) return Result_Index is
|
|
263
|
+
Pos : Char_Index := Z.C;
|
|
268
264
|
Val : Byte;
|
|
269
265
|
begin
|
|
270
|
-
if Pos >=
|
|
266
|
+
if Pos >= Z.L then
|
|
271
267
|
return -1;
|
|
272
268
|
end if;
|
|
273
269
|
Pos := Pos + 1;
|
|
274
|
-
Val := Character'Pos (
|
|
270
|
+
Val := Character'Pos (Z.P (Pos));
|
|
275
271
|
if Val >= 16#C0# then
|
|
276
|
-
while Pos <
|
|
277
|
-
Val := Character'Pos (
|
|
272
|
+
while Pos < Z.L loop
|
|
273
|
+
Val := Character'Pos (Z.P (Pos + 1));
|
|
278
274
|
exit when Val >= 16#C0# or Val < 16#80#;
|
|
279
275
|
Pos := Pos + 1;
|
|
280
276
|
end loop;
|
|
@@ -282,23 +278,23 @@ package body Stemmer with SPARK_Mode is
|
|
|
282
278
|
return Pos;
|
|
283
279
|
end Skip_Utf8;
|
|
284
280
|
|
|
285
|
-
function Skip_Utf8 (
|
|
281
|
+
function Skip_Utf8 (Z : in Context_Type'Class;
|
|
286
282
|
N : in Integer) return Result_Index is
|
|
287
|
-
Pos : Char_Index :=
|
|
283
|
+
Pos : Char_Index := Z.C;
|
|
288
284
|
Val : Byte;
|
|
289
285
|
begin
|
|
290
286
|
if N < 0 then
|
|
291
287
|
return -1;
|
|
292
288
|
end if;
|
|
293
289
|
for I in 1 .. N loop
|
|
294
|
-
if Pos >=
|
|
290
|
+
if Pos >= Z.L then
|
|
295
291
|
return -1;
|
|
296
292
|
end if;
|
|
297
293
|
Pos := Pos + 1;
|
|
298
|
-
Val := Character'Pos (
|
|
294
|
+
Val := Character'Pos (Z.P (Pos));
|
|
299
295
|
if Val >= 16#C0# then
|
|
300
|
-
while Pos <
|
|
301
|
-
Val := Character'Pos (
|
|
296
|
+
while Pos < Z.L loop
|
|
297
|
+
Val := Character'Pos (Z.P (Pos + 1));
|
|
302
298
|
exit when Val >= 16#C0# or Val < 16#80#;
|
|
303
299
|
Pos := Pos + 1;
|
|
304
300
|
end loop;
|
|
@@ -307,18 +303,18 @@ package body Stemmer with SPARK_Mode is
|
|
|
307
303
|
return Pos;
|
|
308
304
|
end Skip_Utf8;
|
|
309
305
|
|
|
310
|
-
function Skip_Utf8_Backward (
|
|
311
|
-
Pos : Char_Index :=
|
|
306
|
+
function Skip_Utf8_Backward (Z : in Context_Type'Class) return Result_Index is
|
|
307
|
+
Pos : Char_Index := Z.C;
|
|
312
308
|
Val : Byte;
|
|
313
309
|
begin
|
|
314
|
-
if Pos <=
|
|
310
|
+
if Pos <= Z.Lb then
|
|
315
311
|
return -1;
|
|
316
312
|
end if;
|
|
317
|
-
Val := Character'Pos (
|
|
313
|
+
Val := Character'Pos (Z.P (Pos));
|
|
318
314
|
Pos := Pos - 1;
|
|
319
315
|
if Val >= 16#80# then
|
|
320
|
-
while Pos >
|
|
321
|
-
Val := Character'Pos (
|
|
316
|
+
while Pos > Z.Lb loop
|
|
317
|
+
Val := Character'Pos (Z.P (Pos + 1));
|
|
322
318
|
exit when Val >= 16#C0#;
|
|
323
319
|
Pos := Pos - 1;
|
|
324
320
|
end loop;
|
|
@@ -326,23 +322,23 @@ package body Stemmer with SPARK_Mode is
|
|
|
326
322
|
return Pos;
|
|
327
323
|
end Skip_Utf8_Backward;
|
|
328
324
|
|
|
329
|
-
function Skip_Utf8_Backward (
|
|
330
|
-
N
|
|
331
|
-
Pos : Char_Index :=
|
|
325
|
+
function Skip_Utf8_Backward (Z : in Context_Type'Class;
|
|
326
|
+
N : in Integer) return Result_Index is
|
|
327
|
+
Pos : Char_Index := Z.C;
|
|
332
328
|
Val : Byte;
|
|
333
329
|
begin
|
|
334
330
|
if N < 0 then
|
|
335
331
|
return -1;
|
|
336
332
|
end if;
|
|
337
333
|
for I in 1 .. N loop
|
|
338
|
-
if Pos <=
|
|
334
|
+
if Pos <= Z.Lb then
|
|
339
335
|
return -1;
|
|
340
336
|
end if;
|
|
341
|
-
Val := Character'Pos (
|
|
337
|
+
Val := Character'Pos (Z.P (Pos));
|
|
342
338
|
Pos := Pos - 1;
|
|
343
339
|
if Val >= 16#80# then
|
|
344
|
-
while Pos >
|
|
345
|
-
Val := Character'Pos (
|
|
340
|
+
while Pos > Z.Lb loop
|
|
341
|
+
Val := Character'Pos (Z.P (Pos + 1));
|
|
346
342
|
exit when Val >= 16#C0#;
|
|
347
343
|
Pos := Pos - 1;
|
|
348
344
|
end loop;
|
|
@@ -355,67 +351,67 @@ package body Stemmer with SPARK_Mode is
|
|
|
355
351
|
Shift : in Natural) return Utf8_Type
|
|
356
352
|
is (Utf8_Type (Interfaces.Shift_Left (Interfaces.Unsigned_32 (Value), Shift)));
|
|
357
353
|
|
|
358
|
-
procedure Get_Utf8 (
|
|
359
|
-
Value
|
|
360
|
-
Count
|
|
354
|
+
procedure Get_Utf8 (Z : in Context_Type'Class;
|
|
355
|
+
Value : out Utf8_Type;
|
|
356
|
+
Count : out Natural) is
|
|
361
357
|
B0, B1, B2, B3 : Byte;
|
|
362
358
|
begin
|
|
363
|
-
if
|
|
359
|
+
if Z.C >= Z.L then
|
|
364
360
|
Value := 0;
|
|
365
361
|
Count := 0;
|
|
366
362
|
return;
|
|
367
363
|
end if;
|
|
368
|
-
B0 := Character'Pos (
|
|
369
|
-
if B0 < 16#C0# or
|
|
364
|
+
B0 := Character'Pos (Z.P (Z.C + 1));
|
|
365
|
+
if B0 < 16#C0# or Z.C + 1 >= Z.L then
|
|
370
366
|
Value := Utf8_Type (B0);
|
|
371
367
|
Count := 1;
|
|
372
368
|
return;
|
|
373
369
|
end if;
|
|
374
|
-
B1 := Character'Pos (
|
|
375
|
-
if B0 < 16#E0# or
|
|
370
|
+
B1 := Character'Pos (Z.P (Z.C + 2)) and 16#3F#;
|
|
371
|
+
if B0 < 16#E0# or Z.C + 2 >= Z.L then
|
|
376
372
|
Value := Shift_Left (Utf8_Type (B0 and 16#1F#), 6) or Utf8_Type (B1);
|
|
377
373
|
Count := 2;
|
|
378
374
|
return;
|
|
379
375
|
end if;
|
|
380
|
-
B2 := Character'Pos (
|
|
381
|
-
if B0 < 16#F0# or
|
|
376
|
+
B2 := Character'Pos (Z.P (Z.C + 3)) and 16#3F#;
|
|
377
|
+
if B0 < 16#F0# or Z.C + 3 >= Z.L then
|
|
382
378
|
Value := Shift_Left (Utf8_Type (B0 and 16#0F#), 12)
|
|
383
379
|
or Shift_Left (Utf8_Type (B1), 6) or Utf8_Type (B2);
|
|
384
380
|
Count := 3;
|
|
385
381
|
return;
|
|
386
382
|
end if;
|
|
387
|
-
B3 := Character'Pos (
|
|
383
|
+
B3 := Character'Pos (Z.P (Z.C + 4)) and 16#3F#;
|
|
388
384
|
Value := Shift_Left (Utf8_Type (B0 and 16#07#), 18)
|
|
389
385
|
or Shift_Left (Utf8_Type (B1), 12)
|
|
390
386
|
or Shift_Left (Utf8_Type (B2), 6) or Utf8_Type (B3);
|
|
391
387
|
Count := 4;
|
|
392
388
|
end Get_Utf8;
|
|
393
389
|
|
|
394
|
-
procedure Get_Utf8_Backward (
|
|
395
|
-
Value
|
|
396
|
-
Count
|
|
390
|
+
procedure Get_Utf8_Backward (Z : in Context_Type'Class;
|
|
391
|
+
Value : out Utf8_Type;
|
|
392
|
+
Count : out Natural) is
|
|
397
393
|
B0, B1, B2, B3 : Byte;
|
|
398
394
|
begin
|
|
399
|
-
if
|
|
395
|
+
if Z.C <= Z.Lb then
|
|
400
396
|
Value := 0;
|
|
401
397
|
Count := 0;
|
|
402
398
|
return;
|
|
403
399
|
end if;
|
|
404
|
-
B3 := Character'Pos (
|
|
405
|
-
if B3 < 16#80# or
|
|
400
|
+
B3 := Character'Pos (Z.P (Z.C));
|
|
401
|
+
if B3 < 16#80# or Z.C - 1 <= Z.Lb then
|
|
406
402
|
Value := Utf8_Type (B3);
|
|
407
403
|
Count := 1;
|
|
408
404
|
return;
|
|
409
405
|
end if;
|
|
410
|
-
B2 := Character'Pos (
|
|
411
|
-
if B2 >= 16#C0# or
|
|
406
|
+
B2 := Character'Pos (Z.P (Z.C - 1));
|
|
407
|
+
if B2 >= 16#C0# or Z.C - 2 <= Z.Lb then
|
|
412
408
|
B3 := B3 and 16#3F#;
|
|
413
409
|
Value := Shift_Left (Utf8_Type (B2 and 16#1F#), 6) or Utf8_Type (B3);
|
|
414
410
|
Count := 2;
|
|
415
411
|
return;
|
|
416
412
|
end if;
|
|
417
|
-
B1 := Character'Pos (
|
|
418
|
-
if B1 >= 16#E0# or
|
|
413
|
+
B1 := Character'Pos (Z.P (Z.C - 2));
|
|
414
|
+
if B1 >= 16#E0# or Z.C - 3 <= Z.Lb then
|
|
419
415
|
B3 := B3 and 16#3F#;
|
|
420
416
|
B2 := B2 and 16#3F#;
|
|
421
417
|
Value := Shift_Left (Utf8_Type (B1 and 16#0F#), 12)
|
|
@@ -423,7 +419,7 @@ package body Stemmer with SPARK_Mode is
|
|
|
423
419
|
Count := 3;
|
|
424
420
|
return;
|
|
425
421
|
end if;
|
|
426
|
-
B0 := Character'Pos (
|
|
422
|
+
B0 := Character'Pos (Z.P (Z.C - 3));
|
|
427
423
|
B1 := B1 and 16#1F#;
|
|
428
424
|
B2 := B2 and 16#3F#;
|
|
429
425
|
B3 := B3 and 16#3F#;
|
|
@@ -433,22 +429,22 @@ package body Stemmer with SPARK_Mode is
|
|
|
433
429
|
Count := 4;
|
|
434
430
|
end Get_Utf8_Backward;
|
|
435
431
|
|
|
436
|
-
procedure Out_Grouping (
|
|
437
|
-
S
|
|
438
|
-
Min
|
|
439
|
-
Max
|
|
440
|
-
Repeat
|
|
441
|
-
Result
|
|
432
|
+
procedure Out_Grouping (Z : in out Context_Type'Class;
|
|
433
|
+
S : in Grouping_Array;
|
|
434
|
+
Min : in Utf8_Type;
|
|
435
|
+
Max : in Utf8_Type;
|
|
436
|
+
Repeat : in Boolean;
|
|
437
|
+
Result : out Result_Index) is
|
|
442
438
|
Ch : Utf8_Type;
|
|
443
439
|
Count : Natural;
|
|
444
440
|
begin
|
|
445
|
-
if
|
|
441
|
+
if Z.C >= Z.L then
|
|
446
442
|
Result := -1;
|
|
447
443
|
return;
|
|
448
444
|
end if;
|
|
449
445
|
|
|
450
446
|
loop
|
|
451
|
-
Get_Utf8 (
|
|
447
|
+
Get_Utf8 (Z, Ch, Count);
|
|
452
448
|
if Count = 0 then
|
|
453
449
|
Result := -1;
|
|
454
450
|
return;
|
|
@@ -460,28 +456,28 @@ package body Stemmer with SPARK_Mode is
|
|
|
460
456
|
return;
|
|
461
457
|
end if;
|
|
462
458
|
end if;
|
|
463
|
-
|
|
459
|
+
Z.C := Z.C + Count;
|
|
464
460
|
exit when not Repeat;
|
|
465
461
|
end loop;
|
|
466
462
|
Result := 0;
|
|
467
463
|
end Out_Grouping;
|
|
468
464
|
|
|
469
|
-
procedure Out_Grouping_Backward (
|
|
470
|
-
S
|
|
471
|
-
Min
|
|
472
|
-
Max
|
|
473
|
-
Repeat
|
|
474
|
-
Result
|
|
465
|
+
procedure Out_Grouping_Backward (Z : in out Context_Type'Class;
|
|
466
|
+
S : in Grouping_Array;
|
|
467
|
+
Min : in Utf8_Type;
|
|
468
|
+
Max : in Utf8_Type;
|
|
469
|
+
Repeat : in Boolean;
|
|
470
|
+
Result : out Result_Index) is
|
|
475
471
|
Ch : Utf8_Type;
|
|
476
472
|
Count : Natural;
|
|
477
473
|
begin
|
|
478
|
-
if
|
|
474
|
+
if Z.C <= Z.Lb then
|
|
479
475
|
Result := -1;
|
|
480
476
|
return;
|
|
481
477
|
end if;
|
|
482
478
|
|
|
483
479
|
loop
|
|
484
|
-
Get_Utf8_Backward (
|
|
480
|
+
Get_Utf8_Backward (Z, Ch, Count);
|
|
485
481
|
if Count = 0 then
|
|
486
482
|
Result := -1;
|
|
487
483
|
return;
|
|
@@ -493,28 +489,28 @@ package body Stemmer with SPARK_Mode is
|
|
|
493
489
|
return;
|
|
494
490
|
end if;
|
|
495
491
|
end if;
|
|
496
|
-
|
|
492
|
+
Z.C := Z.C - Count;
|
|
497
493
|
exit when not Repeat;
|
|
498
494
|
end loop;
|
|
499
495
|
Result := 0;
|
|
500
496
|
end Out_Grouping_Backward;
|
|
501
497
|
|
|
502
|
-
procedure In_Grouping (
|
|
503
|
-
S
|
|
504
|
-
Min
|
|
505
|
-
Max
|
|
506
|
-
Repeat
|
|
507
|
-
Result
|
|
498
|
+
procedure In_Grouping (Z : in out Context_Type'Class;
|
|
499
|
+
S : in Grouping_Array;
|
|
500
|
+
Min : in Utf8_Type;
|
|
501
|
+
Max : in Utf8_Type;
|
|
502
|
+
Repeat : in Boolean;
|
|
503
|
+
Result : out Result_Index) is
|
|
508
504
|
Ch : Utf8_Type;
|
|
509
505
|
Count : Natural;
|
|
510
506
|
begin
|
|
511
|
-
if
|
|
507
|
+
if Z.C >= Z.L then
|
|
512
508
|
Result := -1;
|
|
513
509
|
return;
|
|
514
510
|
end if;
|
|
515
511
|
|
|
516
512
|
loop
|
|
517
|
-
Get_Utf8 (
|
|
513
|
+
Get_Utf8 (Z, Ch, Count);
|
|
518
514
|
if Count = 0 then
|
|
519
515
|
Result := -1;
|
|
520
516
|
return;
|
|
@@ -528,28 +524,28 @@ package body Stemmer with SPARK_Mode is
|
|
|
528
524
|
Result := Count;
|
|
529
525
|
return;
|
|
530
526
|
end if;
|
|
531
|
-
|
|
527
|
+
Z.C := Z.C + Count;
|
|
532
528
|
exit when not Repeat;
|
|
533
529
|
end loop;
|
|
534
530
|
Result := 0;
|
|
535
531
|
end In_Grouping;
|
|
536
532
|
|
|
537
|
-
procedure In_Grouping_Backward (
|
|
538
|
-
S
|
|
539
|
-
Min
|
|
540
|
-
Max
|
|
541
|
-
Repeat
|
|
542
|
-
Result
|
|
533
|
+
procedure In_Grouping_Backward (Z : in out Context_Type'Class;
|
|
534
|
+
S : in Grouping_Array;
|
|
535
|
+
Min : in Utf8_Type;
|
|
536
|
+
Max : in Utf8_Type;
|
|
537
|
+
Repeat : in Boolean;
|
|
538
|
+
Result : out Result_Index) is
|
|
543
539
|
Ch : Utf8_Type;
|
|
544
540
|
Count : Natural;
|
|
545
541
|
begin
|
|
546
|
-
if
|
|
542
|
+
if Z.C <= Z.Lb then
|
|
547
543
|
Result := -1;
|
|
548
544
|
return;
|
|
549
545
|
end if;
|
|
550
546
|
|
|
551
547
|
loop
|
|
552
|
-
Get_Utf8_Backward (
|
|
548
|
+
Get_Utf8_Backward (Z, Ch, Count);
|
|
553
549
|
if Count = 0 then
|
|
554
550
|
Result := -1;
|
|
555
551
|
return;
|
|
@@ -563,69 +559,62 @@ package body Stemmer with SPARK_Mode is
|
|
|
563
559
|
Result := Count;
|
|
564
560
|
return;
|
|
565
561
|
end if;
|
|
566
|
-
|
|
562
|
+
Z.C := Z.C - Count;
|
|
567
563
|
exit when not Repeat;
|
|
568
564
|
end loop;
|
|
569
565
|
Result := 0;
|
|
570
566
|
end In_Grouping_Backward;
|
|
571
567
|
|
|
572
|
-
procedure Replace (
|
|
573
|
-
C_Bra
|
|
574
|
-
C_Ket
|
|
575
|
-
S
|
|
576
|
-
|
|
568
|
+
procedure Replace (Z : in out Context_Type'Class;
|
|
569
|
+
C_Bra : in Char_Index;
|
|
570
|
+
C_Ket : in Char_Index;
|
|
571
|
+
S : in String;
|
|
572
|
+
Len : in Char_Index) is
|
|
573
|
+
Adjustment : Integer;
|
|
577
574
|
begin
|
|
578
|
-
Adjustment :=
|
|
579
|
-
if Adjustment
|
|
580
|
-
|
|
581
|
-
:=
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
575
|
+
Adjustment := Len - (C_Ket - C_Bra);
|
|
576
|
+
if Adjustment /= 0 then
|
|
577
|
+
Z.P (C_Ket + Adjustment + 1 .. Z.Len + Adjustment)
|
|
578
|
+
:= Z.P (C_Ket + 1 .. Z.Len);
|
|
579
|
+
Z.Len := Z.Len + Adjustment;
|
|
580
|
+
Z.L := Z.L + Adjustment;
|
|
581
|
+
if Z.C >= C_Ket then
|
|
582
|
+
Z.C := Z.C + Adjustment;
|
|
583
|
+
elsif Z.C > C_Bra then
|
|
584
|
+
Z.C := C_Bra;
|
|
585
|
+
end if;
|
|
589
586
|
end if;
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
if Context.C >= C_Ket then
|
|
593
|
-
Context.C := Context.C + Adjustment;
|
|
594
|
-
elsif Context.C > C_Bra then
|
|
595
|
-
Context.C := C_Bra;
|
|
587
|
+
if Len > 0 then
|
|
588
|
+
Z.P (C_Bra + 1 .. C_Bra + Len) := S (S'First .. Len);
|
|
596
589
|
end if;
|
|
597
590
|
end Replace;
|
|
598
591
|
|
|
599
|
-
procedure Slice_Del (
|
|
600
|
-
Result : Integer;
|
|
592
|
+
procedure Slice_Del (Z : in out Context_Type'Class) is
|
|
601
593
|
begin
|
|
602
|
-
Replace (
|
|
594
|
+
Replace (Z, Z.Bra, Z.Ket, "", 0);
|
|
595
|
+
Z.Ket := Z.Bra;
|
|
603
596
|
end Slice_Del;
|
|
604
597
|
|
|
605
|
-
procedure Slice_From (
|
|
606
|
-
Text
|
|
607
|
-
|
|
598
|
+
procedure Slice_From (Z : in out Context_Type'Class;
|
|
599
|
+
Text : in String;
|
|
600
|
+
Len : in Char_Index) is
|
|
608
601
|
begin
|
|
609
|
-
Replace (
|
|
602
|
+
Replace (Z, Z.Bra, Z.Ket, Text, Len);
|
|
603
|
+
Z.Ket := Z.Bra + Len;
|
|
610
604
|
end Slice_From;
|
|
611
605
|
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
procedure Insert (Context : in out Context_Type'Class;
|
|
618
|
-
C_Bra : in Char_Index;
|
|
619
|
-
C_Ket : in Char_Index;
|
|
620
|
-
S : in String) is
|
|
621
|
-
Result : Integer;
|
|
606
|
+
procedure Insert (Z : in out Context_Type'Class;
|
|
607
|
+
S : in String;
|
|
608
|
+
Len : in Char_Index) is
|
|
609
|
+
C : Char_Index;
|
|
622
610
|
begin
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
611
|
+
C := Z.C;
|
|
612
|
+
Replace (Z, Z.C, Z.C, S, Len);
|
|
613
|
+
if C <= Z.Bra then
|
|
614
|
+
Z.Bra := Z.Bra + Len;
|
|
626
615
|
end if;
|
|
627
|
-
if
|
|
628
|
-
|
|
616
|
+
if C <= Z.Ket then
|
|
617
|
+
Z.Ket := Z.Ket + Len;
|
|
629
618
|
end if;
|
|
630
619
|
end Insert;
|
|
631
620
|
|