mittens 0.3.3 → 0.3.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +8 -0
- data/LICENSE.txt +1 -1
- data/README.md +1 -1
- data/ext/mittens/ext.c +68 -30
- data/ext/mittens/extconf.rb +2 -0
- data/lib/mittens/version.rb +1 -1
- data/vendor/snowball/.github/workflows/ci.yml +43 -3
- data/vendor/snowball/.github/workflows/coverage.yml +8 -5
- data/vendor/snowball/GNUmakefile +46 -16
- data/vendor/snowball/NEWS +437 -13
- data/vendor/snowball/README.rst +4 -4
- data/vendor/snowball/algorithms/earlymodernenglish.sbl +301 -0
- data/vendor/snowball/algorithms/french.sbl +1 -1
- data/vendor/snowball/algorithms/ukrainian.sbl +1251 -0
- data/vendor/snowball/compiler/analyser.c +460 -156
- data/vendor/snowball/compiler/driver.c +41 -36
- data/vendor/snowball/compiler/generator.c +24 -1
- data/vendor/snowball/compiler/generator_ada.c +46 -45
- data/vendor/snowball/compiler/generator_c.c +1094 -187
- data/vendor/snowball/compiler/generator_csharp.c +34 -11
- data/vendor/snowball/compiler/generator_dart.c +39 -11
- data/vendor/snowball/compiler/generator_go.c +6 -6
- data/vendor/snowball/compiler/generator_java.c +29 -8
- data/vendor/snowball/compiler/generator_js.c +65 -38
- data/vendor/snowball/compiler/generator_pascal.c +69 -26
- data/vendor/snowball/compiler/generator_php.c +33 -12
- data/vendor/snowball/compiler/generator_python.c +23 -4
- data/vendor/snowball/compiler/generator_rust.c +37 -13
- data/vendor/snowball/compiler/generator_zig.c +6 -6
- data/vendor/snowball/compiler/header.h +85 -14
- data/vendor/snowball/compiler/space.c +67 -38
- data/vendor/snowball/compiler/tokeniser.c +28 -16
- data/vendor/snowball/csharp/Snowball/AssemblyInfo.cs +2 -2
- data/vendor/snowball/csharp/Snowball/Stemmer.cs +1 -4
- data/vendor/snowball/cxx/generate_algorithms.pl +5 -4
- data/vendor/snowball/cxx/{utilities.cxx → snowball_runtime.cxx} +1 -1
- data/vendor/snowball/cxx/stemmer.h +4 -0
- data/vendor/snowball/cxx/stemwords.cxx +36 -6
- data/vendor/snowball/dart/lib/src/snowball.dart +2 -2
- data/vendor/snowball/dart/pubspec.yaml +1 -1
- data/vendor/snowball/doc/libstemmer_c_README +3 -0
- data/vendor/snowball/doc/libstemmer_php_README +1 -1
- data/vendor/snowball/doc/libstemmer_python_README +8 -6
- data/vendor/snowball/examples/stemwords.c +4 -14
- data/vendor/snowball/go/env.go +16 -4
- data/vendor/snowball/include/libstemmer.h +0 -3
- data/vendor/snowball/java/org/tartarus/snowball/SnowballProgram.java +4 -4
- data/vendor/snowball/javascript/base-stemmer.js +27 -40
- data/vendor/snowball/javascript/stemwords.js +34 -10
- data/vendor/snowball/libstemmer/libstemmer_c.in +5 -5
- data/vendor/snowball/libstemmer/mkmodules.pl +5 -3
- data/vendor/snowball/libstemmer/modules.txt +4 -2
- data/vendor/snowball/pascal/SnowballProgram.pas +2 -2
- data/vendor/snowball/php/base-stemmer.php +2 -2
- data/vendor/snowball/python/setup.py +5 -1
- data/vendor/snowball/python/snowballstemmer/basestemmer.py +1 -2
- data/vendor/snowball/runtime/api.c +14 -3
- data/vendor/snowball/runtime/api.h +13 -9
- data/vendor/snowball/runtime/{utilities.c → snowball_runtime.c} +115 -181
- data/vendor/snowball/runtime/snowball_runtime.h +35 -24
- data/vendor/snowball/rust/src/snowball/snowball_env.rs +1 -1
- data/vendor/snowball/tests/compilertest +3 -3
- data/vendor/snowball/tests/runtime/among.sbl +96 -2
- data/vendor/snowball/tests/runtime/attachinsert.sbl +4 -0
- data/vendor/snowball/tests/runtime/debug.out +7 -0
- data/vendor/snowball/tests/runtime/debug.sbl +9 -0
- data/vendor/snowball/tests/runtime/integertests.sbl +22 -0
- data/vendor/snowball/tests/runtime/marklimit.sbl +30 -0
- data/vendor/snowball/tests/runtime/naming.sbl +20 -0
- data/vendor/snowball/tests/runtime/reverse.sbl +11 -0
- data/vendor/snowball/tests/stemtest.c +2 -5
- data/vendor/snowball/tests/syntax/aeslicefrom.sbl +7 -0
- data/vendor/snowball/tests/syntax/canon.sbl +14 -0
- data/vendor/snowball/tests/syntax/canon.syntax +36 -0
- data/vendor/snowball/tests/syntax/inline.syntax +4 -4
- data/vendor/snowball/tests/syntax/localvar.sbl +38 -0
- data/vendor/snowball/tests/syntax/localvar.stderr +3 -0
- data/vendor/snowball/tests/syntax/localvar.syntax +76 -0
- data/vendor/snowball/tests/syntax/unused.sbl +5 -1
- data/vendor/snowball/tests/syntax/unused.stderr +3 -1
- data/vendor/snowball/zig/env.zig +24 -4
- metadata +14 -4
|
@@ -54,7 +54,7 @@ static void write_varref(struct generator * g, struct name * p) {
|
|
|
54
54
|
write_varname(g, p);
|
|
55
55
|
}
|
|
56
56
|
|
|
57
|
-
static void write_literal_string(struct generator * g, symbol * p) {
|
|
57
|
+
static void write_literal_string(struct generator * g, const symbol * p) {
|
|
58
58
|
write_char(g, '"');
|
|
59
59
|
for (int i = 0; i < SIZE(p); i++) {
|
|
60
60
|
int ch = p[i];
|
|
@@ -1201,7 +1201,7 @@ static void generate_debug(struct generator * g, struct node * p) {
|
|
|
1201
1201
|
write_comment(g, p);
|
|
1202
1202
|
g->I[0] = g->debug_count++;
|
|
1203
1203
|
g->I[1] = p->line_number;
|
|
1204
|
-
writef(g, "~Mself.
|
|
1204
|
+
writef(g, "~Mself.__debug(~I0, ~I1)~N", p);
|
|
1205
1205
|
}
|
|
1206
1206
|
|
|
1207
1207
|
static void generate(struct generator * g, struct node * p) {
|
|
@@ -1314,7 +1314,7 @@ static void generate_among_table(struct generator * g, struct among * x) {
|
|
|
1314
1314
|
write_newline(g);
|
|
1315
1315
|
write_comment(g, x->node);
|
|
1316
1316
|
|
|
1317
|
-
struct amongvec * v = x->
|
|
1317
|
+
struct amongvec * v = x->v;
|
|
1318
1318
|
|
|
1319
1319
|
g->I[0] = x->number;
|
|
1320
1320
|
|
|
@@ -1348,7 +1348,7 @@ static void generate_among_table(struct generator * g, struct among * x) {
|
|
|
1348
1348
|
|
|
1349
1349
|
static void generate_amongs(struct generator * g) {
|
|
1350
1350
|
for (struct among * x = g->analyser->amongs; x; x = x->next) {
|
|
1351
|
-
generate_among_table(g, x);
|
|
1351
|
+
if (!x->duplicate) generate_among_table(g, x);
|
|
1352
1352
|
}
|
|
1353
1353
|
}
|
|
1354
1354
|
|
|
@@ -1442,6 +1442,25 @@ extern void generate_program_python(struct generator * g) {
|
|
|
1442
1442
|
generate_groupings(g);
|
|
1443
1443
|
|
|
1444
1444
|
generate_members(g);
|
|
1445
|
+
|
|
1446
|
+
if (g->analyser->debug_used) {
|
|
1447
|
+
w(g, "~N"
|
|
1448
|
+
"~Mdef __debug(self, n, line):~N~+"
|
|
1449
|
+
"~Mlength = len(self.current)~N"
|
|
1450
|
+
"~Mprint(\"%3d (line %4d): [%d]'\" % (n, line, length), end='')~N"
|
|
1451
|
+
"~Mfor i in range(0, length + 1):~N~+"
|
|
1452
|
+
"~Mif self.limit_backward == i: print('{', end='')~N"
|
|
1453
|
+
"~Mif self.bra == i: print('[', end='')~N"
|
|
1454
|
+
"~Mif self.cursor == i: print('|', end='')~N"
|
|
1455
|
+
"~Mif self.ket == i: print(']', end='')~N"
|
|
1456
|
+
"~Mif self.limit == i: print('}', end='')~N"
|
|
1457
|
+
"~Mif i < length:~N~+"
|
|
1458
|
+
"~Mch = self.current[i]~N"
|
|
1459
|
+
"~Mprint(ch if ch != '\\0' else '#', end='')~N~-~-"
|
|
1460
|
+
"~Mprint(\"'\")~N~-"
|
|
1461
|
+
"~N");
|
|
1462
|
+
}
|
|
1463
|
+
|
|
1445
1464
|
generate_methods(g);
|
|
1446
1465
|
|
|
1447
1466
|
generate_amongs(g);
|
|
@@ -32,7 +32,7 @@ static void write_varref(struct generator * g, struct name * p) {
|
|
|
32
32
|
write_varname(g, p);
|
|
33
33
|
}
|
|
34
34
|
|
|
35
|
-
static void write_literal_string(struct generator * g, symbol * p) {
|
|
35
|
+
static void write_literal_string(struct generator * g, const symbol * p) {
|
|
36
36
|
int i = 0;
|
|
37
37
|
write_char(g, '"');
|
|
38
38
|
while (i < SIZE(p)) {
|
|
@@ -47,7 +47,7 @@ static void write_literal_string(struct generator * g, symbol * p) {
|
|
|
47
47
|
write_wchar_as_utf8(g, ch);
|
|
48
48
|
} else {
|
|
49
49
|
write_string(g, "\\u{");
|
|
50
|
-
|
|
50
|
+
write_hex(g, ch);
|
|
51
51
|
write_string(g, "}");
|
|
52
52
|
}
|
|
53
53
|
}
|
|
@@ -702,10 +702,10 @@ static void generate_hop(struct generator * g, struct node * p) {
|
|
|
702
702
|
// Generate the AE to a temporary block so we can substitute it in
|
|
703
703
|
// write_failure_if().
|
|
704
704
|
struct str * ae = str_new();
|
|
705
|
-
struct str *
|
|
705
|
+
struct str * saved_outbuf = g->outbuf;
|
|
706
706
|
g->outbuf = ae;
|
|
707
707
|
generate_AE(g, p->AE);
|
|
708
|
-
g->outbuf =
|
|
708
|
+
g->outbuf = saved_outbuf;
|
|
709
709
|
g->B[0] = str_data(ae);
|
|
710
710
|
g->S[0] = p->mode == m_forward ? "" : "_back";
|
|
711
711
|
g->S[1] = p->AE->type == c_number ? "" : "_checked";
|
|
@@ -1086,7 +1086,7 @@ static void generate_define(struct generator * g, struct node * p) {
|
|
|
1086
1086
|
}
|
|
1087
1087
|
}
|
|
1088
1088
|
|
|
1089
|
-
if (
|
|
1089
|
+
if (amongvar_needed(p->left)) {
|
|
1090
1090
|
w(g, "~Mlet mut among_var;~N");
|
|
1091
1091
|
}
|
|
1092
1092
|
|
|
@@ -1143,7 +1143,7 @@ static void generate_substring(struct generator * g, struct node * p) {
|
|
|
1143
1143
|
struct among * x = p->among;
|
|
1144
1144
|
int block = -1;
|
|
1145
1145
|
unsigned int bitmap = 0;
|
|
1146
|
-
struct amongvec * among_cases = x->
|
|
1146
|
+
struct amongvec * among_cases = x->v;
|
|
1147
1147
|
int empty_case = -1;
|
|
1148
1148
|
int n_cases = 0;
|
|
1149
1149
|
symbol cases[2];
|
|
@@ -1340,7 +1340,7 @@ static void generate_debug(struct generator * g, struct node * p) {
|
|
|
1340
1340
|
write_comment(g, p);
|
|
1341
1341
|
g->I[0] = g->debug_count++;
|
|
1342
1342
|
g->I[1] = p->line_number;
|
|
1343
|
-
writef(g, "~
|
|
1343
|
+
writef(g, "~Mdebug(env, ~I0, ~I1);~N", p);
|
|
1344
1344
|
}
|
|
1345
1345
|
|
|
1346
1346
|
static void generate(struct generator * g, struct node * p) {
|
|
@@ -1444,7 +1444,7 @@ static void generate_class_begin(struct generator * g) {
|
|
|
1444
1444
|
static void generate_among_table(struct generator * g, struct among * x) {
|
|
1445
1445
|
write_comment(g, x->node);
|
|
1446
1446
|
|
|
1447
|
-
struct amongvec * v = x->
|
|
1447
|
+
struct amongvec * v = x->v;
|
|
1448
1448
|
|
|
1449
1449
|
g->I[0] = x->number;
|
|
1450
1450
|
g->I[1] = x->literalstring_count;
|
|
@@ -1470,12 +1470,12 @@ static void generate_among_table(struct generator * g, struct among * x) {
|
|
|
1470
1470
|
}
|
|
1471
1471
|
|
|
1472
1472
|
static void generate_amongs(struct generator * g) {
|
|
1473
|
-
struct str *
|
|
1473
|
+
struct str * saved_outbuf = g->outbuf;
|
|
1474
1474
|
g->outbuf = g->declarations;
|
|
1475
1475
|
for (struct among * x = g->analyser->amongs; x; x = x->next) {
|
|
1476
|
-
if (x->used) generate_among_table(g, x);
|
|
1476
|
+
if (x->used && !x->duplicate) generate_among_table(g, x);
|
|
1477
1477
|
}
|
|
1478
|
-
g->outbuf =
|
|
1478
|
+
g->outbuf = saved_outbuf;
|
|
1479
1479
|
}
|
|
1480
1480
|
|
|
1481
1481
|
static void set_bit(symbol * b, int i) { b[i >> 3] |= 1 << (i & 7); }
|
|
@@ -1504,12 +1504,12 @@ static void generate_grouping_table(struct generator * g, struct grouping * q) {
|
|
|
1504
1504
|
}
|
|
1505
1505
|
|
|
1506
1506
|
static void generate_groupings(struct generator * g) {
|
|
1507
|
-
struct str *
|
|
1507
|
+
struct str * saved_outbuf = g->outbuf;
|
|
1508
1508
|
g->outbuf = g->declarations;
|
|
1509
1509
|
for (struct grouping * q = g->analyser->groupings; q; q = q->next) {
|
|
1510
1510
|
generate_grouping_table(g, q);
|
|
1511
1511
|
}
|
|
1512
|
-
g->outbuf =
|
|
1512
|
+
g->outbuf = saved_outbuf;
|
|
1513
1513
|
}
|
|
1514
1514
|
|
|
1515
1515
|
static void generate_members(struct generator * g) {
|
|
@@ -1555,6 +1555,10 @@ extern void generate_program_rust(struct generator * g) {
|
|
|
1555
1555
|
/* std::i32 is used in the code generated for i32::MAX and i32::MIN */
|
|
1556
1556
|
w(g, "use std::i32;~N~N");
|
|
1557
1557
|
}
|
|
1558
|
+
if (g->analyser->debug_used) {
|
|
1559
|
+
w(g, "use std::io;~N");
|
|
1560
|
+
w(g, "use std::io::Write;~N~N");
|
|
1561
|
+
}
|
|
1558
1562
|
generate_class_begin(g);
|
|
1559
1563
|
|
|
1560
1564
|
generate_members(g);
|
|
@@ -1562,6 +1566,26 @@ extern void generate_program_rust(struct generator * g) {
|
|
|
1562
1566
|
g->declarations = g->outbuf;
|
|
1563
1567
|
g->outbuf = str_new();
|
|
1564
1568
|
|
|
1569
|
+
if (g->analyser->debug_used) {
|
|
1570
|
+
w(g, "~Mfn debug(env: &mut SnowballEnv, n: i32, line: i32) {~+~N"
|
|
1571
|
+
"~Mlet length = env.current.len();~N"
|
|
1572
|
+
"~Mprint!(\"{:3} (line {:4}): [{}]'\", n, line, length);~N"
|
|
1573
|
+
"~Mlet mut b = env.current.bytes();~N"
|
|
1574
|
+
"~Mfor i in 0..length + 1 {~N~+"
|
|
1575
|
+
"~Mif env.limit_backward == i as i32 { print!(\"{{\") }~N"
|
|
1576
|
+
"~Mif env.bra == i as i32 { print!(\"[\") }~N"
|
|
1577
|
+
"~Mif env.cursor == i as i32 { print!(\"|\") }~N"
|
|
1578
|
+
"~Mif env.ket == i as i32 { print!(\"]\") }~N"
|
|
1579
|
+
"~Mif env.limit == i as i32 { print!(\"}}\") }~N"
|
|
1580
|
+
"~Mif (i < length) {~N~+"
|
|
1581
|
+
"~Mlet ch = b.next();~N"
|
|
1582
|
+
"~Mif ch == Some(0) { print!(\"#\"); } else { let _ = io::stdout().write(ch.as_slice()); }~N~-"
|
|
1583
|
+
"~M}~N~-"
|
|
1584
|
+
"~M}~N"
|
|
1585
|
+
"~Mprintln!(\"'\");~N~-"
|
|
1586
|
+
"}~N~N");
|
|
1587
|
+
}
|
|
1588
|
+
|
|
1565
1589
|
generate_methods(g);
|
|
1566
1590
|
|
|
1567
1591
|
generate_amongs(g);
|
|
@@ -38,7 +38,7 @@ static void write_stringref(struct generator * g, struct name * p) {
|
|
|
38
38
|
write_string(g, ".slice()");
|
|
39
39
|
}
|
|
40
40
|
|
|
41
|
-
static void write_literal_string(struct generator * g, symbol * p) {
|
|
41
|
+
static void write_literal_string(struct generator * g, const symbol * p) {
|
|
42
42
|
int i = 0;
|
|
43
43
|
write_char(g, '"');
|
|
44
44
|
while (i < SIZE(p)) {
|
|
@@ -718,10 +718,10 @@ static void generate_hop(struct generator * g, struct node * p) {
|
|
|
718
718
|
// Generate the AE to a temporary block so we can substitute it in
|
|
719
719
|
// write_failure_if().
|
|
720
720
|
struct str * ae = str_new();
|
|
721
|
-
struct str *
|
|
721
|
+
struct str * saved_outbuf = g->outbuf;
|
|
722
722
|
g->outbuf = ae;
|
|
723
723
|
generate_AE(g, p->AE);
|
|
724
|
-
g->outbuf =
|
|
724
|
+
g->outbuf = saved_outbuf;
|
|
725
725
|
g->B[0] = str_data(ae);
|
|
726
726
|
g->S[0] = p->mode == m_forward ? "" : "Back";
|
|
727
727
|
g->S[1] = p->AE->type == c_number ? "" : "Checked";
|
|
@@ -1134,7 +1134,7 @@ static void generate_define(struct generator * g, struct node * p) {
|
|
|
1134
1134
|
}
|
|
1135
1135
|
}
|
|
1136
1136
|
|
|
1137
|
-
if (
|
|
1137
|
+
if (amongvar_needed(p->left)) {
|
|
1138
1138
|
w(g, "~Mvar among_var: i32 = 0;~N");
|
|
1139
1139
|
}
|
|
1140
1140
|
|
|
@@ -1364,7 +1364,7 @@ static void generate_class_begin(struct generator * g) {
|
|
|
1364
1364
|
static void generate_among_table(struct generator * g, struct among * x) {
|
|
1365
1365
|
write_comment(g, x->node);
|
|
1366
1366
|
|
|
1367
|
-
struct amongvec * v = x->
|
|
1367
|
+
struct amongvec * v = x->v;
|
|
1368
1368
|
|
|
1369
1369
|
g->I[0] = x->number;
|
|
1370
1370
|
w(g, "~Mconst a_~I0 = [_]snowball.Among{~N~+");
|
|
@@ -1389,7 +1389,7 @@ static void generate_among_table(struct generator * g, struct among * x) {
|
|
|
1389
1389
|
|
|
1390
1390
|
static void generate_amongs(struct generator * g) {
|
|
1391
1391
|
for (struct among * x = g->analyser->amongs; x; x = x->next) {
|
|
1392
|
-
generate_among_table(g, x);
|
|
1392
|
+
if (!x->duplicate) generate_among_table(g, x);
|
|
1393
1393
|
}
|
|
1394
1394
|
}
|
|
1395
1395
|
|
|
@@ -1,9 +1,13 @@
|
|
|
1
1
|
#include <stdio.h>
|
|
2
2
|
#if __STDC_VERSION__ < 202311l
|
|
3
3
|
# include <stdbool.h>
|
|
4
|
+
#elif defined _MSC_VER && __STDC_VERSION__-0 <= 202312l
|
|
5
|
+
// MSVC 2026 with -std:clatest defines __STDC_VERSION__ to 202312 (one more
|
|
6
|
+
// than the correct value for C23!) but doesn't provide bool, true and false.
|
|
7
|
+
# include <stdbool.h>
|
|
4
8
|
#endif
|
|
5
9
|
|
|
6
|
-
#define SNOWBALL_VERSION "3.
|
|
10
|
+
#define SNOWBALL_VERSION "3.2.0"
|
|
7
11
|
|
|
8
12
|
typedef unsigned char byte;
|
|
9
13
|
typedef unsigned short symbol;
|
|
@@ -26,7 +30,17 @@ typedef unsigned short symbol;
|
|
|
26
30
|
extern symbol * create_b(int n);
|
|
27
31
|
extern void report_b(FILE * out, const symbol * p);
|
|
28
32
|
extern void lose_b(symbol * p);
|
|
29
|
-
extern symbol *
|
|
33
|
+
extern symbol * reserve_b_(symbol * p, int n);
|
|
34
|
+
static inline symbol * reserve_b(symbol * p, int n) {
|
|
35
|
+
if (n > CAPACITY(p)) p = reserve_b_(p, n);
|
|
36
|
+
return p;
|
|
37
|
+
}
|
|
38
|
+
extern symbol * resize_b(symbol * p, int n);
|
|
39
|
+
static inline symbol * ensure_nul_b(symbol * p) {
|
|
40
|
+
p = reserve_b(p, SIZE(p) + 1);
|
|
41
|
+
p[SIZE(p)] = 0;
|
|
42
|
+
return p;
|
|
43
|
+
}
|
|
30
44
|
extern symbol * add_to_b(symbol * p, const symbol * q, int n);
|
|
31
45
|
extern symbol * copy_b(const symbol * p);
|
|
32
46
|
extern char * b_to_sz(const symbol * p);
|
|
@@ -40,8 +54,17 @@ extern byte * create_s_from_data(const char * s, int n);
|
|
|
40
54
|
|
|
41
55
|
extern void report_s(FILE * out, const byte * p);
|
|
42
56
|
extern void lose_s(byte * p);
|
|
43
|
-
extern byte *
|
|
44
|
-
|
|
57
|
+
extern byte * reserve_s_(byte * p, int n);
|
|
58
|
+
static inline byte * reserve_s(byte * p, int n) {
|
|
59
|
+
if (n > CAPACITY(p)) p = reserve_s_(p, n);
|
|
60
|
+
return p;
|
|
61
|
+
}
|
|
62
|
+
extern byte * resize_s(byte * p, int n);
|
|
63
|
+
static inline byte * ensure_nul_s(byte * p) {
|
|
64
|
+
p = reserve_s(p, SIZE(p) + 1);
|
|
65
|
+
p[SIZE(p)] = 0;
|
|
66
|
+
return p;
|
|
67
|
+
}
|
|
45
68
|
extern byte * copy_s(const byte * p);
|
|
46
69
|
extern byte * add_s_to_s(byte * p, const byte * s);
|
|
47
70
|
extern byte * add_slen_to_s(byte * p, const char * s, int n);
|
|
@@ -201,6 +224,7 @@ extern byte * get_input(const char * filename);
|
|
|
201
224
|
extern struct tokeniser * create_tokeniser(byte * b, char * file);
|
|
202
225
|
extern int read_token(struct tokeniser * t);
|
|
203
226
|
extern int peek_token(struct tokeniser * t);
|
|
227
|
+
extern void push_token(struct tokeniser * t, int token);
|
|
204
228
|
#define hold_token(T) ((T)->token_held = true)
|
|
205
229
|
extern const char * name_of_token(int code);
|
|
206
230
|
extern void disable_token(struct tokeniser * t, int code);
|
|
@@ -210,8 +234,8 @@ extern int space_count;
|
|
|
210
234
|
extern void * check_malloc(size_t n);
|
|
211
235
|
extern void check_free(void * p);
|
|
212
236
|
|
|
213
|
-
extern int checked_snprintf(char *str, size_t size,
|
|
214
|
-
const char *
|
|
237
|
+
extern int checked_snprintf(char * str, size_t size,
|
|
238
|
+
const char * format, ...)
|
|
215
239
|
#ifdef __GNUC__
|
|
216
240
|
__attribute__ ((__format__ (__printf__, 3, 4)))
|
|
217
241
|
#endif
|
|
@@ -227,9 +251,15 @@ struct name {
|
|
|
227
251
|
bool value_used; /* (For variables) is its value ever used? */
|
|
228
252
|
bool initialised; /* (For variables) is it ever initialised? */
|
|
229
253
|
bool used_in_definition; /* (grouping) used in grouping definition? */
|
|
230
|
-
|
|
231
|
-
bool
|
|
254
|
+
// (routines/externals) contains among
|
|
255
|
+
bool has_among;
|
|
256
|
+
// (routines/externals) contains among with function(s)
|
|
257
|
+
bool has_among_function;
|
|
232
258
|
bool case_collision; /* A name of the same type differs only by case */
|
|
259
|
+
// (routines/externals) Could this directly or indirectly call itself?
|
|
260
|
+
bool recursive;
|
|
261
|
+
// (routines/externals) Temporary flag used while determining `recursive`.
|
|
262
|
+
bool visited;
|
|
233
263
|
struct node * definition; /* (routines/externals) c_define node */
|
|
234
264
|
int uses_in_among; /* (routines/externals) Count of uses in amongs */
|
|
235
265
|
// Initialised to -1; set to -2 if reachable from an external.
|
|
@@ -250,12 +280,19 @@ struct literalstring {
|
|
|
250
280
|
symbol * b;
|
|
251
281
|
};
|
|
252
282
|
|
|
283
|
+
struct c_literalstring {
|
|
284
|
+
struct c_literalstring * next;
|
|
285
|
+
const symbol * b;
|
|
286
|
+
};
|
|
287
|
+
|
|
253
288
|
struct amongvec {
|
|
254
289
|
symbol * b; /* the string giving the case */
|
|
255
290
|
int size; /* - and its size */
|
|
256
291
|
struct node * action; /* the corresponding action */
|
|
257
|
-
|
|
258
|
-
int
|
|
292
|
+
// The amongvec index of the longest substring of b, or -1 for none.
|
|
293
|
+
int i;
|
|
294
|
+
// among_var value for this case (starts from 1, or -1 for empty action).
|
|
295
|
+
int result;
|
|
259
296
|
int line_number; /* for diagnostics */
|
|
260
297
|
int function_index; /* 1-based */
|
|
261
298
|
// 0-based index giving order of strings in source. Used for stable
|
|
@@ -264,24 +301,54 @@ struct amongvec {
|
|
|
264
301
|
struct name * function;
|
|
265
302
|
};
|
|
266
303
|
|
|
304
|
+
struct among_function_scenario {
|
|
305
|
+
struct name * function;
|
|
306
|
+
int t_result; // result (1...).
|
|
307
|
+
int f_result; // 0 or result (1...) or AFS_FLAG | af_index to chain.
|
|
308
|
+
// If `function` signals f, apply this delta to the value the cursor had on
|
|
309
|
+
// entry (add for forwards; subtract for backwards).
|
|
310
|
+
int cursor_adjustment;
|
|
311
|
+
};
|
|
312
|
+
|
|
313
|
+
// Used to find common-subtrees to reuse. We currently linear search this for
|
|
314
|
+
// each subtree we encode, but it seems in practice that's very fast, even for
|
|
315
|
+
// the large amongs in the stemmers we ship (the largest seems to be in greek.sbl
|
|
316
|
+
// which results in 1161 entries in this linked list).
|
|
317
|
+
//
|
|
318
|
+
// Note that if we had a separate list per value of len we would only need to
|
|
319
|
+
// store start in each item (and could store as a symbol*).
|
|
320
|
+
struct among_subtree {
|
|
321
|
+
struct among_subtree * next;
|
|
322
|
+
int start;
|
|
323
|
+
int len;
|
|
324
|
+
};
|
|
325
|
+
|
|
267
326
|
struct among {
|
|
268
327
|
struct among * next;
|
|
269
|
-
struct amongvec *
|
|
328
|
+
struct amongvec * v; /* pointer to the amongvec */
|
|
329
|
+
// Details for among function handling in the state machine implementation.
|
|
330
|
+
struct among_function_scenario * af;
|
|
331
|
+
struct among_subtree * subtree;
|
|
332
|
+
int af_count; /* number of entries in af. */
|
|
333
|
+
symbol * table; /* table used in C implementation. */
|
|
334
|
+
byte * table_endianness; /* flag values needing byteswap on big-endian. */
|
|
270
335
|
int number; /* amongs are numbered 0, 1, 2 ... */
|
|
271
336
|
int literalstring_count; /* in this among */
|
|
272
337
|
int command_count; /* in this among (excludes "no command" entries) */
|
|
273
338
|
int nocommand_count; /* number of "no command" entries in this among */
|
|
274
|
-
int function_count; /* number of
|
|
339
|
+
int function_count; /* number of cases with a function in this among */
|
|
340
|
+
int unique_function_count;/* number of different functions in this among */
|
|
275
341
|
bool amongvar_needed; /* do we need to set among_var? */
|
|
276
342
|
bool always_matches; /* will this among always match? */
|
|
277
343
|
bool used; /* is this among in reachable code? */
|
|
344
|
+
bool c0_used; /* Need c0 variable in C implementation? */
|
|
345
|
+
bool duplicate; /* Is this among a duplicate of another? */
|
|
278
346
|
int same_action; /* type code if same for all actions; <0 otherwise */
|
|
279
347
|
int shortest_size; /* smallest non-zero string length in this among */
|
|
280
348
|
int longest_size; /* longest string length in this among */
|
|
281
349
|
struct node * substring; /* i.e. substring ... among ( ... ) */
|
|
282
350
|
struct node ** commands; /* array with command_count entries */
|
|
283
351
|
struct node * node; /* pointer to the node for this among */
|
|
284
|
-
struct name * in_routine; /* pointer to name for routine this among is in */
|
|
285
352
|
};
|
|
286
353
|
|
|
287
354
|
struct grouping {
|
|
@@ -297,7 +364,7 @@ struct node {
|
|
|
297
364
|
struct node * next;
|
|
298
365
|
struct node * left;
|
|
299
366
|
struct node * aux; /* used in setlimit */
|
|
300
|
-
struct among * among; /* used in among */
|
|
367
|
+
struct among * among; /* used in substring and among */
|
|
301
368
|
struct node * right;
|
|
302
369
|
byte type;
|
|
303
370
|
byte mode;
|
|
@@ -409,6 +476,8 @@ struct generator {
|
|
|
409
476
|
const char * varname_prefix;
|
|
410
477
|
// String to indent by for each margin level (four spaces by default).
|
|
411
478
|
const char * margin_indent;
|
|
479
|
+
// (C/C++) Linked list used to merge string literals.
|
|
480
|
+
struct c_literalstring * c_literalstrings;
|
|
412
481
|
};
|
|
413
482
|
|
|
414
483
|
/* Special values for failure_label in struct generator. */
|
|
@@ -480,6 +549,7 @@ extern void write_int(struct generator * g, int i);
|
|
|
480
549
|
extern void wi3(struct generator * g, int i);
|
|
481
550
|
extern void write_hex4(struct generator * g, unsigned ch);
|
|
482
551
|
extern void write_hex(struct generator * g, unsigned i);
|
|
552
|
+
extern void write_octal3(struct generator * g, unsigned n);
|
|
483
553
|
extern void write_symbol(struct generator * g, symbol s);
|
|
484
554
|
extern void write_s(struct generator * g, const byte * b);
|
|
485
555
|
extern void write_str(struct generator * g, struct str * str);
|
|
@@ -497,6 +567,7 @@ extern int K_needed_node_on_f(struct node * p);
|
|
|
497
567
|
extern int K_needed_for_and(struct node * p);
|
|
498
568
|
extern int K_needed_for_or(struct node * p);
|
|
499
569
|
extern int repeat_restore(struct node * p);
|
|
570
|
+
extern bool amongvar_needed(struct node * p);
|
|
500
571
|
|
|
501
572
|
extern int just_return_on_fail(struct generator * g);
|
|
502
573
|
extern int tailcallable(struct generator * g, struct node * p);
|
|
@@ -3,13 +3,15 @@
|
|
|
3
3
|
#include <stdarg.h>
|
|
4
4
|
#include <stdio.h> /* for printf */
|
|
5
5
|
#include <stdlib.h> /* malloc, free */
|
|
6
|
-
#include <string.h> /*
|
|
6
|
+
#include <string.h> /* memcpy */
|
|
7
7
|
|
|
8
8
|
#include "header.h"
|
|
9
9
|
|
|
10
10
|
#define HEAD (2 * sizeof(int))
|
|
11
11
|
#define EXTENDER 40
|
|
12
12
|
|
|
13
|
+
// Threshold in bytes above which reallocations double the size.
|
|
14
|
+
#define EXPONENTIAL_GROWTH_THRESHOLD 512
|
|
13
15
|
|
|
14
16
|
/* This module provides a simple mechanism for arbitrary length writable
|
|
15
17
|
strings, called 'blocks'. They are 'symbol *' items rather than 'char *'
|
|
@@ -19,8 +21,8 @@
|
|
|
19
21
|
|
|
20
22
|
symbol * b = create_b(n);
|
|
21
23
|
- create an empty block b with room for n symbols
|
|
22
|
-
b =
|
|
23
|
-
-
|
|
24
|
+
b = reserve_b(b, n);
|
|
25
|
+
- ensure that the capacity of b is at least n (b may change)
|
|
24
26
|
b2 = copy_b(b)
|
|
25
27
|
- copy block b into b2
|
|
26
28
|
lose_b(b);
|
|
@@ -44,9 +46,9 @@
|
|
|
44
46
|
|
|
45
47
|
/* For a block b, SIZE(b) is the number of symbols so far written into it,
|
|
46
48
|
CAPACITY(b) the total number it can contain, so SIZE(b) <= CAPACITY(b).
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
49
|
+
|
|
50
|
+
NB In the compiler, blocks do NOT have 1 extra character over the promised
|
|
51
|
+
capacity - use ensure_nul_b() and ensure_nul_s() to zero terminate.
|
|
50
52
|
*/
|
|
51
53
|
|
|
52
54
|
extern symbol * create_b(int n) {
|
|
@@ -75,17 +77,35 @@ extern void lose_b(symbol * p) {
|
|
|
75
77
|
FREE((char *) p - HEAD);
|
|
76
78
|
}
|
|
77
79
|
|
|
78
|
-
extern symbol *
|
|
79
|
-
|
|
80
|
-
|
|
80
|
+
extern symbol * reserve_b_(symbol * p, int n) {
|
|
81
|
+
const int EXP_THRESHOLD = EXPONENTIAL_GROWTH_THRESHOLD / sizeof(symbol);
|
|
82
|
+
int new_size = n + EXTENDER;
|
|
83
|
+
if (new_size > EXP_THRESHOLD) {
|
|
84
|
+
// Switch to exponential growth for large strings.
|
|
85
|
+
int c = (CAPACITY(p) | (EXP_THRESHOLD * 2 - 1)) + 1;
|
|
86
|
+
while (c < new_size) c *= 2;
|
|
87
|
+
new_size = c;
|
|
88
|
+
}
|
|
89
|
+
symbol * q = create_b(new_size);
|
|
90
|
+
memcpy(q, p, SIZE(p) * sizeof(symbol));
|
|
81
91
|
SET_SIZE(q, SIZE(p));
|
|
82
|
-
lose_b(p);
|
|
92
|
+
lose_b(p);
|
|
93
|
+
return q;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
extern symbol * resize_b(symbol * p, int n) {
|
|
97
|
+
int size = SIZE(p);
|
|
98
|
+
if (n > size) {
|
|
99
|
+
p = reserve_b(p, n);
|
|
100
|
+
memset(p + size, 0, (n - size) * sizeof(symbol));
|
|
101
|
+
}
|
|
102
|
+
SET_SIZE(p, n);
|
|
103
|
+
return p;
|
|
83
104
|
}
|
|
84
105
|
|
|
85
106
|
extern symbol * add_to_b(symbol * p, const symbol * q, int n) {
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
memmove(p + SIZE(p), q, n * sizeof(symbol));
|
|
107
|
+
p = reserve_b(p, SIZE(p) + n);
|
|
108
|
+
memcpy(p + SIZE(p), q, n * sizeof(symbol));
|
|
89
109
|
ADD_TO_SIZE(p, n);
|
|
90
110
|
return p;
|
|
91
111
|
}
|
|
@@ -118,8 +138,8 @@ extern void check_free(void * p) {
|
|
|
118
138
|
free(p);
|
|
119
139
|
}
|
|
120
140
|
|
|
121
|
-
extern int checked_snprintf(char *str, size_t size,
|
|
122
|
-
const char *
|
|
141
|
+
extern int checked_snprintf(char * str, size_t size,
|
|
142
|
+
const char * format, ...) {
|
|
123
143
|
va_list ap;
|
|
124
144
|
va_start(ap, format);
|
|
125
145
|
int r = vsnprintf(str, size, format, ap);
|
|
@@ -153,11 +173,12 @@ extern char * b_to_sz(const symbol * p) {
|
|
|
153
173
|
|
|
154
174
|
/* Add a single symbol to a block. If p = 0 the block is created. */
|
|
155
175
|
extern symbol * add_symbol_to_b(symbol * p, symbol ch) {
|
|
156
|
-
if (p == NULL)
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
176
|
+
if (p == NULL) {
|
|
177
|
+
p = create_b(1);
|
|
178
|
+
} else {
|
|
179
|
+
p = reserve_b(p, SIZE(p) + 1);
|
|
180
|
+
}
|
|
181
|
+
p[SIZE(p)] = ch;
|
|
161
182
|
ADD_TO_SIZE(p, 1);
|
|
162
183
|
return p;
|
|
163
184
|
}
|
|
@@ -170,7 +191,7 @@ extern byte * create_s(int n) {
|
|
|
170
191
|
}
|
|
171
192
|
|
|
172
193
|
extern byte * create_s_from_sz(const char * s) {
|
|
173
|
-
int n = strlen(s);
|
|
194
|
+
int n = (int)strlen(s);
|
|
174
195
|
byte * p = create_s(n);
|
|
175
196
|
memcpy(p, s, n + 1);
|
|
176
197
|
SET_SIZE(p, n);
|
|
@@ -180,7 +201,6 @@ extern byte * create_s_from_sz(const char * s) {
|
|
|
180
201
|
extern byte * create_s_from_data(const char * s, int n) {
|
|
181
202
|
byte * p = create_s(n);
|
|
182
203
|
memcpy(p, s, n);
|
|
183
|
-
p[n] = '\0';
|
|
184
204
|
SET_SIZE(p, n);
|
|
185
205
|
return p;
|
|
186
206
|
}
|
|
@@ -194,20 +214,29 @@ extern void lose_s(byte * p) {
|
|
|
194
214
|
FREE((byte *) p - HEAD);
|
|
195
215
|
}
|
|
196
216
|
|
|
197
|
-
extern byte *
|
|
198
|
-
int
|
|
199
|
-
|
|
200
|
-
if (
|
|
201
|
-
|
|
202
|
-
|
|
217
|
+
extern byte * reserve_s_(byte * p, int n) {
|
|
218
|
+
const int EXP_THRESHOLD = EXPONENTIAL_GROWTH_THRESHOLD / sizeof(byte);
|
|
219
|
+
n += EXTENDER;
|
|
220
|
+
if (n > EXP_THRESHOLD) {
|
|
221
|
+
// Switch to exponential growth for large strings.
|
|
222
|
+
int c = (CAPACITY(p) | (EXP_THRESHOLD * 2 - 1)) + 1;
|
|
223
|
+
while (c < n) c *= 2;
|
|
224
|
+
n = c;
|
|
225
|
+
}
|
|
226
|
+
byte * q = create_s(n);
|
|
227
|
+
memcpy(q, p, SIZE(p));
|
|
203
228
|
SET_SIZE(q, SIZE(p));
|
|
204
229
|
lose_s(p);
|
|
205
230
|
return q;
|
|
206
231
|
}
|
|
207
232
|
|
|
208
|
-
extern byte *
|
|
209
|
-
int
|
|
210
|
-
if (
|
|
233
|
+
extern byte * resize_s(byte * p, int n) {
|
|
234
|
+
int size = SIZE(p);
|
|
235
|
+
if (n > size) {
|
|
236
|
+
p = reserve_s(p, n);
|
|
237
|
+
memset(p + size, 0, n - size);
|
|
238
|
+
}
|
|
239
|
+
SET_SIZE(p, n);
|
|
211
240
|
return p;
|
|
212
241
|
}
|
|
213
242
|
|
|
@@ -222,7 +251,7 @@ extern byte * add_slen_to_s(byte * p, const char * s, int n) {
|
|
|
222
251
|
if (p == NULL) {
|
|
223
252
|
p = create_s(n);
|
|
224
253
|
} else {
|
|
225
|
-
p =
|
|
254
|
+
p = reserve_s(p, SIZE(p) + n);
|
|
226
255
|
}
|
|
227
256
|
int k = SIZE(p);
|
|
228
257
|
memcpy(p + k, s, n);
|
|
@@ -241,7 +270,7 @@ extern byte * add_s_to_s(byte * p, const byte * s) {
|
|
|
241
270
|
block is created. */
|
|
242
271
|
|
|
243
272
|
extern byte * add_sz_to_s(byte * p, const char * s) {
|
|
244
|
-
return add_slen_to_s(p, s, strlen(s));
|
|
273
|
+
return add_slen_to_s(p, s, (int)strlen(s));
|
|
245
274
|
}
|
|
246
275
|
|
|
247
276
|
/* Add a single character to a byte block. If p = 0 the
|
|
@@ -251,7 +280,7 @@ extern byte * add_char_to_s(byte * p, char ch) {
|
|
|
251
280
|
if (p == NULL) {
|
|
252
281
|
p = create_s(1);
|
|
253
282
|
} else {
|
|
254
|
-
p =
|
|
283
|
+
p = reserve_s(p, SIZE(p) + 1);
|
|
255
284
|
}
|
|
256
285
|
p[SIZE(p)] = ch;
|
|
257
286
|
ADD_TO_SIZE(p, 1);
|
|
@@ -308,17 +337,17 @@ extern void str_append_int(struct str * str, int i) {
|
|
|
308
337
|
}
|
|
309
338
|
|
|
310
339
|
// Ensure there's enough space then snprintf() directly onto the end.
|
|
340
|
+
int size = SIZE(str->data);
|
|
311
341
|
int max_size = (CHAR_BIT * sizeof(int) + 5) / 3;
|
|
312
|
-
str->data =
|
|
313
|
-
int r = checked_snprintf((char*)str->data +
|
|
314
|
-
"%d", i);
|
|
342
|
+
str->data = reserve_s(str->data, size + max_size);
|
|
343
|
+
int r = checked_snprintf((char*)str->data + size, max_size, "%d", i);
|
|
315
344
|
ADD_TO_SIZE(str->data, r);
|
|
316
345
|
}
|
|
317
346
|
|
|
318
347
|
/* Append wide character to a string as UTF-8. */
|
|
319
348
|
extern void str_append_wchar_as_utf8(struct str * str, symbol ch) {
|
|
320
349
|
if (ch < 0x80) {
|
|
321
|
-
str_append_ch(str, ch);
|
|
350
|
+
str_append_ch(str, (char)ch);
|
|
322
351
|
return;
|
|
323
352
|
}
|
|
324
353
|
if (ch < 0x800) {
|