mittens 0.3.3 → 0.3.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +8 -0
  3. data/LICENSE.txt +1 -1
  4. data/README.md +1 -1
  5. data/ext/mittens/ext.c +68 -30
  6. data/ext/mittens/extconf.rb +2 -0
  7. data/lib/mittens/version.rb +1 -1
  8. data/vendor/snowball/.github/workflows/ci.yml +43 -3
  9. data/vendor/snowball/.github/workflows/coverage.yml +8 -5
  10. data/vendor/snowball/GNUmakefile +46 -16
  11. data/vendor/snowball/NEWS +437 -13
  12. data/vendor/snowball/README.rst +4 -4
  13. data/vendor/snowball/algorithms/earlymodernenglish.sbl +301 -0
  14. data/vendor/snowball/algorithms/french.sbl +1 -1
  15. data/vendor/snowball/algorithms/ukrainian.sbl +1251 -0
  16. data/vendor/snowball/compiler/analyser.c +460 -156
  17. data/vendor/snowball/compiler/driver.c +41 -36
  18. data/vendor/snowball/compiler/generator.c +24 -1
  19. data/vendor/snowball/compiler/generator_ada.c +46 -45
  20. data/vendor/snowball/compiler/generator_c.c +1094 -187
  21. data/vendor/snowball/compiler/generator_csharp.c +34 -11
  22. data/vendor/snowball/compiler/generator_dart.c +39 -11
  23. data/vendor/snowball/compiler/generator_go.c +6 -6
  24. data/vendor/snowball/compiler/generator_java.c +29 -8
  25. data/vendor/snowball/compiler/generator_js.c +65 -38
  26. data/vendor/snowball/compiler/generator_pascal.c +69 -26
  27. data/vendor/snowball/compiler/generator_php.c +33 -12
  28. data/vendor/snowball/compiler/generator_python.c +23 -4
  29. data/vendor/snowball/compiler/generator_rust.c +37 -13
  30. data/vendor/snowball/compiler/generator_zig.c +6 -6
  31. data/vendor/snowball/compiler/header.h +85 -14
  32. data/vendor/snowball/compiler/space.c +67 -38
  33. data/vendor/snowball/compiler/tokeniser.c +28 -16
  34. data/vendor/snowball/csharp/Snowball/AssemblyInfo.cs +2 -2
  35. data/vendor/snowball/csharp/Snowball/Stemmer.cs +1 -4
  36. data/vendor/snowball/cxx/generate_algorithms.pl +5 -4
  37. data/vendor/snowball/cxx/{utilities.cxx → snowball_runtime.cxx} +1 -1
  38. data/vendor/snowball/cxx/stemmer.h +4 -0
  39. data/vendor/snowball/cxx/stemwords.cxx +36 -6
  40. data/vendor/snowball/dart/lib/src/snowball.dart +2 -2
  41. data/vendor/snowball/dart/pubspec.yaml +1 -1
  42. data/vendor/snowball/doc/libstemmer_c_README +3 -0
  43. data/vendor/snowball/doc/libstemmer_php_README +1 -1
  44. data/vendor/snowball/doc/libstemmer_python_README +8 -6
  45. data/vendor/snowball/examples/stemwords.c +4 -14
  46. data/vendor/snowball/go/env.go +16 -4
  47. data/vendor/snowball/include/libstemmer.h +0 -3
  48. data/vendor/snowball/java/org/tartarus/snowball/SnowballProgram.java +4 -4
  49. data/vendor/snowball/javascript/base-stemmer.js +27 -40
  50. data/vendor/snowball/javascript/stemwords.js +34 -10
  51. data/vendor/snowball/libstemmer/libstemmer_c.in +5 -5
  52. data/vendor/snowball/libstemmer/mkmodules.pl +5 -3
  53. data/vendor/snowball/libstemmer/modules.txt +4 -2
  54. data/vendor/snowball/pascal/SnowballProgram.pas +2 -2
  55. data/vendor/snowball/php/base-stemmer.php +2 -2
  56. data/vendor/snowball/python/setup.py +5 -1
  57. data/vendor/snowball/python/snowballstemmer/basestemmer.py +1 -2
  58. data/vendor/snowball/runtime/api.c +14 -3
  59. data/vendor/snowball/runtime/api.h +13 -9
  60. data/vendor/snowball/runtime/{utilities.c → snowball_runtime.c} +115 -181
  61. data/vendor/snowball/runtime/snowball_runtime.h +35 -24
  62. data/vendor/snowball/rust/src/snowball/snowball_env.rs +1 -1
  63. data/vendor/snowball/tests/compilertest +3 -3
  64. data/vendor/snowball/tests/runtime/among.sbl +96 -2
  65. data/vendor/snowball/tests/runtime/attachinsert.sbl +4 -0
  66. data/vendor/snowball/tests/runtime/debug.out +7 -0
  67. data/vendor/snowball/tests/runtime/debug.sbl +9 -0
  68. data/vendor/snowball/tests/runtime/integertests.sbl +22 -0
  69. data/vendor/snowball/tests/runtime/marklimit.sbl +30 -0
  70. data/vendor/snowball/tests/runtime/naming.sbl +20 -0
  71. data/vendor/snowball/tests/runtime/reverse.sbl +11 -0
  72. data/vendor/snowball/tests/stemtest.c +2 -5
  73. data/vendor/snowball/tests/syntax/aeslicefrom.sbl +7 -0
  74. data/vendor/snowball/tests/syntax/canon.sbl +14 -0
  75. data/vendor/snowball/tests/syntax/canon.syntax +36 -0
  76. data/vendor/snowball/tests/syntax/inline.syntax +4 -4
  77. data/vendor/snowball/tests/syntax/localvar.sbl +38 -0
  78. data/vendor/snowball/tests/syntax/localvar.stderr +3 -0
  79. data/vendor/snowball/tests/syntax/localvar.syntax +76 -0
  80. data/vendor/snowball/tests/syntax/unused.sbl +5 -1
  81. data/vendor/snowball/tests/syntax/unused.stderr +3 -1
  82. data/vendor/snowball/zig/env.zig +24 -4
  83. metadata +14 -4
@@ -54,7 +54,7 @@ static void write_varref(struct generator * g, struct name * p) {
54
54
  write_varname(g, p);
55
55
  }
56
56
 
57
- static void write_literal_string(struct generator * g, symbol * p) {
57
+ static void write_literal_string(struct generator * g, const symbol * p) {
58
58
  write_char(g, '"');
59
59
  for (int i = 0; i < SIZE(p); i++) {
60
60
  int ch = p[i];
@@ -1201,7 +1201,7 @@ static void generate_debug(struct generator * g, struct node * p) {
1201
1201
  write_comment(g, p);
1202
1202
  g->I[0] = g->debug_count++;
1203
1203
  g->I[1] = p->line_number;
1204
- writef(g, "~Mself.debug(~I0, ~I1)~N", p);
1204
+ writef(g, "~Mself.__debug(~I0, ~I1)~N", p);
1205
1205
  }
1206
1206
 
1207
1207
  static void generate(struct generator * g, struct node * p) {
@@ -1314,7 +1314,7 @@ static void generate_among_table(struct generator * g, struct among * x) {
1314
1314
  write_newline(g);
1315
1315
  write_comment(g, x->node);
1316
1316
 
1317
- struct amongvec * v = x->b;
1317
+ struct amongvec * v = x->v;
1318
1318
 
1319
1319
  g->I[0] = x->number;
1320
1320
 
@@ -1348,7 +1348,7 @@ static void generate_among_table(struct generator * g, struct among * x) {
1348
1348
 
1349
1349
  static void generate_amongs(struct generator * g) {
1350
1350
  for (struct among * x = g->analyser->amongs; x; x = x->next) {
1351
- generate_among_table(g, x);
1351
+ if (!x->duplicate) generate_among_table(g, x);
1352
1352
  }
1353
1353
  }
1354
1354
 
@@ -1442,6 +1442,25 @@ extern void generate_program_python(struct generator * g) {
1442
1442
  generate_groupings(g);
1443
1443
 
1444
1444
  generate_members(g);
1445
+
1446
+ if (g->analyser->debug_used) {
1447
+ w(g, "~N"
1448
+ "~Mdef __debug(self, n, line):~N~+"
1449
+ "~Mlength = len(self.current)~N"
1450
+ "~Mprint(\"%3d (line %4d): [%d]'\" % (n, line, length), end='')~N"
1451
+ "~Mfor i in range(0, length + 1):~N~+"
1452
+ "~Mif self.limit_backward == i: print('{', end='')~N"
1453
+ "~Mif self.bra == i: print('[', end='')~N"
1454
+ "~Mif self.cursor == i: print('|', end='')~N"
1455
+ "~Mif self.ket == i: print(']', end='')~N"
1456
+ "~Mif self.limit == i: print('}', end='')~N"
1457
+ "~Mif i < length:~N~+"
1458
+ "~Mch = self.current[i]~N"
1459
+ "~Mprint(ch if ch != '\\0' else '#', end='')~N~-~-"
1460
+ "~Mprint(\"'\")~N~-"
1461
+ "~N");
1462
+ }
1463
+
1445
1464
  generate_methods(g);
1446
1465
 
1447
1466
  generate_amongs(g);
@@ -32,7 +32,7 @@ static void write_varref(struct generator * g, struct name * p) {
32
32
  write_varname(g, p);
33
33
  }
34
34
 
35
- static void write_literal_string(struct generator * g, symbol * p) {
35
+ static void write_literal_string(struct generator * g, const symbol * p) {
36
36
  int i = 0;
37
37
  write_char(g, '"');
38
38
  while (i < SIZE(p)) {
@@ -47,7 +47,7 @@ static void write_literal_string(struct generator * g, symbol * p) {
47
47
  write_wchar_as_utf8(g, ch);
48
48
  } else {
49
49
  write_string(g, "\\u{");
50
- write_hex4(g, ch);
50
+ write_hex(g, ch);
51
51
  write_string(g, "}");
52
52
  }
53
53
  }
@@ -702,10 +702,10 @@ static void generate_hop(struct generator * g, struct node * p) {
702
702
  // Generate the AE to a temporary block so we can substitute it in
703
703
  // write_failure_if().
704
704
  struct str * ae = str_new();
705
- struct str * s = g->outbuf;
705
+ struct str * saved_outbuf = g->outbuf;
706
706
  g->outbuf = ae;
707
707
  generate_AE(g, p->AE);
708
- g->outbuf = s;
708
+ g->outbuf = saved_outbuf;
709
709
  g->B[0] = str_data(ae);
710
710
  g->S[0] = p->mode == m_forward ? "" : "_back";
711
711
  g->S[1] = p->AE->type == c_number ? "" : "_checked";
@@ -1086,7 +1086,7 @@ static void generate_define(struct generator * g, struct node * p) {
1086
1086
  }
1087
1087
  }
1088
1088
 
1089
- if (q->amongvar_needed) {
1089
+ if (amongvar_needed(p->left)) {
1090
1090
  w(g, "~Mlet mut among_var;~N");
1091
1091
  }
1092
1092
 
@@ -1143,7 +1143,7 @@ static void generate_substring(struct generator * g, struct node * p) {
1143
1143
  struct among * x = p->among;
1144
1144
  int block = -1;
1145
1145
  unsigned int bitmap = 0;
1146
- struct amongvec * among_cases = x->b;
1146
+ struct amongvec * among_cases = x->v;
1147
1147
  int empty_case = -1;
1148
1148
  int n_cases = 0;
1149
1149
  symbol cases[2];
@@ -1340,7 +1340,7 @@ static void generate_debug(struct generator * g, struct node * p) {
1340
1340
  write_comment(g, p);
1341
1341
  g->I[0] = g->debug_count++;
1342
1342
  g->I[1] = p->line_number;
1343
- writef(g, "~Menv.debug(~I0, ~I1);~N", p);
1343
+ writef(g, "~Mdebug(env, ~I0, ~I1);~N", p);
1344
1344
  }
1345
1345
 
1346
1346
  static void generate(struct generator * g, struct node * p) {
@@ -1444,7 +1444,7 @@ static void generate_class_begin(struct generator * g) {
1444
1444
  static void generate_among_table(struct generator * g, struct among * x) {
1445
1445
  write_comment(g, x->node);
1446
1446
 
1447
- struct amongvec * v = x->b;
1447
+ struct amongvec * v = x->v;
1448
1448
 
1449
1449
  g->I[0] = x->number;
1450
1450
  g->I[1] = x->literalstring_count;
@@ -1470,12 +1470,12 @@ static void generate_among_table(struct generator * g, struct among * x) {
1470
1470
  }
1471
1471
 
1472
1472
  static void generate_amongs(struct generator * g) {
1473
- struct str * s = g->outbuf;
1473
+ struct str * saved_outbuf = g->outbuf;
1474
1474
  g->outbuf = g->declarations;
1475
1475
  for (struct among * x = g->analyser->amongs; x; x = x->next) {
1476
- if (x->used) generate_among_table(g, x);
1476
+ if (x->used && !x->duplicate) generate_among_table(g, x);
1477
1477
  }
1478
- g->outbuf = s;
1478
+ g->outbuf = saved_outbuf;
1479
1479
  }
1480
1480
 
1481
1481
  static void set_bit(symbol * b, int i) { b[i >> 3] |= 1 << (i & 7); }
@@ -1504,12 +1504,12 @@ static void generate_grouping_table(struct generator * g, struct grouping * q) {
1504
1504
  }
1505
1505
 
1506
1506
  static void generate_groupings(struct generator * g) {
1507
- struct str * s = g->outbuf;
1507
+ struct str * saved_outbuf = g->outbuf;
1508
1508
  g->outbuf = g->declarations;
1509
1509
  for (struct grouping * q = g->analyser->groupings; q; q = q->next) {
1510
1510
  generate_grouping_table(g, q);
1511
1511
  }
1512
- g->outbuf = s;
1512
+ g->outbuf = saved_outbuf;
1513
1513
  }
1514
1514
 
1515
1515
  static void generate_members(struct generator * g) {
@@ -1555,6 +1555,10 @@ extern void generate_program_rust(struct generator * g) {
1555
1555
  /* std::i32 is used in the code generated for i32::MAX and i32::MIN */
1556
1556
  w(g, "use std::i32;~N~N");
1557
1557
  }
1558
+ if (g->analyser->debug_used) {
1559
+ w(g, "use std::io;~N");
1560
+ w(g, "use std::io::Write;~N~N");
1561
+ }
1558
1562
  generate_class_begin(g);
1559
1563
 
1560
1564
  generate_members(g);
@@ -1562,6 +1566,26 @@ extern void generate_program_rust(struct generator * g) {
1562
1566
  g->declarations = g->outbuf;
1563
1567
  g->outbuf = str_new();
1564
1568
 
1569
+ if (g->analyser->debug_used) {
1570
+ w(g, "~Mfn debug(env: &mut SnowballEnv, n: i32, line: i32) {~+~N"
1571
+ "~Mlet length = env.current.len();~N"
1572
+ "~Mprint!(\"{:3} (line {:4}): [{}]'\", n, line, length);~N"
1573
+ "~Mlet mut b = env.current.bytes();~N"
1574
+ "~Mfor i in 0..length + 1 {~N~+"
1575
+ "~Mif env.limit_backward == i as i32 { print!(\"{{\") }~N"
1576
+ "~Mif env.bra == i as i32 { print!(\"[\") }~N"
1577
+ "~Mif env.cursor == i as i32 { print!(\"|\") }~N"
1578
+ "~Mif env.ket == i as i32 { print!(\"]\") }~N"
1579
+ "~Mif env.limit == i as i32 { print!(\"}}\") }~N"
1580
+ "~Mif (i < length) {~N~+"
1581
+ "~Mlet ch = b.next();~N"
1582
+ "~Mif ch == Some(0) { print!(\"#\"); } else { let _ = io::stdout().write(ch.as_slice()); }~N~-"
1583
+ "~M}~N~-"
1584
+ "~M}~N"
1585
+ "~Mprintln!(\"'\");~N~-"
1586
+ "}~N~N");
1587
+ }
1588
+
1565
1589
  generate_methods(g);
1566
1590
 
1567
1591
  generate_amongs(g);
@@ -38,7 +38,7 @@ static void write_stringref(struct generator * g, struct name * p) {
38
38
  write_string(g, ".slice()");
39
39
  }
40
40
 
41
- static void write_literal_string(struct generator * g, symbol * p) {
41
+ static void write_literal_string(struct generator * g, const symbol * p) {
42
42
  int i = 0;
43
43
  write_char(g, '"');
44
44
  while (i < SIZE(p)) {
@@ -718,10 +718,10 @@ static void generate_hop(struct generator * g, struct node * p) {
718
718
  // Generate the AE to a temporary block so we can substitute it in
719
719
  // write_failure_if().
720
720
  struct str * ae = str_new();
721
- struct str * s = g->outbuf;
721
+ struct str * saved_outbuf = g->outbuf;
722
722
  g->outbuf = ae;
723
723
  generate_AE(g, p->AE);
724
- g->outbuf = s;
724
+ g->outbuf = saved_outbuf;
725
725
  g->B[0] = str_data(ae);
726
726
  g->S[0] = p->mode == m_forward ? "" : "Back";
727
727
  g->S[1] = p->AE->type == c_number ? "" : "Checked";
@@ -1134,7 +1134,7 @@ static void generate_define(struct generator * g, struct node * p) {
1134
1134
  }
1135
1135
  }
1136
1136
 
1137
- if (q->amongvar_needed) {
1137
+ if (amongvar_needed(p->left)) {
1138
1138
  w(g, "~Mvar among_var: i32 = 0;~N");
1139
1139
  }
1140
1140
 
@@ -1364,7 +1364,7 @@ static void generate_class_begin(struct generator * g) {
1364
1364
  static void generate_among_table(struct generator * g, struct among * x) {
1365
1365
  write_comment(g, x->node);
1366
1366
 
1367
- struct amongvec * v = x->b;
1367
+ struct amongvec * v = x->v;
1368
1368
 
1369
1369
  g->I[0] = x->number;
1370
1370
  w(g, "~Mconst a_~I0 = [_]snowball.Among{~N~+");
@@ -1389,7 +1389,7 @@ static void generate_among_table(struct generator * g, struct among * x) {
1389
1389
 
1390
1390
  static void generate_amongs(struct generator * g) {
1391
1391
  for (struct among * x = g->analyser->amongs; x; x = x->next) {
1392
- generate_among_table(g, x);
1392
+ if (!x->duplicate) generate_among_table(g, x);
1393
1393
  }
1394
1394
  }
1395
1395
 
@@ -1,9 +1,13 @@
1
1
  #include <stdio.h>
2
2
  #if __STDC_VERSION__ < 202311l
3
3
  # include <stdbool.h>
4
+ #elif defined _MSC_VER && __STDC_VERSION__-0 <= 202312l
5
+ // MSVC 2026 with -std:clatest defines __STDC_VERSION__ to 202312 (one more
6
+ // than the correct value for C23!) but doesn't provide bool, true and false.
7
+ # include <stdbool.h>
4
8
  #endif
5
9
 
6
- #define SNOWBALL_VERSION "3.1.1"
10
+ #define SNOWBALL_VERSION "3.2.0"
7
11
 
8
12
  typedef unsigned char byte;
9
13
  typedef unsigned short symbol;
@@ -26,7 +30,17 @@ typedef unsigned short symbol;
26
30
  extern symbol * create_b(int n);
27
31
  extern void report_b(FILE * out, const symbol * p);
28
32
  extern void lose_b(symbol * p);
29
- extern symbol * increase_capacity_b(symbol * p, int n);
33
+ extern symbol * reserve_b_(symbol * p, int n);
34
+ static inline symbol * reserve_b(symbol * p, int n) {
35
+ if (n > CAPACITY(p)) p = reserve_b_(p, n);
36
+ return p;
37
+ }
38
+ extern symbol * resize_b(symbol * p, int n);
39
+ static inline symbol * ensure_nul_b(symbol * p) {
40
+ p = reserve_b(p, SIZE(p) + 1);
41
+ p[SIZE(p)] = 0;
42
+ return p;
43
+ }
30
44
  extern symbol * add_to_b(symbol * p, const symbol * q, int n);
31
45
  extern symbol * copy_b(const symbol * p);
32
46
  extern char * b_to_sz(const symbol * p);
@@ -40,8 +54,17 @@ extern byte * create_s_from_data(const char * s, int n);
40
54
 
41
55
  extern void report_s(FILE * out, const byte * p);
42
56
  extern void lose_s(byte * p);
43
- extern byte * increase_capacity_s(byte * p, int n);
44
- extern byte * ensure_capacity_s(byte * p, int n);
57
+ extern byte * reserve_s_(byte * p, int n);
58
+ static inline byte * reserve_s(byte * p, int n) {
59
+ if (n > CAPACITY(p)) p = reserve_s_(p, n);
60
+ return p;
61
+ }
62
+ extern byte * resize_s(byte * p, int n);
63
+ static inline byte * ensure_nul_s(byte * p) {
64
+ p = reserve_s(p, SIZE(p) + 1);
65
+ p[SIZE(p)] = 0;
66
+ return p;
67
+ }
45
68
  extern byte * copy_s(const byte * p);
46
69
  extern byte * add_s_to_s(byte * p, const byte * s);
47
70
  extern byte * add_slen_to_s(byte * p, const char * s, int n);
@@ -201,6 +224,7 @@ extern byte * get_input(const char * filename);
201
224
  extern struct tokeniser * create_tokeniser(byte * b, char * file);
202
225
  extern int read_token(struct tokeniser * t);
203
226
  extern int peek_token(struct tokeniser * t);
227
+ extern void push_token(struct tokeniser * t, int token);
204
228
  #define hold_token(T) ((T)->token_held = true)
205
229
  extern const char * name_of_token(int code);
206
230
  extern void disable_token(struct tokeniser * t, int code);
@@ -210,8 +234,8 @@ extern int space_count;
210
234
  extern void * check_malloc(size_t n);
211
235
  extern void check_free(void * p);
212
236
 
213
- extern int checked_snprintf(char *str, size_t size,
214
- const char *restrict format, ...)
237
+ extern int checked_snprintf(char * str, size_t size,
238
+ const char * format, ...)
215
239
  #ifdef __GNUC__
216
240
  __attribute__ ((__format__ (__printf__, 3, 4)))
217
241
  #endif
@@ -227,9 +251,15 @@ struct name {
227
251
  bool value_used; /* (For variables) is its value ever used? */
228
252
  bool initialised; /* (For variables) is it ever initialised? */
229
253
  bool used_in_definition; /* (grouping) used in grouping definition? */
230
- bool amongvar_needed; /* for routines, externals */
231
- bool among_with_function; /* (routines/externals) contains among with func */
254
+ // (routines/externals) contains among
255
+ bool has_among;
256
+ // (routines/externals) contains among with function(s)
257
+ bool has_among_function;
232
258
  bool case_collision; /* A name of the same type differs only by case */
259
+ // (routines/externals) Could this directly or indirectly call itself?
260
+ bool recursive;
261
+ // (routines/externals) Temporary flag used while determining `recursive`.
262
+ bool visited;
233
263
  struct node * definition; /* (routines/externals) c_define node */
234
264
  int uses_in_among; /* (routines/externals) Count of uses in amongs */
235
265
  // Initialised to -1; set to -2 if reachable from an external.
@@ -250,12 +280,19 @@ struct literalstring {
250
280
  symbol * b;
251
281
  };
252
282
 
283
+ struct c_literalstring {
284
+ struct c_literalstring * next;
285
+ const symbol * b;
286
+ };
287
+
253
288
  struct amongvec {
254
289
  symbol * b; /* the string giving the case */
255
290
  int size; /* - and its size */
256
291
  struct node * action; /* the corresponding action */
257
- int i; /* the amongvec index of the longest substring of b */
258
- int result; /* the numeric result for the case */
292
+ // The amongvec index of the longest substring of b, or -1 for none.
293
+ int i;
294
+ // among_var value for this case (starts from 1, or -1 for empty action).
295
+ int result;
259
296
  int line_number; /* for diagnostics */
260
297
  int function_index; /* 1-based */
261
298
  // 0-based index giving order of strings in source. Used for stable
@@ -264,24 +301,54 @@ struct amongvec {
264
301
  struct name * function;
265
302
  };
266
303
 
304
+ struct among_function_scenario {
305
+ struct name * function;
306
+ int t_result; // result (1...).
307
+ int f_result; // 0 or result (1...) or AFS_FLAG | af_index to chain.
308
+ // If `function` signals f, apply this delta to the value the cursor had on
309
+ // entry (add for forwards; subtract for backwards).
310
+ int cursor_adjustment;
311
+ };
312
+
313
+ // Used to find common-subtrees to reuse. We currently linear search this for
314
+ // each subtree we encode, but it seems in practice that's very fast, even for
315
+ // the large amongs in the stemmers we ship (the largest seems to be in greek.sbl
316
+ // which results in 1161 entries in this linked list).
317
+ //
318
+ // Note that if we had a separate list per value of len we would only need to
319
+ // store start in each item (and could store as a symbol*).
320
+ struct among_subtree {
321
+ struct among_subtree * next;
322
+ int start;
323
+ int len;
324
+ };
325
+
267
326
  struct among {
268
327
  struct among * next;
269
- struct amongvec * b; /* pointer to the amongvec */
328
+ struct amongvec * v; /* pointer to the amongvec */
329
+ // Details for among function handling in the state machine implementation.
330
+ struct among_function_scenario * af;
331
+ struct among_subtree * subtree;
332
+ int af_count; /* number of entries in af. */
333
+ symbol * table; /* table used in C implementation. */
334
+ byte * table_endianness; /* flag values needing byteswap on big-endian. */
270
335
  int number; /* amongs are numbered 0, 1, 2 ... */
271
336
  int literalstring_count; /* in this among */
272
337
  int command_count; /* in this among (excludes "no command" entries) */
273
338
  int nocommand_count; /* number of "no command" entries in this among */
274
- int function_count; /* number of different functions in this among */
339
+ int function_count; /* number of cases with a function in this among */
340
+ int unique_function_count;/* number of different functions in this among */
275
341
  bool amongvar_needed; /* do we need to set among_var? */
276
342
  bool always_matches; /* will this among always match? */
277
343
  bool used; /* is this among in reachable code? */
344
+ bool c0_used; /* Need c0 variable in C implementation? */
345
+ bool duplicate; /* Is this among a duplicate of another? */
278
346
  int same_action; /* type code if same for all actions; <0 otherwise */
279
347
  int shortest_size; /* smallest non-zero string length in this among */
280
348
  int longest_size; /* longest string length in this among */
281
349
  struct node * substring; /* i.e. substring ... among ( ... ) */
282
350
  struct node ** commands; /* array with command_count entries */
283
351
  struct node * node; /* pointer to the node for this among */
284
- struct name * in_routine; /* pointer to name for routine this among is in */
285
352
  };
286
353
 
287
354
  struct grouping {
@@ -297,7 +364,7 @@ struct node {
297
364
  struct node * next;
298
365
  struct node * left;
299
366
  struct node * aux; /* used in setlimit */
300
- struct among * among; /* used in among */
367
+ struct among * among; /* used in substring and among */
301
368
  struct node * right;
302
369
  byte type;
303
370
  byte mode;
@@ -409,6 +476,8 @@ struct generator {
409
476
  const char * varname_prefix;
410
477
  // String to indent by for each margin level (four spaces by default).
411
478
  const char * margin_indent;
479
+ // (C/C++) Linked list used to merge string literals.
480
+ struct c_literalstring * c_literalstrings;
412
481
  };
413
482
 
414
483
  /* Special values for failure_label in struct generator. */
@@ -480,6 +549,7 @@ extern void write_int(struct generator * g, int i);
480
549
  extern void wi3(struct generator * g, int i);
481
550
  extern void write_hex4(struct generator * g, unsigned ch);
482
551
  extern void write_hex(struct generator * g, unsigned i);
552
+ extern void write_octal3(struct generator * g, unsigned n);
483
553
  extern void write_symbol(struct generator * g, symbol s);
484
554
  extern void write_s(struct generator * g, const byte * b);
485
555
  extern void write_str(struct generator * g, struct str * str);
@@ -497,6 +567,7 @@ extern int K_needed_node_on_f(struct node * p);
497
567
  extern int K_needed_for_and(struct node * p);
498
568
  extern int K_needed_for_or(struct node * p);
499
569
  extern int repeat_restore(struct node * p);
570
+ extern bool amongvar_needed(struct node * p);
500
571
 
501
572
  extern int just_return_on_fail(struct generator * g);
502
573
  extern int tailcallable(struct generator * g, struct node * p);
@@ -3,13 +3,15 @@
3
3
  #include <stdarg.h>
4
4
  #include <stdio.h> /* for printf */
5
5
  #include <stdlib.h> /* malloc, free */
6
- #include <string.h> /* memmove */
6
+ #include <string.h> /* memcpy */
7
7
 
8
8
  #include "header.h"
9
9
 
10
10
  #define HEAD (2 * sizeof(int))
11
11
  #define EXTENDER 40
12
12
 
13
+ // Threshold in bytes above which reallocations double the size.
14
+ #define EXPONENTIAL_GROWTH_THRESHOLD 512
13
15
 
14
16
  /* This module provides a simple mechanism for arbitrary length writable
15
17
  strings, called 'blocks'. They are 'symbol *' items rather than 'char *'
@@ -19,8 +21,8 @@
19
21
 
20
22
  symbol * b = create_b(n);
21
23
  - create an empty block b with room for n symbols
22
- b = increase_capacity_b(b, n);
23
- - increase the capacity of block b by n symbols (b may change)
24
+ b = reserve_b(b, n);
25
+ - ensure that the capacity of b is at least n (b may change)
24
26
  b2 = copy_b(b)
25
27
  - copy block b into b2
26
28
  lose_b(b);
@@ -44,9 +46,9 @@
44
46
 
45
47
  /* For a block b, SIZE(b) is the number of symbols so far written into it,
46
48
  CAPACITY(b) the total number it can contain, so SIZE(b) <= CAPACITY(b).
47
- In fact blocks have 1 extra character over the promised capacity so
48
- they can be zero terminated by 'b[SIZE(b)] = 0;' without fear of
49
- overwriting.
49
+
50
+ NB In the compiler, blocks do NOT have 1 extra character over the promised
51
+ capacity - use ensure_nul_b() and ensure_nul_s() to zero terminate.
50
52
  */
51
53
 
52
54
  extern symbol * create_b(int n) {
@@ -75,17 +77,35 @@ extern void lose_b(symbol * p) {
75
77
  FREE((char *) p - HEAD);
76
78
  }
77
79
 
78
- extern symbol * increase_capacity_b(symbol * p, int n) {
79
- symbol * q = create_b(CAPACITY(p) + n + EXTENDER);
80
- memmove(q, p, CAPACITY(p) * sizeof(symbol));
80
+ extern symbol * reserve_b_(symbol * p, int n) {
81
+ const int EXP_THRESHOLD = EXPONENTIAL_GROWTH_THRESHOLD / sizeof(symbol);
82
+ int new_size = n + EXTENDER;
83
+ if (new_size > EXP_THRESHOLD) {
84
+ // Switch to exponential growth for large strings.
85
+ int c = (CAPACITY(p) | (EXP_THRESHOLD * 2 - 1)) + 1;
86
+ while (c < new_size) c *= 2;
87
+ new_size = c;
88
+ }
89
+ symbol * q = create_b(new_size);
90
+ memcpy(q, p, SIZE(p) * sizeof(symbol));
81
91
  SET_SIZE(q, SIZE(p));
82
- lose_b(p); return q;
92
+ lose_b(p);
93
+ return q;
94
+ }
95
+
96
+ extern symbol * resize_b(symbol * p, int n) {
97
+ int size = SIZE(p);
98
+ if (n > size) {
99
+ p = reserve_b(p, n);
100
+ memset(p + size, 0, (n - size) * sizeof(symbol));
101
+ }
102
+ SET_SIZE(p, n);
103
+ return p;
83
104
  }
84
105
 
85
106
  extern symbol * add_to_b(symbol * p, const symbol * q, int n) {
86
- int x = SIZE(p) + n - CAPACITY(p);
87
- if (x > 0) p = increase_capacity_b(p, x);
88
- memmove(p + SIZE(p), q, n * sizeof(symbol));
107
+ p = reserve_b(p, SIZE(p) + n);
108
+ memcpy(p + SIZE(p), q, n * sizeof(symbol));
89
109
  ADD_TO_SIZE(p, n);
90
110
  return p;
91
111
  }
@@ -118,8 +138,8 @@ extern void check_free(void * p) {
118
138
  free(p);
119
139
  }
120
140
 
121
- extern int checked_snprintf(char *str, size_t size,
122
- const char *restrict format, ...) {
141
+ extern int checked_snprintf(char * str, size_t size,
142
+ const char * format, ...) {
123
143
  va_list ap;
124
144
  va_start(ap, format);
125
145
  int r = vsnprintf(str, size, format, ap);
@@ -153,11 +173,12 @@ extern char * b_to_sz(const symbol * p) {
153
173
 
154
174
  /* Add a single symbol to a block. If p = 0 the block is created. */
155
175
  extern symbol * add_symbol_to_b(symbol * p, symbol ch) {
156
- if (p == NULL) p = create_b(1);
157
- int k = SIZE(p);
158
- int x = k + 1 - CAPACITY(p);
159
- if (x > 0) p = increase_capacity_b(p, x);
160
- p[k] = ch;
176
+ if (p == NULL) {
177
+ p = create_b(1);
178
+ } else {
179
+ p = reserve_b(p, SIZE(p) + 1);
180
+ }
181
+ p[SIZE(p)] = ch;
161
182
  ADD_TO_SIZE(p, 1);
162
183
  return p;
163
184
  }
@@ -170,7 +191,7 @@ extern byte * create_s(int n) {
170
191
  }
171
192
 
172
193
  extern byte * create_s_from_sz(const char * s) {
173
- int n = strlen(s);
194
+ int n = (int)strlen(s);
174
195
  byte * p = create_s(n);
175
196
  memcpy(p, s, n + 1);
176
197
  SET_SIZE(p, n);
@@ -180,7 +201,6 @@ extern byte * create_s_from_sz(const char * s) {
180
201
  extern byte * create_s_from_data(const char * s, int n) {
181
202
  byte * p = create_s(n);
182
203
  memcpy(p, s, n);
183
- p[n] = '\0';
184
204
  SET_SIZE(p, n);
185
205
  return p;
186
206
  }
@@ -194,20 +214,29 @@ extern void lose_s(byte * p) {
194
214
  FREE((byte *) p - HEAD);
195
215
  }
196
216
 
197
- extern byte * increase_capacity_s(byte * p, int n) {
198
- int new_size = CAPACITY(p) + n + EXTENDER;
199
- // Switch to exponential growth for large strings.
200
- if (new_size > 512) new_size *= 2;
201
- byte * q = create_s(new_size);
202
- memmove(q, p, CAPACITY(p));
217
+ extern byte * reserve_s_(byte * p, int n) {
218
+ const int EXP_THRESHOLD = EXPONENTIAL_GROWTH_THRESHOLD / sizeof(byte);
219
+ n += EXTENDER;
220
+ if (n > EXP_THRESHOLD) {
221
+ // Switch to exponential growth for large strings.
222
+ int c = (CAPACITY(p) | (EXP_THRESHOLD * 2 - 1)) + 1;
223
+ while (c < n) c *= 2;
224
+ n = c;
225
+ }
226
+ byte * q = create_s(n);
227
+ memcpy(q, p, SIZE(p));
203
228
  SET_SIZE(q, SIZE(p));
204
229
  lose_s(p);
205
230
  return q;
206
231
  }
207
232
 
208
- extern byte * ensure_capacity_s(byte * p, int n) {
209
- int x = SIZE(p) + n - CAPACITY(p);
210
- if (x > 0) p = increase_capacity_s(p, x);
233
+ extern byte * resize_s(byte * p, int n) {
234
+ int size = SIZE(p);
235
+ if (n > size) {
236
+ p = reserve_s(p, n);
237
+ memset(p + size, 0, n - size);
238
+ }
239
+ SET_SIZE(p, n);
211
240
  return p;
212
241
  }
213
242
 
@@ -222,7 +251,7 @@ extern byte * add_slen_to_s(byte * p, const char * s, int n) {
222
251
  if (p == NULL) {
223
252
  p = create_s(n);
224
253
  } else {
225
- p = ensure_capacity_s(p, n);
254
+ p = reserve_s(p, SIZE(p) + n);
226
255
  }
227
256
  int k = SIZE(p);
228
257
  memcpy(p + k, s, n);
@@ -241,7 +270,7 @@ extern byte * add_s_to_s(byte * p, const byte * s) {
241
270
  block is created. */
242
271
 
243
272
  extern byte * add_sz_to_s(byte * p, const char * s) {
244
- return add_slen_to_s(p, s, strlen(s));
273
+ return add_slen_to_s(p, s, (int)strlen(s));
245
274
  }
246
275
 
247
276
  /* Add a single character to a byte block. If p = 0 the
@@ -251,7 +280,7 @@ extern byte * add_char_to_s(byte * p, char ch) {
251
280
  if (p == NULL) {
252
281
  p = create_s(1);
253
282
  } else {
254
- p = ensure_capacity_s(p, 1);
283
+ p = reserve_s(p, SIZE(p) + 1);
255
284
  }
256
285
  p[SIZE(p)] = ch;
257
286
  ADD_TO_SIZE(p, 1);
@@ -308,17 +337,17 @@ extern void str_append_int(struct str * str, int i) {
308
337
  }
309
338
 
310
339
  // Ensure there's enough space then snprintf() directly onto the end.
340
+ int size = SIZE(str->data);
311
341
  int max_size = (CHAR_BIT * sizeof(int) + 5) / 3;
312
- str->data = ensure_capacity_s(str->data, max_size);
313
- int r = checked_snprintf((char*)str->data + SIZE(str->data), max_size,
314
- "%d", i);
342
+ str->data = reserve_s(str->data, size + max_size);
343
+ int r = checked_snprintf((char*)str->data + size, max_size, "%d", i);
315
344
  ADD_TO_SIZE(str->data, r);
316
345
  }
317
346
 
318
347
  /* Append wide character to a string as UTF-8. */
319
348
  extern void str_append_wchar_as_utf8(struct str * str, symbol ch) {
320
349
  if (ch < 0x80) {
321
- str_append_ch(str, ch);
350
+ str_append_ch(str, (char)ch);
322
351
  return;
323
352
  }
324
353
  if (ch < 0x800) {