mittens 0.3.3 → 0.3.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +8 -0
  3. data/LICENSE.txt +1 -1
  4. data/README.md +1 -1
  5. data/ext/mittens/ext.c +68 -30
  6. data/ext/mittens/extconf.rb +2 -0
  7. data/lib/mittens/version.rb +1 -1
  8. data/vendor/snowball/.github/workflows/ci.yml +43 -3
  9. data/vendor/snowball/.github/workflows/coverage.yml +8 -5
  10. data/vendor/snowball/GNUmakefile +46 -16
  11. data/vendor/snowball/NEWS +437 -13
  12. data/vendor/snowball/README.rst +4 -4
  13. data/vendor/snowball/algorithms/earlymodernenglish.sbl +301 -0
  14. data/vendor/snowball/algorithms/french.sbl +1 -1
  15. data/vendor/snowball/algorithms/ukrainian.sbl +1251 -0
  16. data/vendor/snowball/compiler/analyser.c +460 -156
  17. data/vendor/snowball/compiler/driver.c +41 -36
  18. data/vendor/snowball/compiler/generator.c +24 -1
  19. data/vendor/snowball/compiler/generator_ada.c +46 -45
  20. data/vendor/snowball/compiler/generator_c.c +1094 -187
  21. data/vendor/snowball/compiler/generator_csharp.c +34 -11
  22. data/vendor/snowball/compiler/generator_dart.c +39 -11
  23. data/vendor/snowball/compiler/generator_go.c +6 -6
  24. data/vendor/snowball/compiler/generator_java.c +29 -8
  25. data/vendor/snowball/compiler/generator_js.c +65 -38
  26. data/vendor/snowball/compiler/generator_pascal.c +69 -26
  27. data/vendor/snowball/compiler/generator_php.c +33 -12
  28. data/vendor/snowball/compiler/generator_python.c +23 -4
  29. data/vendor/snowball/compiler/generator_rust.c +37 -13
  30. data/vendor/snowball/compiler/generator_zig.c +6 -6
  31. data/vendor/snowball/compiler/header.h +85 -14
  32. data/vendor/snowball/compiler/space.c +67 -38
  33. data/vendor/snowball/compiler/tokeniser.c +28 -16
  34. data/vendor/snowball/csharp/Snowball/AssemblyInfo.cs +2 -2
  35. data/vendor/snowball/csharp/Snowball/Stemmer.cs +1 -4
  36. data/vendor/snowball/cxx/generate_algorithms.pl +5 -4
  37. data/vendor/snowball/cxx/{utilities.cxx → snowball_runtime.cxx} +1 -1
  38. data/vendor/snowball/cxx/stemmer.h +4 -0
  39. data/vendor/snowball/cxx/stemwords.cxx +36 -6
  40. data/vendor/snowball/dart/lib/src/snowball.dart +2 -2
  41. data/vendor/snowball/dart/pubspec.yaml +1 -1
  42. data/vendor/snowball/doc/libstemmer_c_README +3 -0
  43. data/vendor/snowball/doc/libstemmer_php_README +1 -1
  44. data/vendor/snowball/doc/libstemmer_python_README +8 -6
  45. data/vendor/snowball/examples/stemwords.c +4 -14
  46. data/vendor/snowball/go/env.go +16 -4
  47. data/vendor/snowball/include/libstemmer.h +0 -3
  48. data/vendor/snowball/java/org/tartarus/snowball/SnowballProgram.java +4 -4
  49. data/vendor/snowball/javascript/base-stemmer.js +27 -40
  50. data/vendor/snowball/javascript/stemwords.js +34 -10
  51. data/vendor/snowball/libstemmer/libstemmer_c.in +5 -5
  52. data/vendor/snowball/libstemmer/mkmodules.pl +5 -3
  53. data/vendor/snowball/libstemmer/modules.txt +4 -2
  54. data/vendor/snowball/pascal/SnowballProgram.pas +2 -2
  55. data/vendor/snowball/php/base-stemmer.php +2 -2
  56. data/vendor/snowball/python/setup.py +5 -1
  57. data/vendor/snowball/python/snowballstemmer/basestemmer.py +1 -2
  58. data/vendor/snowball/runtime/api.c +14 -3
  59. data/vendor/snowball/runtime/api.h +13 -9
  60. data/vendor/snowball/runtime/{utilities.c → snowball_runtime.c} +115 -181
  61. data/vendor/snowball/runtime/snowball_runtime.h +35 -24
  62. data/vendor/snowball/rust/src/snowball/snowball_env.rs +1 -1
  63. data/vendor/snowball/tests/compilertest +3 -3
  64. data/vendor/snowball/tests/runtime/among.sbl +96 -2
  65. data/vendor/snowball/tests/runtime/attachinsert.sbl +4 -0
  66. data/vendor/snowball/tests/runtime/debug.out +7 -0
  67. data/vendor/snowball/tests/runtime/debug.sbl +9 -0
  68. data/vendor/snowball/tests/runtime/integertests.sbl +22 -0
  69. data/vendor/snowball/tests/runtime/marklimit.sbl +30 -0
  70. data/vendor/snowball/tests/runtime/naming.sbl +20 -0
  71. data/vendor/snowball/tests/runtime/reverse.sbl +11 -0
  72. data/vendor/snowball/tests/stemtest.c +2 -5
  73. data/vendor/snowball/tests/syntax/aeslicefrom.sbl +7 -0
  74. data/vendor/snowball/tests/syntax/canon.sbl +14 -0
  75. data/vendor/snowball/tests/syntax/canon.syntax +36 -0
  76. data/vendor/snowball/tests/syntax/inline.syntax +4 -4
  77. data/vendor/snowball/tests/syntax/localvar.sbl +38 -0
  78. data/vendor/snowball/tests/syntax/localvar.stderr +3 -0
  79. data/vendor/snowball/tests/syntax/localvar.syntax +76 -0
  80. data/vendor/snowball/tests/syntax/unused.sbl +5 -1
  81. data/vendor/snowball/tests/syntax/unused.stderr +3 -1
  82. data/vendor/snowball/zig/env.zig +24 -4
  83. metadata +14 -4
@@ -36,7 +36,7 @@ static void write_varref(struct generator * g, struct name * p) {
36
36
  write_varname(g, p);
37
37
  }
38
38
 
39
- static void write_literal_string(struct generator * g, symbol * p) {
39
+ static void write_literal_string(struct generator * g, const symbol * p) {
40
40
  write_char(g, '"');
41
41
  for (int i = 0; i < SIZE(p); i++) {
42
42
  int ch = p[i];
@@ -194,21 +194,21 @@ static void writef(struct generator * g, const char * input, struct node * p) {
194
194
  }
195
195
  case 'F': { // Among function dispatcher.
196
196
  struct among * x = p->among;
197
- if (x->function_count == 0) {
197
+ if (x->unique_function_count == 0) {
198
198
  write_string(g, "null");
199
199
  continue;
200
200
  }
201
201
 
202
- if (x->function_count == 1) {
202
+ if (x->unique_function_count == 1) {
203
203
  // Only one different function used in this among.
204
- struct amongvec * v = x->b;
204
+ struct amongvec * v = x->v;
205
205
  for (int j = 0; j < x->literalstring_count; j++) {
206
206
  if (v[j].function) {
207
207
  write_varref(g, v[j].function);
208
208
  goto continue_outer_loop;
209
209
  }
210
210
  }
211
- fprintf(stderr, "function_count == 1 but no among functions\n");
211
+ fprintf(stderr, "unique_function_count == 1 but no among functions\n");
212
212
  exit(1);
213
213
  continue_outer_loop:
214
214
  continue;
@@ -1069,7 +1069,7 @@ static void generate_define(struct generator * g, struct node * p) {
1069
1069
  g->next_label = 0;
1070
1070
  g->var_number = 0;
1071
1071
 
1072
- if (q->amongvar_needed) {
1072
+ if (amongvar_needed(p->left)) {
1073
1073
  w(g, "~Mint among_var;~N");
1074
1074
  }
1075
1075
 
@@ -1209,7 +1209,7 @@ static void generate_debug(struct generator * g, struct node * p) {
1209
1209
  write_comment(g, p);
1210
1210
  g->I[0] = g->debug_count++;
1211
1211
  g->I[1] = p->line_number;
1212
- writef(g, "~Mdebug(~I0, ~I1);~N", p);
1212
+ writef(g, "~M_debug(~I0, ~I1);~N", p);
1213
1213
  }
1214
1214
 
1215
1215
  static void generate(struct generator * g, struct node * p) {
@@ -1329,7 +1329,7 @@ static void generate_among_table(struct generator * g, struct among * x) {
1329
1329
  write_newline(g);
1330
1330
  write_comment(g, x->node);
1331
1331
 
1332
- struct amongvec * v = x->b;
1332
+ struct amongvec * v = x->v;
1333
1333
 
1334
1334
  g->I[0] = x->number;
1335
1335
  w(g, "~Mprivate static readonly Among[] a_~I0 = new[] ~N~M{~N~+");
@@ -1345,11 +1345,11 @@ static void generate_among_table(struct generator * g, struct among * x) {
1345
1345
  }
1346
1346
  w(g, "~N~-~M};~N");
1347
1347
 
1348
- if (x->function_count <= 1) return;
1348
+ if (x->unique_function_count <= 1) return;
1349
1349
 
1350
1350
  w(g, "~N~Mprivate bool af_~I0() {~N~+");
1351
1351
  w(g, "~Mswitch (af) {~N~+");
1352
- for (int n = 1; n <= x->function_count; n++) {
1352
+ for (int n = 1; n <= x->unique_function_count; n++) {
1353
1353
  w(g, "~Mcase ");
1354
1354
  write_int(g, n);
1355
1355
  w(g, ": return ");
@@ -1368,7 +1368,7 @@ static void generate_among_table(struct generator * g, struct among * x) {
1368
1368
 
1369
1369
  static void generate_amongs(struct generator * g) {
1370
1370
  for (struct among * x = g->analyser->amongs; x; x = x->next) {
1371
- generate_among_table(g, x);
1371
+ if (!x->duplicate) generate_among_table(g, x);
1372
1372
  }
1373
1373
  w(g, "~N");
1374
1374
  }
@@ -1440,6 +1440,29 @@ extern void generate_program_csharp(struct generator * g) {
1440
1440
  generate_members(g);
1441
1441
  generate_groupings(g);
1442
1442
  generate_amongs(g);
1443
+
1444
+ if (g->analyser->debug_used) {
1445
+ w(g, "~N"
1446
+ "~Mprivate void _debug(int n, int line)~N"
1447
+ "~M{~N~+"
1448
+ "~MConsole.Write(\"{0,3} (line {1,4}): [{2}]'\", n, line, current.Length);~N"
1449
+ "~Mfor (int i = 0; i <= current.Length; ++i)~N"
1450
+ "~M{~N~+"
1451
+ "~Mif (limit_backward == i) Console.Write('{');~N"
1452
+ "~Mif (bra == i) Console.Write('[');~N"
1453
+ "~Mif (cursor == i) Console.Write('|');~N"
1454
+ "~Mif (ket == i) Console.Write(']');~N"
1455
+ "~Mif (limit == i) Console.Write('}');~N"
1456
+ "~Mif (i < current.Length)~N"
1457
+ "~M{~N~+"
1458
+ "~Mchar ch = current[i];~N"
1459
+ "~Mif (ch == '\\0') Console.Write('#'); else Console.Write(ch);~N~-"
1460
+ "~M}~N~-"
1461
+ "~M}~N"
1462
+ "~MConsole.WriteLine(\"'\");~N~-"
1463
+ "~M}~N");
1464
+ }
1465
+
1443
1466
  generate_methods(g);
1444
1467
 
1445
1468
  generate_class_end(g);
@@ -36,7 +36,7 @@ static void write_varref(struct generator * g, struct name * p) {
36
36
  write_varname(g, p);
37
37
  }
38
38
 
39
- static void write_literal_string(struct generator * g, symbol * p) {
39
+ static void write_literal_string(struct generator * g, const symbol * p) {
40
40
  write_char(g, '"');
41
41
  for (int i = 0; i < SIZE(p); i++) {
42
42
  int ch = p[i];
@@ -205,12 +205,12 @@ static void writef(struct generator * g, const char * input, struct node * p) {
205
205
  write_string(g, "null");
206
206
  continue;
207
207
  }
208
- if (x->function_count == 0) {
208
+ if (x->unique_function_count == 0) {
209
209
  write_string(g, "null");
210
210
  continue;
211
211
  }
212
- if (x->function_count == 1) {
213
- struct amongvec * v = x->b;
212
+ if (x->unique_function_count == 1) {
213
+ struct amongvec * v = x->v;
214
214
  int found = 0;
215
215
  for (int j = 0; j < x->literalstring_count; j++) {
216
216
  if (v[j].function) {
@@ -220,7 +220,7 @@ static void writef(struct generator * g, const char * input, struct node * p) {
220
220
  }
221
221
  }
222
222
  if (!found) {
223
- fprintf(stderr, "function_count == 1 but no among functions\n");
223
+ fprintf(stderr, "unique_function_count == 1 but no among functions\n");
224
224
  exit(1);
225
225
  }
226
226
  continue;
@@ -1102,7 +1102,7 @@ static void generate_define(struct generator * g, struct node * p) {
1102
1102
  g->next_label = 0;
1103
1103
  g->var_number = 0;
1104
1104
 
1105
- if (q->amongvar_needed) {
1105
+ if (amongvar_needed(p->left)) {
1106
1106
  w(g, "~Mint among_var;~N");
1107
1107
  }
1108
1108
 
@@ -1228,7 +1228,7 @@ static void generate_debug(struct generator * g, struct node * p) {
1228
1228
  write_comment(g, p);
1229
1229
  g->I[0] = g->debug_count++;
1230
1230
  g->I[1] = p->line_number;
1231
- writef(g, "~Mdebug(~I0, ~I1);~N", p);
1231
+ writef(g, "~M_debug(~I0, ~I1);~N", p);
1232
1232
  }
1233
1233
 
1234
1234
  static void generate(struct generator * g, struct node * p) {
@@ -1344,7 +1344,7 @@ static void generate_equals(struct generator * g) {
1344
1344
  static void generate_among_table(struct generator * g, struct among * x) {
1345
1345
  write_comment(g, x->node);
1346
1346
 
1347
- struct amongvec * v = x->b;
1347
+ struct amongvec * v = x->v;
1348
1348
 
1349
1349
  g->I[0] = x->number;
1350
1350
 
@@ -1361,11 +1361,11 @@ static void generate_among_table(struct generator * g, struct among * x) {
1361
1361
  }
1362
1362
  w(g, "~N~-~M];~N~N");
1363
1363
 
1364
- if (x->function_count <= 1) return;
1364
+ if (x->unique_function_count <= 1) return;
1365
1365
 
1366
1366
  w(g, "~N~Mbool af_~I0() {~N~+");
1367
1367
  w(g, "~Mswitch (af) {~N~+");
1368
- for (int n = 1; n <= x->function_count; n++) {
1368
+ for (int n = 1; n <= x->unique_function_count; n++) {
1369
1369
  w(g, "~Mcase ");
1370
1370
  write_int(g, n);
1371
1371
  w(g, ": return ");
@@ -1384,7 +1384,7 @@ static void generate_among_table(struct generator * g, struct among * x) {
1384
1384
 
1385
1385
  static void generate_amongs(struct generator * g) {
1386
1386
  for (struct among * x = g->analyser->amongs; x; x = x->next) {
1387
- generate_among_table(g, x);
1387
+ if (!x->duplicate) generate_among_table(g, x);
1388
1388
  }
1389
1389
  }
1390
1390
 
@@ -1465,6 +1465,34 @@ extern void generate_program_dart(struct generator * g) {
1465
1465
  generate_groupings(g);
1466
1466
 
1467
1467
  generate_members(g);
1468
+
1469
+ if (g->analyser->debug_used) {
1470
+ w(g, "~N"
1471
+ "~Mvoid _debug(int n, int line) {~N~+"
1472
+ "~MString s = '';~N"
1473
+ "~Mfinal len = current.length;~N"
1474
+ "~Mif (n < 10) s += ' ';~N"
1475
+ "~Mif (n < 100) s += ' ';~N"
1476
+ "~Ms += \"$n (line \";~N"
1477
+ "~Mif (line < 10) s += ' ';~N"
1478
+ "~Mif (line < 100) s += ' ';~N"
1479
+ "~Mif (line < 1000) s += ' ';~N"
1480
+ "~Ms += \"$line): [$len]'\";~N"
1481
+ "~Mfor (int i = 0; i <= len; ++i) {~N~+"
1482
+ "~Mif (this.limit_backward == i) s = s + '{';~N"
1483
+ "~Mif (this.bra == i) s = s + '[';~N"
1484
+ "~Mif (this.cursor == i) s = s + '|';~N"
1485
+ "~Mif (this.ket == i) s = s + ']';~N"
1486
+ "~Mif (this.limit == i) s = s + '}';~N"
1487
+ "~Mif (i < len) {~N~+"
1488
+ "~Mfinal ch = current.codeUnitAt(i);~N"
1489
+ "~Mif (ch == 0) s = s + '#'; else s = s + String.fromCharCode(ch);~N~-"
1490
+ "~M}~N~-"
1491
+ "~M}~N"
1492
+ "~Mprint(s + \"'\");~N~-"
1493
+ "~M}~N");
1494
+ }
1495
+
1468
1496
  generate_methods(g);
1469
1497
  generate_equals(g);
1470
1498
 
@@ -33,7 +33,7 @@ static void write_varref(struct generator * g, struct name * p) {
33
33
  write_varname(g, p);
34
34
  }
35
35
 
36
- static void write_literal_string(struct generator * g, symbol * p) {
36
+ static void write_literal_string(struct generator * g, const symbol * p) {
37
37
  int i = 0;
38
38
  write_char(g, '"');
39
39
  while (i < SIZE(p)) {
@@ -706,10 +706,10 @@ static void generate_hop(struct generator * g, struct node * p) {
706
706
  // Generate the AE to a temporary block so we can substitute it in
707
707
  // write_failure_if().
708
708
  struct str * ae = str_new();
709
- struct str * s = g->outbuf;
709
+ struct str * saved_outbuf = g->outbuf;
710
710
  g->outbuf = ae;
711
711
  generate_AE(g, p->AE);
712
- g->outbuf = s;
712
+ g->outbuf = saved_outbuf;
713
713
  g->B[0] = str_data(ae);
714
714
  g->S[0] = p->mode == m_forward ? "" : "Back";
715
715
  g->S[1] = p->AE->type == c_number ? "" : "Checked";
@@ -1090,7 +1090,7 @@ static void generate_define(struct generator * g, struct node * p) {
1090
1090
  }
1091
1091
  }
1092
1092
 
1093
- if (q->amongvar_needed) {
1093
+ if (amongvar_needed(p->left)) {
1094
1094
  w(g, "~Mvar among_var int32~N");
1095
1095
  }
1096
1096
 
@@ -1313,7 +1313,7 @@ static void generate_class_begin(struct generator * g) {
1313
1313
  static void generate_among_table(struct generator * g, struct among * x) {
1314
1314
  write_comment(g, x->node);
1315
1315
 
1316
- struct amongvec * v = x->b;
1316
+ struct amongvec * v = x->v;
1317
1317
 
1318
1318
  g->I[0] = x->number;
1319
1319
  w(g, "~Mvar A_~I0 = []*snowballRuntime.Among{~N~+");
@@ -1338,7 +1338,7 @@ static void generate_among_table(struct generator * g, struct among * x) {
1338
1338
 
1339
1339
  static void generate_amongs(struct generator * g) {
1340
1340
  for (struct among * x = g->analyser->amongs; x; x = x->next) {
1341
- generate_among_table(g, x);
1341
+ if (!x->duplicate) generate_among_table(g, x);
1342
1342
  }
1343
1343
  }
1344
1344
 
@@ -36,7 +36,7 @@ static void write_varref(struct generator * g, struct name * p) {
36
36
  write_varname(g, p);
37
37
  }
38
38
 
39
- static void write_literal_string(struct generator * g, symbol * p) {
39
+ static void write_literal_string(struct generator * g, const symbol * p) {
40
40
  write_char(g, '"');
41
41
  for (int i = 0; i < SIZE(p); i++) {
42
42
  int ch = p[i];
@@ -46,9 +46,7 @@ static void write_literal_string(struct generator * g, symbol * p) {
46
46
  } else if (ch < 128) {
47
47
  // Escape as octal.
48
48
  write_char(g, '\\');
49
- write_char(g, '0' + ((ch >> 6) & 0x03));
50
- write_char(g, '0' + ((ch >> 3) & 0x07));
51
- write_char(g, '0' + (ch & 0x07));
49
+ write_octal3(g, ch);
52
50
  } else {
53
51
  write_string(g, "\\u");
54
52
  write_hex4(g, ch);
@@ -1076,7 +1074,7 @@ static void generate_define(struct generator * g, struct node * p) {
1076
1074
  g->next_label = 0;
1077
1075
  g->var_number = 0;
1078
1076
 
1079
- if (q->amongvar_needed) {
1077
+ if (amongvar_needed(p->left)) {
1080
1078
  w(g, "~Mint among_var;~N");
1081
1079
  }
1082
1080
 
@@ -1200,7 +1198,7 @@ static void generate_debug(struct generator * g, struct node * p) {
1200
1198
  write_comment(g, p);
1201
1199
  g->I[0] = g->debug_count++;
1202
1200
  g->I[1] = p->line_number;
1203
- writef(g, "~Mdebug(~I0, ~I1);~N", p);
1201
+ writef(g, "~M_debug(~I0, ~I1);~N", p);
1204
1202
  }
1205
1203
 
1206
1204
  static void generate(struct generator * g, struct node * p) {
@@ -1348,7 +1346,7 @@ static void generate_equals(struct generator * g) {
1348
1346
  static void generate_among_table(struct generator * g, struct among * x) {
1349
1347
  write_comment(g, x->node);
1350
1348
 
1351
- struct amongvec * v = x->b;
1349
+ struct amongvec * v = x->v;
1352
1350
 
1353
1351
  g->I[0] = x->number;
1354
1352
  w(g, "~Mprivate final static Among[] a_~I0 = {~N~+");
@@ -1372,7 +1370,7 @@ static void generate_among_table(struct generator * g, struct among * x) {
1372
1370
 
1373
1371
  static void generate_amongs(struct generator * g) {
1374
1372
  for (struct among * x = g->analyser->amongs; x; x = x->next) {
1375
- generate_among_table(g, x);
1373
+ if (!x->duplicate) generate_among_table(g, x);
1376
1374
  }
1377
1375
  }
1378
1376
 
@@ -1455,6 +1453,29 @@ extern void generate_program_java(struct generator * g) {
1455
1453
  generate_groupings(g);
1456
1454
 
1457
1455
  generate_members(g);
1456
+
1457
+ if (g->analyser->debug_used) {
1458
+ w(g, "~N"
1459
+ "~Mprivate void _debug(int n, int line)~N"
1460
+ "~M{~N~+"
1461
+ "~MSystem.out.format(\"%3d (line %4d): [%d]'\", n, line, length);~N"
1462
+ "~Mfor (int i = 0; i <= length; ++i)~N"
1463
+ "~M{~N~+"
1464
+ "~Mif (limit_backward == i) System.out.write('{');~N"
1465
+ "~Mif (bra == i) System.out.write('[');~N"
1466
+ "~Mif (cursor == i) System.out.write('|');~N"
1467
+ "~Mif (ket == i) System.out.write(']');~N"
1468
+ "~Mif (limit == i) System.out.write('}');~N"
1469
+ "~Mif (i < length)~N"
1470
+ "~M{~N~+"
1471
+ "~Mchar ch = current[i];~N"
1472
+ "~Mif (ch == '\\0') System.out.write('#'); else System.out.write(ch);~N~-"
1473
+ "~M}~N~-"
1474
+ "~M}~N"
1475
+ "~MSystem.out.println(\"'\");~N~-"
1476
+ "~M}~N");
1477
+ }
1478
+
1458
1479
  generate_methods(g);
1459
1480
  generate_equals(g);
1460
1481