mittens 0.3.0 → 0.3.2

data/vendor/snowball/tests/runtime/setlimit.sbl

@@ -0,0 +1,24 @@
+// Test setlimit
+externals (stem)
+routines (setlimit_tomark)
+
+define stem as (
+  setlimit_tomark
+  ='ok'
+)
+
+define setlimit_tomark as (
+  // Regression test for bug fixed in 3.1.0.
+  test (
+    ='abcd'
+    next
+    // Forwards `setlimit tomark` restored limit incorrectly.
+    setlimit tomark 3 for $(limit == 3) or fail='setlimit tomark set wrong limit'
+    $(cursor == 1) or fail='cursor changed after setlimit tomark'
+    $(limit == 4) or fail='limit not restored after setlimit tomark'
+    // Make sure restoring of limit copes with setting the limit to itself!
+    setlimit tomark limit for $(limit == 4) or fail='setlimit tomark limit set wrong limit'
+    $(cursor == 1) or fail='cursor changed after setlimit tomark'
+    $(limit == 4) or fail='limit not restored after setlimit tomark'
+  )
+)

data/vendor/snowball/tests/runtime/sizelen.sbl

@@ -0,0 +1,56 @@
+// Test len/lenof/size/sizeof.
+
+externals (stem)
+
+integers (expected_size)
+
+strings (s)
+
+stringescapes {}
+
+define stem as (
+  = ''
+  $(len == 0) or fail='len empty incorrect'
+  $(size == 0) or fail='size empty incorrect'
+  = 'test'
+  $(len == 4) or fail='len incorrect'
+  $(size == 4) or fail='size incorrect'
+
+  $(lenof '' == 0) or fail='lenof empty literal incorrect'
+  $(sizeof '' == 0) or fail='sizeof empty literal incorrect'
+  $(lenof 'test' == 4) or fail='lenof literal incorrect'
+  $(sizeof 'test' == 4) or fail='sizeof literal incorrect'
+
+  $s = ''
+  $(lenof s == 0) or fail='lenof empty string variable incorrect'
+  $(sizeof s == 0) or fail='sizeof empty string variable incorrect'
+  $s = 'test'
+  $(lenof s == 4) or fail='lenof string variable incorrect'
+  $(sizeof s == 4) or fail='sizeof string variable incorrect'
+
+  // Probe if character representation is UTF-8.
+  (
+    // UTF-8.
+    $(sizeof '{U+00A0}' == 2)  $expected_size = 8
+  ) or (
+    $(sizeof '{U+00A0}' == 1)  $expected_size = 7
+  ) or fail='sizeof U+00A0 unexpected'
+
+  = '{U+00E9}preuve'
+  $(len == 7) or fail='Unicode len incorrect'
+
+  $(lenof '{U+00E9}preuve' == 7) or fail='Unicode lenof literal incorrect'
+
+  $s = '{U+00E9}preuve'
+  $(lenof s == 7) or fail='Unicode lenof string variable incorrect'
+
+  = '{U+00E9}preuve'
+  $(size == expected_size) or fail='Unicode size incorrect'
+
+  $(sizeof '{U+00E9}preuve' == expected_size) or fail='Unicode sizeof literal incorrect'
+
+  $s = '{U+00E9}preuve'
+  $(sizeof s == expected_size) or fail='Unicode sizeof string variable incorrect'
+
+  ='ok'
+)

data/vendor/snowball/tests/runtime/slice.sbl

@@ -0,0 +1,27 @@
+// Test slice-related operations.
+
+externals (stem)
+
+strings (s)
+
+stringescapes {}
+
+define stem as (
+  test ([ tolimit ] <-'abcd') $(size == 4) or fail='<- replacement did not work as expected'
+  test ([ tolimit ] <-'ok') $(size == 2) or fail='<- replacement did not work as expected (2)'
+
+  test ([ tolimit ] <-'abcd' $(cursor == 4)) or fail='<- replacement did not adjust cursor as expected'
+  test ([ tolimit ] <-'ok' $(cursor == 2)) or fail='<- replacement did not adjust cursor as expected (2)'
+
+  test ([ tolimit ]) <-'abcd' $(cursor == 0) or fail='<- replacement did not adjust cursor as expected (3)'
+  test ([ tolimit ]) <-'ok' $(cursor == 0) or fail='<- replacement did not adjust cursor as expected (4)'
+
+  test ([ next ] delete <-'boo') test ('book' atlimit) or fail='delete did not adjust slice as expected'
+  test ([ next ] <-'ch' <-'outl') test ('outlook' atlimit) or fail='<- did not adjust slice as expected'
+  test (='abc' [ next ] ->s delete <-s) test ('abc' atlimit) or fail='stored slice did not survive later mutation'
+
+  // Regression test for https://github.com/snowballstem/snowball/issues/242
+  [] ->s $(sizeof s == 0) or fail='-> on empty slice was not empty'
+
+  ='ok'
+)

data/vendor/snowball/tests/runtime/stringdollar.sbl

@@ -0,0 +1,39 @@
+// Test string-$.
+
+externals (stem)
+
+strings (s s2)
+
+stringescapes {}
+
+define stem as (
+  $s=''
+  $(lenof s == 0) or fail='setting s empty failed'
+
+  $s='xyz'
+  $(lenof s == 3) or fail='setting s failed'
+  test ('ok' atlimit) or fail='setting s changed current string'
+
+  test (next $s hop 2 $(cursor == 1)) or fail='cursor not restored after string-$'
+  test (next $s hop 2 $(limit == 2)) or fail='limit not restored after string-$'
+
+  test (next $s $(cursor == 0)) or fail='cursor not zeroed inside string-$'
+  test (next $s $(limit == 3)) or fail='limit not reset inside string-$'
+
+  // Test things are reset on failure (regression test for Dart, Go, Rust).
+  (not $s fail ='news' test ('ok' atlimit)) or fail='current string not restored after failing string-$'
+  test (next not $s fail (='news' next) $(cursor == 1)) or fail='cursor not restored after failing string-$'
+  (not $s fail (='news' next) $(limit == 2)) or fail='limit not restored after failing string-$'
+
+  // Test things are reset on failure with nested use (regression test for
+  // everything, though for C the only problem was -Wshadow warnings).
+  (not $s2 $s fail ='news' test ('ok' atlimit)) or fail='current string not restored after failing string-$'
+  test (next not $s2 $s fail (='news' next) $(cursor == 1)) or fail='cursor not restored after failing string-$'
+  (not $s2 $s fail (='news' next) $(limit == 2)) or fail='limit not restored after failing string-$'
+  (not $s2 $s (='news' $(cursor == 99)) test ('ok' atlimit)) or fail='current string not restored after failing string-$'
+  test (next not $s2 $s (='news' next $(cursor == 99)) $(cursor == 1)) or fail='cursor not restored after failing string-$'
+  (not $s2 $s (='news' next $(cursor == 99)) $(limit == 2)) or fail='limit not restored after failing string-$'
+
+  // Test successful nested use.
+  $s2 (='x' $s='y') or fail='nested string-$ failed'
+)

data/vendor/snowball/tests/runtime/strings.sbl

@@ -0,0 +1,66 @@
+// Test strings
+
+externals (stem)
+
+strings (s)
+
+define stem as (
+  '' or fail='failed to match empty string'
+  test 'o' or fail='failed to match 1-char string'
+  test ('o' $(cursor == 1)) or fail='failed to update cursor after 1-char string match'
+  test 'ok' or fail='failed to match string'
+  test ('ok' $(cursor == 2)) or fail='failed to update cursor after string match'
+  test ('ok' $(cursor == 2) atlimit) or fail='atlimit failed after string match'
+  not 'x' or fail='incorrectly matched single character string'
+  not 'ko' or fail='incorrectly matched string'
+  $(cursor == 0) or fail='cursor not restored after string literal tests'
+
+  $s=''
+  s or fail='failed to match empty string var'
+  $s='o'
+  test s or fail='failed to match 1-char string var'
+  test (s $(cursor == 1)) or fail='failed to update cursor after 1-char string var match'
+  $s='ok'
+  test s or fail='failed to match string var'
+  test (s $(cursor == 2)) or fail='failed to update cursor after string var match'
+  test (s $(cursor == 2) atlimit) or fail='atlimit failed after string var match'
+  $s='x' not s or fail='incorrectly matched single character string var'
+  $s='ko' not s or fail='incorrectly matched string var'
+
+  do (
+    ='test'
+  )
+  $(cursor == 0) or fail='cursor not restored by do'
+
+  backwards (
+    do (
+      ='test'
+    )
+    $(cursor == size) or fail='cursor not restored by do'
+  )
+
+  ='abcde'
+  test ( $(len == 5) $(cursor == 0) 'abcde' atlimit ) or fail='string assignment wrong'
+
+  setlimit hop 4 for test (next ='nkl')
+  test ( $(len == 5) $(cursor == 0) 'ankle' atlimit ) or fail='string assignment does not replace c:l (same len)'
+
+  setlimit hop 4 for test (next ='x')
+  test ( $(len == 3) $(cursor == 0) 'axe' atlimit ) or fail='string assignment does not replace c:l (shorter)'
+
+  setlimit hop 2 for test (next ='ustralopithecin')
+  test ( $(len == 17) $(cursor == 0) 'australopithecine' atlimit ) or fail='string assignment does not replace c:l (longer)'
+
+  // Test string literals are properly escaped for various target languages.
+  $s='$xxx \x42 \u0042 \u{0042} \u000a {$xxx}' $(lenof s == 39) or fail='bad target language string literal escaping'
+
+  // Test ASCII control characters in string literals
+  stringescapes {}
+  $s='{U+A}{U+1F}{U+7F}{U+0}x' $(lenof s == 5) or fail='ASCII control characters not handled'
+  $s='a{U+80}{U+A0}b{U+00FF}' $(lenof s == 5) or fail='Unicode characters not handled'
+
+  // Test that using a C close-comment sequence doesn't trip anything up.
+  not '*/' or fail='C close comment sequence mishandled'
+
+  ='ok'
+)

data/vendor/snowball/tests/runtime/test.sbl

@@ -0,0 +1,19 @@
+// Test `test`.
+
+externals (stem)
+
+strings (s)
+
+define stem as (
+  backwards (
+    $(cursor == 2) or fail='backwards failed to swap cursor and limit'
+    test (
+      ='juggernaut'
+      $(cursor == 10) or fail='string= failed to update cursor'
+      next or fail='next failed'
+    )
+    $(cursor == 10) or fail='test failed to restore cursor'
+  )
+
+  ='ok'
+)

data/vendor/snowball/tests/stemtest.c

@@ -1,4 +1,24 @@
 /* Unit tests for handling of cases the vocabularies don't cover.
+ *
+ * In general we test the stemming algorithms via vocabulary lists
+ * and corresponding lists of expected stems, which are in the snowball-data
+ * repo.  These are used to verify that each stemmer produces the expected
+ * output for all the different programming languages Snowball can generate
+ * code for (whereas testcases here only exercise the C stemmers).
+ *
+ * Test coverage for a change to a stemmer's behaviour should generally be
+ * implemented by adding new words its the vocabulary (if words already there
+ * don't adequately cover the change).
+ *
+ * Only cases which can't be exercised by real words in the language being
+ * stemmed should go here.  Each vocabulary lists also serves as a sample
+ * vocabulary for evaluating its corresponding stemmer, so we don't want to
+ * add entries which would be junk for such use (such as emoji, numbers, hex
+ * codes, etc).
+ *
+ * Testcases here can run for a specified stemmer or for all stemmers.  This
+ * provides an easy way to test behaviours which any good stemmer should
+ * have (such as not mangling numbers).
  */
 
 #include <stdio.h>
@@ -7,8 +27,13 @@
 
 #include "libstemmer.h"
 
+#define U_0622 "\xd8\xa2"
+#define U_0627 "\xd8\xa7"
+#define U_062B "\xd8\xab"
+#define U_0631 "\xd8\xb1"
 #define EMOJI_FACE_THROWING_A_KISS "\xf0\x9f\x98\x98"
 #define U_40079 "\xf1\x80\x81\xb9"
+
 static const struct testcase {
     /* Stemmer to use, or 0 to test with all stemmers */
     const char * language;
@@ -25,12 +50,27 @@ static const struct testcase {
       "a" EMOJI_FACE_THROWING_A_KISS "ing",
       "a" EMOJI_FACE_THROWING_A_KISS "e" },
     { "en", 0, U_40079 "wing", 0 },
+      "a" EMOJI_FACE_THROWING_A_KISS "e" },
+    { "en", 0, U_40079 "wing", 0 },
+
+    // The Persian stemmer removes ASCII space inside a word.  It shouldn't
+    // appear there if our tokenisation recommendations are followed, and
+    // it seems more appropriate to test here rather than adding instances
+    // to persian/voc.txt.
+    { "fa", 0, U_0622 U_062B " " U_0627 U_0631,
+      U_0622 U_062B U_0627 U_0631, 0 },
+
     // The Finnish stemmer used to damage numbers ending with two or more of
     // the same digit.  Regression test, applied to all stemmers.
     // https://github.com/snowballstem/snowball/issues/66
-    { 0, 0, "2000", 0 },
+    { 0, 0, "00", 0 },
+    { 0, 0, "555", 0 },
     { 0, 0, "999", 0 },
+    { 0, 0, "1899", 0 },
+    { 0, 0, "2000", 0 },
+    { 0, 0, "9999", 0 },
     { 0, 0, "1000000000", 0 },
+
     // The Danish stemmer used to damage a number at the end of a word.
     // Regression test, applied to all stemmers.
     // https://github.com/snowballstem/snowball/issues/81

data/vendor/snowball/tests/syntax/canon.sbl

@@ -0,0 +1,32 @@
+// Test canonicalisation of commands.
+externals (stem)
+integers (x y)
+define stem as (
+  $x = cursor
+  setmark y
+
+  try 'a'
+
+  atmark x
+  $(cursor == y)
+
+  atlimit
+  $(cursor >= limit)
+  not next
+  not hop 1
+
+  not atlimit
+  not $(cursor >= limit)
+  $(cursor < limit)
+  test next
+  test hop 1
+
+  // `<-''` is the same as `delete`.
+  // `hop 1` is the same as `next`
+  [substring] among (
+    'a' (delete next)
+    'b' (<-''  hop 1)
+  )
+
+  $(x == y)
+)

data/vendor/snowball/tests/syntax/canon.syntax

@@ -0,0 +1,57 @@
+define stem
+  (
+    = x
+    # cursor
+    = y
+    # cursor
+    try
+      literal 'a'
+    ==
+    # name x
+      cursor
+    ==
+    # name y
+      cursor
+    >=
+    # limit
+      cursor
+    >=
+    # limit
+      cursor
+    >=
+    # limit
+      cursor
+    >=
+    # limit
+      cursor
+    <
+    # limit
+      cursor
+    <
+    # limit
+      cursor
+    <
+    # limit
+      cursor
+    <
+    # limit
+      cursor
+    <
+    # limit
+      cursor
+    [
+    substring
+    ]
+    among
+      literal 'a'
+      (
+        <- ''
+        next
+      literal 'b'
+      (
+        <- ''
+        next
+    ==
+    # name y
+      name x
+    Function end

data/vendor/snowball/tests/syntax/emptyprogram.sbl

@@ -0,0 +1,2 @@
+// This segfaulted in print_node_() with 3.0.1 because the program was NULL.
+stringescapes{}

data/vendor/snowball/tests/syntax/groupings.sbl

@@ -0,0 +1,11 @@
+// Test groupings.
+externals (stem)
+groupings (g)
+stringescapes {}
+define g 'r{U+EB}p{U+EB}{U+E5}tss'
+define stem as (
+  g
+  non-g
+  gopast g
+  gopast non-g
+)

data/vendor/snowball/tests/syntax/inline.sbl

@@ -0,0 +1,144 @@
+// Test inlining of routines.
+externals (stem external2)
+booleans (b)
+groupings (g)
+integers (i)
+strings (s)
+routines (
+  r_eq
+  r_ge
+  r_gt
+  r_le
+  r_lt
+  r_ne
+  r_booltest
+  r_not_booltest
+  r_grouping
+  r_non
+  r_goto_grouping
+  r_gopast_grouping
+  r_goto_non
+  r_gopast_non
+  r_literalstring
+  r_name
+  r_true
+  r_false
+  r_fail
+  r_set
+  r_unset
+  r_assign
+  r_plusassign
+  r_minusassign
+  r_multiplyassign
+  r_divideassign
+  r_tomark
+  r_tolimit
+  r_hop
+  r_next
+  r_delete
+  r_insert
+  r_attach
+  r_leftslice
+  r_rightslice
+  r_sliceto
+  r_stringassign
+  r_slicefrom
+  // These are canonicalised to an integer assignment or test.
+  r_atlimit
+  r_atmark
+  r_setmark
+)
+
+define g 'aeiou'
+
+define stem as (
+  set b
+  [next] -> s
+  r_eq
+  r_ge
+  r_gt
+  r_le
+  r_lt
+  r_ne
+  r_booltest
+  r_not_booltest
+  r_grouping
+  r_non
+  r_goto_grouping
+  r_gopast_grouping
+  r_goto_non
+  r_gopast_non
+  r_literalstring
+  r_name
+  r_true
+  do r_false
+  do r_fail
+  r_set
+  r_unset
+  r_assign
+  r_plusassign
+  r_minusassign
+  r_multiplyassign
+  r_divideassign
+  r_tomark
+  r_tolimit
+  r_hop
+  r_next
+  r_delete
+  r_insert
+  r_attach
+  r_leftslice
+  r_rightslice
+  r_sliceto
+  r_stringassign
+  r_slicefrom
+  r_atlimit
+  r_atmark
+  r_setmark
+  external2
+)
+
+// Externals should not get inlined.
+define external2 as true
+
+define r_eq as $(cursor == 1)
+define r_ge as $(cursor >= 1)
+define r_gt as $(cursor > 1)
+define r_le as $(cursor <= 1)
+define r_lt as $(cursor < 1)
+define r_ne as $(cursor != 1)
+define r_booltest as b
+define r_not_booltest as not b
+define r_grouping as g
+define r_non as non-g
+define r_goto_grouping as goto g
+define r_gopast_grouping as gopast g
+define r_goto_non as goto non-g
+define r_gopast_non as gopast non-g
+define r_literalstring as 'xyzzy'
+define r_name as s
+define r_true as true
+define r_false as false
+define r_fail as fail next
+define r_set as set b
+define r_unset as unset b
+define r_assign as $i = 1
+define r_plusassign as $i += 1
+define r_minusassign as $i -= 1
+define r_multiplyassign as $i *= 42
+define r_divideassign as $i /= 6
+define r_tomark as tomark i
+define r_tolimit as tolimit
+define r_hop as hop 2
+define r_next as next
+define r_delete as delete
+define r_insert as insert 'x'
+define r_attach as attach 'y'
+define r_leftslice as [
+define r_rightslice as ]
+define r_sliceto as ->s
+define r_stringassign as ='xyz'
+define r_slicefrom as <-'x'
+define r_atlimit as atlimit
+define r_atmark as atmark i
+define r_setmark as setmark i

data/vendor/snowball/tests/syntax/inline.syntax

@@ -0,0 +1,121 @@
+define stem
+  (
+    set b
+    [
+    next
+    ]
+    -> s
+    (
+      ==
+      # number 1
+        cursor
+    (
+      >=
+      # number 1
+        cursor
+    (
+      >
+      # number 1
+        cursor
+    (
+      <=
+      # number 1
+        cursor
+    (
+      <
+      # number 1
+        cursor
+    (
+      !=
+      # number 1
+        cursor
+    (
+      Boolean test b
+    (
+      Inverted boolean test b
+    (
+      grouping g
+    (
+      non g
+    (
+      goto grouping g
+    (
+      gopast grouping g
+    (
+      goto non g
+    (
+      gopast non g
+    (
+      literal 'xyzzy'
+    (
+      name s
+    (
+      true
+    do
+      (
+        false
+    do
+      (
+        fail
+          next
+    (
+      set b
+    (
+      unset b
+    (
+      = i
+      # number 1
+    (
+      += i
+      # number 1
+    (
+      -= i
+      # number 1
+    (
+      *= i
+      # number 42
+    (
+      /= i
+      # number 6
+    (
+      tomark
+      # name i
+    (
+      tolimit
+    (
+      hop
+      # number 2
+    (
+      next
+    (
+      <- ''
+    (
+      insert 'x'
+    (
+      attach 'y'
+    (
+      [
+    (
+      ]
+    (
+      -> s
+    (
+      = 'xyz'
+    (
+      <- 'x'
+    (
+      >=
+      # limit
+        cursor
+    (
+      ==
+      # name i
+        cursor
+    (
+      = i
+      # cursor
+    call external2
+    Function end
+define external2
+  true
+  Function end

data/vendor/snowball/tests/syntax/legacy.sbl

@@ -0,0 +1,17 @@
+// Test legacy commands.
+externals (stem)
+strings (ch)
+stringescapes {}
+stringdef o" decimal '246'
+stringdef o^ hex 'f4'
+define stem as (
+  among (
+    (next)
+    'a' 'e' 'i' 'o' 'u' (
+      [next]
+      => ch
+      delete
+      <+ ch
+    )
+  )
+)