mittens 0.3.1 → 0.3.2

data/vendor/snowball/compiler/tokens.h

@@ -0,0 +1,71 @@
+struct token {
+    byte code;        /* Token code (from enum token_code) */
+    byte s_size;      /* Size of token */
+    const char s[14]; /* Token */
+};
+
+/* List of alphabetical tokens and their corresponding codes (symbol tokens are
+ * instead handled using `switch` in tokenise.c).
+ *
+ * Tokens below are ordered primarily by length, then by ASCII collating
+ * order amongst tokens of the same length.
+ */
+static const struct token alpha_tokens[] = {
+  { c_as,             2, "as" },
+  { c_do,             2, "do" },
+  { c_or,             2, "or" },
+  { c_and,            3, "and" },
+  { c_for,            3, "for" },
+  { c_get,            3, "get" },
+  { c_hex,            3, "hex" },
+  { c_hop,            3, "hop" },
+  { c_len,            3, "len" },
+  { c_non,            3, "non" },
+  { c_not,            3, "not" },
+  { c_set,            3, "set" },
+  { c_try,            3, "try" },
+  { c_fail,           4, "fail" },
+  { c_goto,           4, "goto" },
+  { c_loop,           4, "loop" },
+  { c_next,           4, "next" },
+  { c_size,           4, "size" },
+  { c_test,           4, "test" },
+  { c_true,           4, "true" },
+  { c_among,          5, "among" },
+  { c_false,          5, "false" },
+  { c_lenof,          5, "lenof" },
+  { c_limit,          5, "limit" },
+  { c_unset,          5, "unset" },
+  { c_atmark,         6, "atmark" },
+  { c_attach,         6, "attach" },
+  { c_cursor,         6, "cursor" },
+  { c_define,         6, "define" },
+  { c_delete,         6, "delete" },
+  { c_gopast,         6, "gopast" },
+  { c_insert,         6, "insert" },
+  { c_maxint,         6, "maxint" },
+  { c_minint,         6, "minint" },
+  { c_repeat,         6, "repeat" },
+  { c_sizeof,         6, "sizeof" },
+  { c_tomark,         6, "tomark" },
+  { c_atleast,        7, "atleast" },
+  { c_atlimit,        7, "atlimit" },
+  { c_decimal,        7, "decimal" },
+  { c_reverse,        7, "reverse" },
+  { c_setmark,        7, "setmark" },
+  { c_strings,        7, "strings" },
+  { c_tolimit,        7, "tolimit" },
+  { c_booleans,       8, "booleans" },
+  { c_integers,       8, "integers" },
+  { c_routines,       8, "routines" },
+  { c_setlimit,       8, "setlimit" },
+  { c_backwards,      9, "backwards" },
+  { c_externals,      9, "externals" },
+  { c_groupings,      9, "groupings" },
+  { c_stringdef,      9, "stringdef" },
+  { c_substring,      9, "substring" },
+  { c_backwardmode,  12, "backwardmode" },
+  { c_stringescapes, 13, "stringescapes" }
+};
+
+#define NUM_ALPHA_TOKENS ((int)(sizeof(alpha_tokens) / sizeof(alpha_tokens[0])))

data/vendor/snowball/csharp/Snowball/Among.cs

@@ -1,19 +1,20 @@
 // Copyright (c) 2001, Dr Martin Porter
 // Copyright (c) 2002, Richard Boulton
 // Copyright (c) 2015, Cesar Souza
+// Copyright (c) 2025, Olly Betts
 // All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are met:
 //
-//     * Redistributions of source code must retain the above copyright notice,
-//     * this list of conditions and the following disclaimer.
-//     * Redistributions in binary form must reproduce the above copyright
-//     * notice, this list of conditions and the following disclaimer in the
-//     * documentation and/or other materials provided with the distribution.
-//     * Neither the name of the copyright holders nor the names of its contributors
-//     * may be used to endorse or promote products derived from this software
-//     * without specific prior written permission.
+//    1. Redistributions of source code must retain the above copyright notice,
+//       this list of conditions and the following disclaimer.
+//    2. Redistributions in binary form must reproduce the above copyright
+//       notice, this list of conditions and the following disclaimer in the
+//       documentation and/or other materials provided with the distribution.
+//    3. Neither the name of the Snowball project nor the names of its contributors
+//       may be used to endorse or promote products derived from this software
+//       without specific prior written permission.
 //
 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
@@ -56,10 +57,10 @@ namespace Snowball
         public int Result { get; private set; }
 
         /// <summary>
-        ///   Action to be invoked.
+        ///   Condition function index (or 0 for no condition).
         /// </summary>
         ///
-        public Func<bool> Action { get; private set; }
+        public int Condition { get; private set; }
 
         /// <summary>
         ///   Initializes a new instance of the <see cref="Among"/> class.
@@ -68,27 +69,14 @@ namespace Snowball
         /// <param name="str">The search string.</param>
         /// <param name="index">The index to the longest matching substring.</param>
         /// <param name="result">The result of the lookup.</param>
+        /// <param name="condition">The index of the condition function (0 if none).</param>
         ///
-        public Among(String str, int index, int result)
-            : this(str, index, result, null)
-        {
-        }
-
-        /// <summary>
-        ///   Initializes a new instance of the <see cref="Among"/> class.
-        /// </summary>
-        ///
-        /// <param name="str">The search string.</param>
-        /// <param name="index">The index to the longest matching substring.</param>
-        /// <param name="result">The result of the lookup.</param>
-        /// <param name="action">The action to be performed, if any.</param>
-        ///
-        public Among(String str, int index, int result, Func<bool> action)
+        public Among(String str, int index, int result, int condition)
         {
             this.SearchString = str;
             this.MatchIndex = index;
             this.Result = result;
-            this.Action = action;
+            this.Condition = condition;
         }
 
         /// <summary>

data/vendor/snowball/csharp/Snowball/AssemblyInfo.cs

@@ -10,7 +10,7 @@ using System.Runtime.InteropServices;
 [assembly: AssemblyConfiguration("")]
 [assembly: AssemblyCompany("")]
 [assembly: AssemblyProduct("Snowball")]
-[assembly: AssemblyCopyright("Copyright © 2015-2019")]
+[assembly: AssemblyCopyright("Copyright © 2015-2025")]
 [assembly: AssemblyTrademark("")]
 [assembly: AssemblyCulture("")]
 
@@ -32,5 +32,5 @@ using System.Runtime.InteropServices;
 // You can specify all the values or you can default the Build and Revision Numbers
 // by using the '*' as shown below:
 // [assembly: AssemblyVersion("1.0.*")]
-[assembly: AssemblyVersion(/*SNOWBALL_VERSION*/"3.0.1.0")]
-[assembly: AssemblyFileVersion(/*SNOWBALL_VERSION*/"3.0.1.0")]
+[assembly: AssemblyVersion(/*SNOWBALL_VERSION*/"3.1.1.0")]
+[assembly: AssemblyFileVersion(/*SNOWBALL_VERSION*/"3.1.1.0")]

data/vendor/snowball/csharp/Snowball/Stemmer.cs

@@ -1,20 +1,20 @@
 // Copyright (c) 2001, Dr Martin Porter
 // Copyright (c) 2002, Richard Boulton
 // Copyright (c) 2015, Cesar Souza
-// Copyright (c) 2018, Olly Betts
+// Copyright (c) 2018-2026, Olly Betts
 // All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are met:
 //
-//     * Redistributions of source code must retain the above copyright notice,
-//     * this list of conditions and the following disclaimer.
-//     * Redistributions in binary form must reproduce the above copyright
-//     * notice, this list of conditions and the following disclaimer in the
-//     * documentation and/or other materials provided with the distribution.
-//     * Neither the name of the copyright holders nor the names of its contributors
-//     * may be used to endorse or promote products derived from this software
-//     * without specific prior written permission.
+//    1. Redistributions of source code must retain the above copyright notice,
+//       this list of conditions and the following disclaimer.
+//    2. Redistributions in binary form must reproduce the above copyright
+//       notice, this list of conditions and the following disclaimer in the
+//       documentation and/or other materials provided with the distribution.
+//    3. Neither the name of the Snowball project nor the names of its contributors
+//       may be used to endorse or promote products derived from this software
+//       without specific prior written permission.
 //
 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
@@ -30,6 +30,7 @@
 namespace Snowball
 {
     using System;
+    using System.Diagnostics;
     using System.Linq;
     using System.Text;
 
@@ -48,7 +49,7 @@ namespace Snowball
         }
 
         /// <summary>
-        ///   Gets the current string.
+        ///   The current string.
         /// </summary>
         ///
         protected StringBuilder current;
@@ -72,13 +73,13 @@ namespace Snowball
         protected int limit_backward;
 
         /// <summary>
-        ///   Starting bracket position.
+        ///   Start of the slice.
         /// </summary>
         ///
         protected int bra;
 
         /// <summary>
-        ///   Ending bracket position.
+        ///   End of the slice.
         /// </summary>
         ///
         protected int ket;
@@ -114,6 +115,12 @@ namespace Snowball
     ///
     public abstract class Stemmer : Env
     {
+        /// <summary>
+        ///   Among function being called.
+        /// </summary>
+        ///
+        protected int af;
+
         /// <summary>
         ///   Initializes a new instance of the <see cref="Stemmer"/> class.
         /// </summary>
@@ -390,7 +397,7 @@ namespace Snowball
         ///   forward.
         /// </summary>
         ///
-        protected int find_among(Among[] v)
+        protected int find_among(Among[] v, Func<bool> call_among_func)
         {
             int i = 0;
             int j = v.Length;
@@ -465,14 +472,14 @@ namespace Snowball
                 {
                     cursor = c + w.SearchString.Length;
 
-                    if (w.Action == null)
+                    if (w.Condition == 0)
                         return w.Result;
 
-                    bool res = w.Action();
-                    cursor = c + w.SearchString.Length;
-
-                    if (res)
+                    af = w.Condition;
+                    if (call_among_func()) {
+                        cursor = c + w.SearchString.Length;
                         return w.Result;
+                    }
                 }
 
                 i = w.MatchIndex;
@@ -488,7 +495,7 @@ namespace Snowball
         ///   backwards.
         /// </summary>
         ///
-        protected int find_among_b(Among[] v)
+        protected int find_among_b(Among[] v, Func<bool> call_among_func)
         {
             int i = 0;
             int j = v.Length;
@@ -558,14 +565,14 @@ namespace Snowball
                 {
                     cursor = c - w.SearchString.Length;
 
-                    if (w.Action == null)
+                    if (w.Condition == 0)
                         return w.Result;
 
-                    bool res = w.Action();
-                    cursor = c - w.SearchString.Length;
-
-                    if (res)
+                    af = w.Condition;
+                    if (call_among_func()) {
+                        cursor = c - w.SearchString.Length;
                         return w.Result;
+                    }
                 }
 
                 i = w.MatchIndex;
@@ -597,25 +604,26 @@ namespace Snowball
         /// </summary>
         protected void slice_check()
         {
-            if (bra < 0 || bra > ket || ket > limit || limit > current.Length)
-            {
-                System.Diagnostics.Trace.WriteLine("faulty slice operation");
-            }
+            Debug.Assert(bra >= 0);
+            Debug.Assert(bra <= ket);
+            Debug.Assert(ket <= limit);
+            Debug.Assert(limit <= current.Length);
         }
 
         /// <summary>
-        ///   Replaces the contents of the bracket with the string s.
+        ///   Replaces the contents of the slice with the string s.
         /// </summary>
         ///
-        /// <param name="s">The s.</param>
+        /// <param name="s">The string.</param>
         protected void slice_from(String s)
         {
             slice_check();
             replace_s(bra, ket, s);
+            ket = bra + s.Length;
         }
 
         /// <summary>
-        ///   Removes the current bracket contents.
+        ///   Removes the current slice contents.
         /// </summary>
         ///
         protected void slice_del()
@@ -635,18 +643,7 @@ namespace Snowball
         }
 
         /// <summary>
-        ///   Replaces the contents of the bracket with the string s.
-        /// </summary>
-        ///
-        protected void insert(int c_bra, int c_ket, StringBuilder s)
-        {
-            int adjustment = replace_s(c_bra, c_ket, s.ToString());
-            if (c_bra <= bra) bra += adjustment;
-            if (c_bra <= ket) ket += adjustment;
-        }
-
-        /// <summary>
-        ///   Replaces the contents of the bracket with the string s.
+        ///   Replaces the contents of s with the slice.
         /// </summary>
         ///
         protected void slice_to(StringBuilder s)
@@ -669,11 +666,10 @@ namespace Snowball
         /// <summary>
         ///   Replaces a specific region of the buffer with another text.
         /// </summary>
-        public static StringBuilder Replace(StringBuilder sb, int index, int length, string text)
+        public static void Replace(StringBuilder sb, int index, int length, string text)
         {
             sb.Remove(index, length - index);
             sb.Insert(index, text);
-            return sb;
         }
 
     }

data/vendor/snowball/csharp/Stemwords/Program.cs

@@ -7,14 +7,14 @@
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are met:
 //
-//     * Redistributions of source code must retain the above copyright notice,
-//     * this list of conditions and the following disclaimer.
-//     * Redistributions in binary form must reproduce the above copyright
-//     * notice, this list of conditions and the following disclaimer in the
-//     * documentation and/or other materials provided with the distribution.
-//     * Neither the name of the copyright holders nor the names of its contributors
-//     * may be used to endorse or promote products derived from this software
-//     * without specific prior written permission.
+//    1. Redistributions of source code must retain the above copyright notice,
+//       this list of conditions and the following disclaimer.
+//    2. Redistributions in binary form must reproduce the above copyright
+//       notice, this list of conditions and the following disclaimer in the
+//       documentation and/or other materials provided with the distribution.
+//    3. Neither the name of the Snowball project nor the names of its contributors
+//       may be used to endorse or promote products derived from this software
+//       without specific prior written permission.
 //
 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
@@ -86,16 +86,22 @@ namespace Snowball
                 return;
             }
 
-            Console.WriteLine("Using " + stemmer.GetType());
-
-            TextWriter output = System.Console.Out;
-            if (outputName != null)
-                output = new StreamWriter(outputName);
-
             TextReader input = System.Console.In;
             if (inputName != null)
                 input = new StreamReader(inputName);
 
+            TextWriter output;
+            if (outputName != null) {
+                output = new StreamWriter(outputName);
+            } else {
+                // For some reason this is much faster than using
+                // System.Console.Out, at least with mono on Linux.
+                //
+                // `make check_sharp_tamil` takes 0.842s wallclock instead of
+                // 1.848s.
+                output = new StreamWriter(Console.OpenStandardOutput());
+            }
+
             while (true)
             {
                 var line = input.ReadLine();

data/vendor/snowball/cxx/.gitignore

@@ -0,0 +1,4 @@
+/stemwords
+/stemmer.cxx
+/*_stemmer.cxx
+/*_stemmer.h

data/vendor/snowball/cxx/generate_algorithms.pl

@@ -0,0 +1,87 @@
+#!/usr/bin/env perl
+use strict;
+use 5.006;
+use warnings;
+
+my $progname = $0;
+
+if (scalar @ARGV != 2) {
+  print "Usage: $progname <outfile> <modules description file>\n";
+  exit 1;
+}
+
+my $outname = shift(@ARGV);
+my $modules = shift(@ARGV);
+
+my @algorithms_by_length = ();
+
+my %algorithm_name = ();
+
+sub add_name {
+    my ($alias, $name) = @_;
+    $algorithm_name{$alias} = $name;
+    push @{$algorithms_by_length[length($alias)]}, $alias;
+}
+
+open my $out, '>', $outname or die "Can't open output file '$outname': $!\n";
+
+print $out <<END;
+#include "stemmer.h"
+END
+
+open my $in, '<', $modules or die $!;
+while (<$in>) {
+    next if m/^\s*#/;
+    next if m/^\s*$/;
+    my ($alg, $encstr, $aliases) = split(/\s+/, $_);
+
+    print $out <<END;
+#include "${alg}_stemmer.h"
+END
+    foreach my $alias (split(/,/, $aliases)) {
+        add_name($alias, $alg);
+    }
+}
+close $in;
+
+print $out <<END;
+// $outname: Snowball stemmer factory
+//
+// This file is generated by $0 from a list of module names.
+// Do not edit manually.
+
+#include <string.h>
+
+namespace Snowball {
+
+Stemmer* make_stemmer(const char* language)
+{
+    switch (strlen(language)) {
+END
+
+for (my $i = 1; $i < @algorithms_by_length; ++$i) {
+    my $l = $algorithms_by_length[$i];
+    next unless defined $l;
+print $out <<END;
+        case $i:
+END
+    for (@$l) {
+        my $class = $algorithm_name{$_};
+        $class =~ s/_(.)/\U$1/g;
+        $class = ucfirst $class . "Stemmer";
+print $out <<END;
+            if (memcmp(language, "$_", $i) == 0) return new Snowball::$class();
+END
+    }
+print $out <<END;
+            break;
+END
+}
+
+print $out <<END;
+    }
+    return NULL;
+}
+
+}
+END

data/vendor/snowball/cxx/stemmer.h

@@ -0,0 +1,12 @@
+#include <string>
+
+namespace Snowball {
+
+class Stemmer {
+  public:
+    virtual std::string operator()(const std::string& word) = 0;
+};
+
+Stemmer* make_stemmer(const char* language);
+
+}