mittens 0.2.0 → 0.3.0

data/vendor/snowball/csharp/Snowball/Stemmer.cs

@@ -3,10 +3,10 @@
 // Copyright (c) 2015, Cesar Souza
 // Copyright (c) 2018, Olly Betts
 // All rights reserved.
-// 
+//
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are met:
-// 
+//
 //     * Redistributions of source code must retain the above copyright notice,
 //     * this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above copyright
@@ -15,7 +15,7 @@
 //     * Neither the name of the copyright holders nor the names of its contributors
 //     * may be used to endorse or promote products derived from this software
 //     * without specific prior written permission.
-// 
+//
 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
@@ -36,13 +36,13 @@ namespace Snowball
     /// <summary>
     ///   Class holding current state.
     /// </summary>
-    /// 
+    ///
     public class Env
     {
         /// <summary>
         ///   Initializes a new instance of the <see cref="Env"/> class.
         /// </summary>
-        /// 
+        ///
         protected Env()
         {
         }
@@ -50,43 +50,43 @@ namespace Snowball
         /// <summary>
         ///   Gets the current string.
         /// </summary>
-        /// 
+        ///
         protected StringBuilder current;
 
         /// <summary>
         ///   Current cursor position.
         /// </summary>
-        /// 
+        ///
         protected int cursor;
 
         /// <summary>
         ///   Forward limit for inspecting the buffer.
         /// </summary>
-        /// 
+        ///
         protected int limit;
 
         /// <summary>
         ///   Backward limit for inspecting the buffer.
         /// </summary>
-        /// 
+        ///
         protected int limit_backward;
 
         /// <summary>
         ///   Starting bracket position.
         /// </summary>
-        /// 
+        ///
         protected int bra;
 
         /// <summary>
         ///   Ending bracket position.
         /// </summary>
-        /// 
+        ///
         protected int ket;
 
         /// <summary>
         ///   Copy another Env object.
         /// </summary>
-        /// 
+        ///
         public Env(Env other)
         {
             copy_from(other);
@@ -95,7 +95,7 @@ namespace Snowball
         /// <summary>
         ///   Copy another Env object.
         /// </summary>
-        /// 
+        ///
         protected void copy_from(Env other)
         {
             current          = other.current;
@@ -111,13 +111,13 @@ namespace Snowball
     /// <summary>
     ///   Base class for Snowball's stemmer algorithms.
     /// </summary>
-    /// 
+    ///
     public abstract class Stemmer : Env
     {
         /// <summary>
         ///   Initializes a new instance of the <see cref="Stemmer"/> class.
         /// </summary>
-        /// 
+        ///
         protected Stemmer()
         {
             current = new StringBuilder();
@@ -128,14 +128,14 @@ namespace Snowball
         /// <summary>
         ///   Calls the stemmer to process the next word.
         /// </summary>
-        /// 
+        ///
         protected abstract bool stem();
 
 
         /// <summary>
         ///   Stems the buffer's contents.
         /// </summary>
-        /// 
+        ///
         public bool Stem()
         {
             return this.stem();
@@ -144,11 +144,11 @@ namespace Snowball
         /// <summary>
         ///   Stems a given word.
         /// </summary>
-        /// 
+        ///
         /// <param name="word">The word to be stemmed.</param>
-        /// 
+        ///
         /// <returns>The stemmed word.</returns>
-        /// 
+        ///
         public string Stem(string word)
         {
             setBufferContents(word);
@@ -160,7 +160,7 @@ namespace Snowball
         /// <summary>
         ///   Gets the current processing buffer.
         /// </summary>
-        /// 
+        ///
         public StringBuilder Buffer
         {
             get { return current; }
@@ -171,7 +171,7 @@ namespace Snowball
         ///   or the stemmed word, if the stemmer has been
         ///   processed.
         /// </summary>
-        /// 
+        ///
         public string Current
         {
             get { return current.ToString(); }
@@ -192,7 +192,7 @@ namespace Snowball
 
 
         /// <summary>
-        ///   Determines whether the current character is 
+        ///   Determines whether the current character is
         ///   inside a given group of characters <c>s</c>.
         /// </summary>
         protected int in_grouping(string s, int min, int max, bool repeat)
@@ -217,7 +217,7 @@ namespace Snowball
         }
 
         /// <summary>
-        ///   Determines whether the current character is 
+        ///   Determines whether the current character is
         ///   inside a given group of characters <c>s</c>.
         /// </summary>
         protected int in_grouping_b(string s, int min, int max, bool repeat)
@@ -241,7 +241,7 @@ namespace Snowball
         }
 
         /// <summary>
-        ///   Determines whether the current character is 
+        ///   Determines whether the current character is
         ///   outside a given group of characters <c>s</c>.
         /// </summary>
         protected int out_grouping(string s, int min, int max, bool repeat)
@@ -272,7 +272,7 @@ namespace Snowball
         }
 
         /// <summary>
-        ///   Determines whether the current character is 
+        ///   Determines whether the current character is
         ///   outside a given group of characters <c>s</c>.
         /// </summary>
         protected int out_grouping_b(string s, int min, int max, bool repeat)
@@ -323,6 +323,26 @@ namespace Snowball
             return true;
         }
 
+        /// <summary>
+        ///   Determines if the current buffer contains the
+        ///   string s, starting from the current position and
+        ///   going forward.
+        /// </summary>
+        protected bool eq_s(StringBuilder s)
+        {
+            if (limit - cursor < s.Length)
+                return false;
+
+            for (int i = 0; i != s.Length; i++)
+            {
+                if (current[cursor + i] != s[i])
+                    return false;
+            }
+
+            cursor += s.Length;
+            return true;
+        }
+
         /// <summary>
         ///   Determines if the current buffer contains the
         ///   string s, starting from the current position and
@@ -365,11 +385,11 @@ namespace Snowball
 
 
         /// <summary>
-        ///   Searches if the current buffer matches against one of the 
+        ///   Searches if the current buffer matches against one of the
         ///   amongs, starting from the current cursor position and going
         ///   forward.
         /// </summary>
-        /// 
+        ///
         protected int find_among(Among[] v)
         {
             int i = 0;
@@ -463,11 +483,11 @@ namespace Snowball
         }
 
         /// <summary>
-        ///   Searches if the current buffer matches against one of the 
+        ///   Searches if the current buffer matches against one of the
         ///   amongs, starting from the current cursor position and going
         ///   backwards.
         /// </summary>
-        /// 
+        ///
         protected int find_among_b(Among[] v)
         {
             int i = 0;
@@ -559,7 +579,7 @@ namespace Snowball
         ///   Replaces the characters between <c>c_bra</c>
         ///   and <c>c_ket</c> by the characters in s.
         /// </summary>
-        ///   
+        ///
         protected int replace_s(int c_bra, int c_ket, String s)
         {
             int adjustment = s.Length - (c_ket - c_bra);
@@ -586,7 +606,7 @@ namespace Snowball
         /// <summary>
         ///   Replaces the contents of the bracket with the string s.
         /// </summary>
-        /// 
+        ///
         /// <param name="s">The s.</param>
         protected void slice_from(String s)
         {
@@ -597,7 +617,7 @@ namespace Snowball
         /// <summary>
         ///   Removes the current bracket contents.
         /// </summary>
-        /// 
+        ///
         protected void slice_del()
         {
             slice_from("");
@@ -606,7 +626,7 @@ namespace Snowball
         /// <summary>
         ///   Replaces the contents of the bracket with the string s.
         /// </summary>
-        /// 
+        ///
         protected void insert(int c_bra, int c_ket, String s)
         {
             int adjustment = replace_s(c_bra, c_ket, s);
@@ -617,7 +637,7 @@ namespace Snowball
         /// <summary>
         ///   Replaces the contents of the bracket with the string s.
         /// </summary>
-        /// 
+        ///
         protected void insert(int c_bra, int c_ket, StringBuilder s)
         {
             int adjustment = replace_s(c_bra, c_ket, s.ToString());
@@ -628,7 +648,7 @@ namespace Snowball
         /// <summary>
         ///   Replaces the contents of the bracket with the string s.
         /// </summary>
-        /// 
+        ///
         protected void slice_to(StringBuilder s)
         {
             slice_check();
@@ -638,7 +658,7 @@ namespace Snowball
         /// <summary>
         ///   Replaces the contents of the bracket with the string s.
         /// </summary>
-        /// 
+        ///
         protected void assign_to(StringBuilder s)
         {
             Replace(s, 0, s.Length, current.ToString(0, limit));

data/vendor/snowball/csharp/Stemwords/App.config

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="utf-8" ?>
 <configuration>
-    <startup> 
+    <startup>
         <supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5" />
     </startup>
-</configuration>
+</configuration>

data/vendor/snowball/csharp/Stemwords/Program.cs

@@ -1,11 +1,12 @@
 // Copyright (c) 2001, Dr Martin Porter
 // Copyright (c) 2002, Richard Boulton
 // Copyright (c) 2015, Cesar Souza
+// Copyright (c) 2025, Olly Betts
 // All rights reserved.
-// 
+//
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are met:
-// 
+//
 //     * Redistributions of source code must retain the above copyright notice,
 //     * this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above copyright
@@ -14,7 +15,7 @@
 //     * Neither the name of the copyright holders nor the names of its contributors
 //     * may be used to endorse or promote products derived from this software
 //     * without specific prior written permission.
-// 
+//
 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
@@ -32,24 +33,23 @@ namespace Snowball
     using System.IO;
     using System.Reflection;
     using System.Linq;
-    using System.Text;
 
     /// <summary>
     ///   Snowball's Stemmer program.
     /// </summary>
-    /// 
+    ///
     public static class Program
     {
 
         private static void usage()
         {
-            Console.WriteLine("Usage: stemwords.exe -l <language> -i <input file> [-o <output file>]");
+            Console.WriteLine("Usage: stemwords.exe -l <language> [-i <input file>] [-o <output file>]");
         }
 
         /// <summary>
         ///   Main program entrypoint.
         /// </summary>
-        /// 
+        ///
         public static void Main(String[] args)
         {
             string language = null;
@@ -62,11 +62,11 @@ namespace Snowball
                     language = args[i + 1];
                 else if (args[i] == "-i")
                     inputName = args[i + 1];
-                if (args[i] == "-o")
+                else if (args[i] == "-o")
                     outputName = args[i + 1];
             }
 
-            if (language == null || inputName == null)
+            if (language == null)
             {
                 usage();
                 return;
@@ -89,13 +89,17 @@ namespace Snowball
             Console.WriteLine("Using " + stemmer.GetType());
 
             TextWriter output = System.Console.Out;
-
             if (outputName != null)
                 output = new StreamWriter(outputName);
 
+            TextReader input = System.Console.In;
+            if (inputName != null)
+                input = new StreamReader(inputName);
 
-            foreach (var line in File.ReadAllLines(inputName))
+            while (true)
             {
+                var line = input.ReadLine();
+                if (line == null) break;
                 var o = stemmer.Stem(line);
                 output.WriteLine(o);
             }
@@ -105,7 +109,7 @@ namespace Snowball
 
         private static bool match(string stemmerName, string language)
         {
-            string expectedName = language.Replace("_", "") + "Stemmer";
+            string expectedName = language + "Stemmer";
 
             return stemmerName.StartsWith(expectedName,
                 StringComparison.CurrentCultureIgnoreCase);

data/vendor/snowball/doc/libstemmer_c_README

@@ -12,7 +12,7 @@ What is Stemming?
 
 Stemming maps different forms of the same word to a common "stem" - for
 example, the English stemmer maps *connection*, *connections*, *connective*,
-*connected*, and *connecting* to *connect*.  So a searching for *connected*
+*connected*, and *connecting* to *connect*.  So a search for *connected*
 would also find documents which only have the other forms.
 
 This stem form is often a word itself, but this is not always the case as this
@@ -50,6 +50,9 @@ files for the UTF-8 only version of the library.
 Using the library
 =================
 
+The stemming algorithms generally expect the input text to use composed accents
+(Unicode NFC or NFKC) and to have been folded to lower case already.
+
 The library provides a simple C API.  Essentially, a new stemmer can
 be obtained by using "sb_stemmer_new".  "sb_stemmer_stem" is then
 used to stem a word, "sb_stemmer_length" returns the stemmed
@@ -122,11 +125,11 @@ GNU autotool framework (and in particular, automake and autoconf) as follows:
    libstemmer_c subdirectory of the top level directory of the project.
 
 2) Add a file "Makefile.am" to the unpacked libstemmer_c folder, containing:
-   
+
 noinst_LTLIBRARIES = libstemmer.la
 include $(srcdir)/mkinc.mak
 noinst_HEADERS = $(snowball_headers)
-libstemmer_la_SOURCES = $(snowball_sources) 
+libstemmer_la_SOURCES = $(snowball_sources)
 
 (You may also need to add other lines to this, for example, if you are using
 compiler options which are not compatible with compiling the libstemmer
@@ -144,5 +147,5 @@ SUBDIRS=libstemmer_c
 <name>_LIBADD = libstemmer_c/libstemmer.la
 
 (Where <name> is the name of the library or executable which links against
-libstemmer.) 
+libstemmer.)
 

data/vendor/snowball/doc/libstemmer_csharp_README

@@ -12,7 +12,7 @@ What is Stemming?
 
 Stemming maps different forms of the same word to a common "stem" - for
 example, the English stemmer maps *connection*, *connections*, *connective*,
-*connected*, and *connecting* to *connect*.  So a searching for *connected*
+*connected*, and *connecting* to *connect*.  So a search for *connected*
 would also find documents which only have the other forms.
 
 This stem form is often a word itself, but this is not always the case as this
@@ -39,6 +39,9 @@ mcs -target:exe -out:stemwords.exe -r:snowballstemmer.dll csharp/Stemwords/Progr
 Using the library
 =================
 
+The stemming algorithms generally expect the input text to use composed accents
+(Unicode NFC or NFKC) and to have been folded to lower case already.
+
 There is currently no formal documentation on the use of the C# version
 of the library. Additionally, its interface is not guaranteed to be
 stable.

data/vendor/snowball/doc/libstemmer_java_README

@@ -12,7 +12,7 @@ What is Stemming?
 
 Stemming maps different forms of the same word to a common "stem" - for
 example, the English stemmer maps *connection*, *connections*, *connective*,
-*connected*, and *connecting* to *connect*.  So a searching for *connected*
+*connected*, and *connecting* to *connect*.  So a search for *connected*
 would also find documents which only have the other forms.
 
 This stem form is often a word itself, but this is not always the case as this
@@ -25,6 +25,14 @@ to a root form and/or get a root form which is itself a word then Snowball's
 stemming algorithms likely aren't the right answer.
 
 
+Requirements
+============
+
+The Java code generated by Snowball requires Java >= 7 (since Snowball 3.0.0).
+Java 7 was released in 2011, and Java 6's EOL was 2013 so we don't expect this
+to be a problematic requirement.
+
+
 Compiling the library
 =====================
 
@@ -41,6 +49,9 @@ provides a command line interface to the library.
 Using the library
 =================
 
+The stemming algorithms generally expect the input text to use composed accents
+(Unicode NFC or NFKC) and to have been folded to lower case already.
+
 There is currently no formal documentation on the use of the Java version
 of the library.  Additionally, its interface is not guaranteed to be
 stable.

data/vendor/snowball/doc/libstemmer_js_README

@@ -6,7 +6,7 @@ What is Stemming?
 
 Stemming maps different forms of the same word to a common "stem" - for
 example, the English stemmer maps *connection*, *connections*, *connective*,
-*connected*, and *connecting* to *connect*.  So a searching for *connected*
+*connected*, and *connecting* to *connect*.  So a search for *connected*
 would also find documents which only have the other forms.
 
 This stem form is often a word itself, but this is not always the case as this
@@ -22,16 +22,18 @@ stemming algorithms likely aren't the right answer.
 How to use library
 ------------------
 
+The stemming algorithms generally expect the input text to use composed accents
+(Unicode NFC or NFKC) and to have been folded to lower case already.
+
 You can use each stemming modules from Javascript code - e.g to use them
 with node:
 
 .. code-block:: javascript
 
-    const stemmer = require('base-stemmer.js');
-    const english_stemmer = require('english-stemmer.js');
+    var EnglishStemmer = require('english-stemmer.js');
 
     var stemmer = new EnglishStemmer();
-    alert(stemmer.stemWord("testing"));
+    console.log(stemmer.stemWord("testing"));
 
 You'll need to bundle ``base-stemmer.js`` and whichever languages you want
 stemmers for (e.g. ``english-stemmer.js`` for English).

data/vendor/snowball/doc/libstemmer_python_README

@@ -1,16 +1,18 @@
 Snowball stemming library collection for Python
 ===============================================
 
-Python 3 (>= 3.3) is supported.  We no longer actively support Python 2 as
-the Python developers stopped supporting it at the start of 2020.  Snowball
-2.1.0 was the last release to officially support Python 2.
+Python 3 (>= 3.3) is supported.  We no longer support Python 2 as the Python
+developers stopped supporting it at the start of 2020.  Snowball 2.1.0 was the
+last release to officially support Python 2; Snowball 3.0.0 was the last
+release which had the code to support Python 2, but we were no longer testing
+it.
 
 What is Stemming?
 -----------------
 
 Stemming maps different forms of the same word to a common "stem" - for
 example, the English stemmer maps *connection*, *connections*, *connective*,
-*connected*, and *connecting* to *connect*.  So a searching for *connected*
+*connected*, and *connecting* to *connect*.  So a search for *connected*
 would also find documents which only have the other forms.
 
 This stem form is often a word itself, but this is not always the case as this
@@ -25,6 +27,9 @@ stemming algorithms likely aren't the right answer.
 How to use library
 ------------------
 
+The stemming algorithms generally expect the input text to use composed accents
+(Unicode NFC or NFKC) and to have been folded to lower case already.
+
 The ``snowballstemmer`` module has two functions.
 
 The ``snowballstemmer.algorithms`` function returns a list of available

data/vendor/snowball/examples/stemwords.c

@@ -32,7 +32,7 @@ stem_file(struct sb_stemmer * stemmer, FILE * f_in, FILE * f_out)
                     sb_symbol * newb;
                     newb = (sb_symbol *)
                             realloc(b, (lim + INC) * sizeof(sb_symbol));
-                    if (newb == 0) goto error;
+                    if (newb == NULL) goto error;
                     b = newb;
                     lim = lim + INC;
                 }
@@ -50,7 +50,7 @@ stem_file(struct sb_stemmer * stemmer, FILE * f_in, FILE * f_out)
                 const sb_symbol * stemmed = sb_stemmer_stem(stemmer, b, i);
                 if (stemmed == NULL)
                 {
-                    fprintf(stderr, "Out of memory");
+                    fprintf(stderr, "Out of memory or internal error\n");
                     exit(1);
                 }
 
@@ -78,7 +78,7 @@ stem_file(struct sb_stemmer * stemmer, FILE * f_in, FILE * f_out)
         }
     }
 error:
-    if (b != 0) free(b);
+    free(b);
     return;
 }
 
@@ -114,8 +114,8 @@ usage(int n)
 int
 main(int argc, char * argv[])
 {
-    const char * in = 0;
-    const char * out = 0;
+    const char * in = NULL;
+    const char * out = NULL;
     FILE * f_in;
     FILE * f_out;
     struct sb_stemmer * stemmer;
@@ -172,20 +172,20 @@ main(int argc, char * argv[])
     }
 
     /* prepare the files */
-    f_in = (in == 0) ? stdin : fopen(in, "r");
-    if (f_in == 0) {
+    f_in = (in == NULL) ? stdin : fopen(in, "r");
+    if (f_in == NULL) {
         fprintf(stderr, "file %s not found\n", in);
         exit(1);
     }
-    f_out = (out == 0) ? stdout : fopen(out, "w");
-    if (f_out == 0) {
+    f_out = (out == NULL) ? stdout : fopen(out, "w");
+    if (f_out == NULL) {
         fprintf(stderr, "file %s cannot be opened\n", out);
         exit(1);
     }
 
     /* do the stemming process: */
     stemmer = sb_stemmer_new(language, charenc);
-    if (stemmer == 0) {
+    if (stemmer == NULL) {
         if (charenc == NULL) {
             fprintf(stderr, "language `%s' not available for stemming\n", language);
             exit(1);
@@ -197,8 +197,8 @@ main(int argc, char * argv[])
     stem_file(stemmer, f_in, f_out);
     sb_stemmer_delete(stemmer);
 
-    if (in != 0) (void) fclose(f_in);
-    if (out != 0) (void) fclose(f_out);
+    if (in != NULL) (void) fclose(f_in);
+    if (out != NULL) (void) fclose(f_out);
 
     return 0;
 }