mittens 0.3.3 → 0.3.5

data/vendor/snowball/compiler/tokeniser.c

@@ -22,24 +22,27 @@ extern byte * get_input(const char * filename) {
             fprintf(stderr, "%s: Read error\n", filename);
             exit(1);
         }
+        SET_SIZE(u, size);
     } else {
         // Unseekable stream, e.g. piped stdin.
         size = 0;
         u = create_s(INITIAL_INPUT_BUFFER_SIZE);
         while (true) {
-            int s = CAPACITY(u) - size;
-            int r = fread(u + size, 1, s, input);
-            if (r < 0) {
-                fprintf(stderr, "%s: Read error\n", filename);
-                exit(1);
+            size_t s = CAPACITY(u) - size;
+            size_t r = fread(u + size, 1, s, input);
+            size += (int)r;
+            SET_SIZE(u, size);
+            if (r < s) {
+                if (ferror(input)) {
+                    fprintf(stderr, "%s: Read error\n", filename);
+                    exit(1);
+                }
+                break;
             }
-            size += r;
-            if (r < s) break;
-            u = increase_capacity_s(u, size);
+            u = reserve_s(u, size * 2);
         }
     }
     if (input != stdin) fclose(input);
-    SET_SIZE(u, size);
     return u;
 }
 
@@ -171,12 +174,12 @@ static int read_literal_string(struct tokeniser * t, int c) {
                                 error1(t, "character values exceed 0x01ffff");
                             }
                             /* Ensure there's enough space for a max length
-                             * UTF-8 sequence. */
+                             * UTF-8 sequence and then encode the character
+                             * directly into that space. */
                             int b_size = SIZE(t->b);
-                            if (CAPACITY(t->b) < b_size + 3) {
-                                t->b = increase_capacity_b(t->b, 3);
-                            }
-                            SET_SIZE(t->b, b_size + put_utf8(codepoint, t->b + b_size));
+                            t->b = reserve_b(t->b, b_size + 4);
+                            SET_SIZE(t->b, b_size + put_utf8(codepoint,
+                                                             t->b + b_size));
                         } else {
                             if (t->encoding == ENC_SINGLEBYTE) {
                                 /* Only ISO-8859-1 is handled this way - for
@@ -533,7 +536,7 @@ extern int read_token(struct tokeniser * t) {
                     for (r = t->includes; r; r = r->next) {
                         byte * s = copy_s(r->s);
                         s = add_sz_to_s(s, file);
-                        s[SIZE(s)] = 0;
+                        s = ensure_nul_s(s);
                         if (file_owned > 0) {
                             free(file);
                         } else {
@@ -546,7 +549,7 @@ extern int read_token(struct tokeniser * t) {
                     }
                 }
                 if (u == NULL) {
-                    error(t, "Can't get '", (byte *)file, strlen(file), "'");
+                    error(t, "Can't get '", (byte *)file, (int)strlen(file), "'");
                     exit(1);
                 }
                 memmove(q, t, sizeof(struct input));
@@ -587,6 +590,15 @@ extern int peek_token(struct tokeniser * t) {
     return token;
 }
 
+extern void push_token(struct tokeniser * t, int token) {
+    if (t->token_held) {
+        error1(t, "push_token() called but token already held");
+        exit(1);
+    }
+    t->token = token;
+    t->token_held = true;
+}
+
 extern const char * name_of_token(int code) {
     for (int i = 0; i < NUM_ALPHA_TOKENS; i++) {
         if (alpha_tokens[i].code == code)

data/vendor/snowball/csharp/Snowball/AssemblyInfo.cs

@@ -32,5 +32,5 @@ using System.Runtime.InteropServices;
 // You can specify all the values or you can default the Build and Revision Numbers
 // by using the '*' as shown below:
 // [assembly: AssemblyVersion("1.0.*")]
-[assembly: AssemblyVersion(/*SNOWBALL_VERSION*/"3.1.1.0")]
-[assembly: AssemblyFileVersion(/*SNOWBALL_VERSION*/"3.1.1.0")]
+[assembly: AssemblyVersion(/*SNOWBALL_VERSION*/"3.2.0.0")]
+[assembly: AssemblyFileVersion(/*SNOWBALL_VERSION*/"3.2.0.0")]

data/vendor/snowball/csharp/Snowball/Stemmer.cs

@@ -190,11 +190,8 @@ namespace Snowball
             current.Clear();
             current.Insert(0, value);
 
-            cursor = 0;
+            limit_backward = bra = ket = cursor = 0;
             limit = current.Length;
-            limit_backward = 0;
-            bra = cursor;
-            ket = limit;
         }
 
 

data/vendor/snowball/cxx/generate_algorithms.pl

@@ -26,6 +26,11 @@ sub add_name {
 open my $out, '>', $outname or die "Can't open output file '$outname': $!\n";
 
 print $out <<END;
+// $outname: Snowball stemmer factory
+//
+// This file is generated by $0 from a list of module names.
+// Do not edit manually.
+
 #include "stemmer.h"
 END
 
@@ -45,10 +50,6 @@ END
 close $in;
 
 print $out <<END;
-// $outname: Snowball stemmer factory
-//
-// This file is generated by $0 from a list of module names.
-// Do not edit manually.
 
 #include <string.h>
 

data/vendor/snowball/cxx/{utilities.cxx → snowball_runtime.cxx}

@@ -1,2 +1,2 @@
 #define SNOWBALL_RUNTIME_THROW_EXCEPTIONS
-#include "../runtime/utilities.c"
+#include "../runtime/snowball_runtime.c"

data/vendor/snowball/cxx/stemmer.h

@@ -4,7 +4,11 @@ namespace Snowball {
 
 class Stemmer {
   public:
+#ifdef SNOWBALL_WIDE
+    virtual std::wstring operator()(const std::wstring& word) = 0;
+#else
     virtual std::string operator()(const std::string& word) = 0;
+#endif
 };
 
 Stemmer* make_stemmer(const char* language);

data/vendor/snowball/cxx/stemwords.cxx

@@ -14,10 +14,13 @@
 const char * progname;
 static int pretty = 1;
 
-static void
-stem_file(Snowball::Stemmer& stemmer, std::istream& f_in, std::ostream& f_out)
+template<typename Char>
+void
+stem_file(Snowball::Stemmer& stemmer,
+          std::basic_istream<Char>& f_in,
+          std::basic_ostream<Char>& f_out)
 {
-    std::string word;
+    std::basic_string<Char> word;
 
     while (true) {
         int ch = f_in.get();
@@ -28,8 +31,12 @@ stem_file(Snowball::Stemmer& stemmer, std::istream& f_in, std::ostream& f_out)
             word.clear();
             int inlen = 0;
             while (ch != '\n' && ch >= 0) {
-                /* Update count of utf-8 characters. */
-                if (ch < 0x80 || ch > 0xBF) inlen += 1;
+                if (sizeof(Char) == 1) {
+                    /* Update count of utf-8 characters. */
+                    if (ch < 0x80 || ch > 0xBF) inlen += 1;
+                } else {
+                    inlen += 1;
+                }
                 /* force lower case: */
                 if (ch < 128) ch = tolower(ch);
 
@@ -71,7 +78,7 @@ stem_file(Snowball::Stemmer& stemmer, std::istream& f_in, std::ostream& f_out)
 static void
 usage(int n)
 {
-    printf("usage: %s [-l <language>] [-i <input file>] [-o <output file>] [-c <character encoding>] [-p[2]] [-h]\n"
+    printf("usage: %s [-l <language>] [-i <input file>] [-o <output file>] [-p[2]] [-h]\n"
           "\n"
           "The input file consists of a list of words to be stemmed, one per\n"
           "line. Words should be in lower case, but (for English) A-Z letters\n"
@@ -147,6 +154,7 @@ try {
         exit(1);
     }
 
+#ifndef SNOWBALL_WIDE
     /* prepare the files */
     std::ifstream f_in;
     if (in) {
@@ -167,6 +175,28 @@ try {
 
     /* do the stemming process: */
     stem_file(*stemmer, in ? f_in : std::cin, out ? f_out : std::cout);
+#else
+    /* prepare the files */
+    std::wifstream f_in;
+    f_in.open(in ? in : "/dev/stdin");
+    if (!f_in.is_open()) {
+        fprintf(stderr, "file %s not found\n", in);
+        exit(1);
+    }
+    std::wofstream f_out;
+    f_out.open(out ? out : "/dev/stdout");
+    if (!f_out.is_open()) {
+        fprintf(stderr, "file %s cannot be opened\n", out);
+        exit(1);
+    }
+
+    std::locale c_utf8("C.UTF8");
+    f_in.imbue(c_utf8);
+    f_out.imbue(c_utf8);
+
+    /* do the stemming process: */
+    stem_file(*stemmer, f_in, f_out);
+#endif
 
     return 0;
 }

data/vendor/snowball/dart/lib/src/snowball.dart

@@ -45,8 +45,8 @@ class SnowballProgram {
     cursor = 0;
     limit = current.length;
     limit_backward = 0;
-    bra = cursor;
-    ket = limit;
+    bra = 0;
+    ket = 0;
   }
 
   void copy_from(SnowballProgram other) {

data/vendor/snowball/dart/pubspec.yaml

@@ -1,6 +1,6 @@
 name: snowball
 description: This package provides stemmers generated from Snowball algorithms.
-version: 3.1.1 # SNOWBALL_VERSION
+version: 3.2.0 # SNOWBALL_VERSION
 repository: https://github.com/snowballstem/snowball
 issue_tracker: https://github.com/snowballstem/snowball/issues
 topics:

data/vendor/snowball/doc/libstemmer_c_README

@@ -6,6 +6,9 @@ available for download from:
 
 https://snowballstem.org/download.html
 
+We aim to keep libstemmer compatible with ISO C90.  We have CI tests which
+build using modern compilers set to C90 mode, but if you hit compatibility
+problems please report them.
 
 What is Stemming?
 -----------------

data/vendor/snowball/doc/libstemmer_php_README

@@ -35,4 +35,4 @@ You can stem a word from PHP as follows:
     $stem = $stemmer->stemWord($word);
     printf("%s => %s\n", $word, $stem );
 
-	// prints "compiled => compil"
+    // prints "compiled => compil"

data/vendor/snowball/doc/libstemmer_python_README

@@ -1,11 +1,13 @@
 Snowball stemming library collection for Python
 ===============================================
 
-Python 3 (>= 3.3) is supported.  We no longer support Python 2 as the Python
-developers stopped supporting it at the start of 2020.  Snowball 2.1.0 was the
-last release to officially support Python 2; Snowball 3.0.1 was the last
-release which had the code to support Python 2, but we were no longer testing
-it.
+Python >= 3.3 is supported, though the oldest version we currently test in CI
+is Python 3.8.
+
+We no longer support Python 2 as the Python developers stopped supporting it at
+the start of 2020.  Snowball 2.1.0 was the last release to officially support
+Python 2; Snowball 3.0.1 was the last release which had the code to support
+Python 2, but we were no longer testing it.
 
 What is Stemming?
 -----------------
@@ -69,7 +71,7 @@ Snowball's ``libstemmer_c`` and should provide results 100% compatible to
 **snowballstemmer**.
 
 **PyStemmer** is faster because it wraps generated C versions of the stemmers;
-**snowballstemmer** uses generate Python code and is slower but offers a pure
+**snowballstemmer** uses generated Python code and is slower, but offers a pure
 Python solution.
 
 If PyStemmer is installed, ``snowballstemmer.stemmer`` returns a ``PyStemmer``

data/vendor/snowball/examples/stemwords.c

@@ -2,16 +2,15 @@
  * line interface for stemming using any of the algorithms provided.
  */
 
+#ifdef _MSC_VER
+# define _CRT_SECURE_NO_WARNINGS // Suppress stupid warning about fopen().
+#endif
+
 #include <stdio.h>
 #include <stdlib.h> /* for malloc, free */
 #include <string.h> /* for memmove */
 #include <ctype.h>  /* for isupper, tolower */
 
-#ifdef __cplusplus
-// Support compiling as C++ to test code generated by `snowball -c++`.
-# include <exception>
-#endif
-
 #include "libstemmer.h"
 
 const char * progname;
@@ -119,9 +118,6 @@ usage(int n)
 
 int
 main(int argc, char * argv[])
-#ifdef __cplusplus
-try
-#endif
 {
     const char * in = NULL;
     const char * out = NULL;
@@ -211,9 +207,3 @@ try
 
     return 0;
 }
-#ifdef __cplusplus
-catch (const std::exception& e) {
-    fprintf(stderr, "Exception: %s\n", e.what());
-    exit(1);
-}
-#endif

data/vendor/snowball/go/env.go

@@ -1,7 +1,7 @@
 package snowball
 
 import (
-	"log"
+	"fmt"
 	"strings"
 	"unicode/utf8"
 )
@@ -38,7 +38,7 @@ func (env *Env) SetCurrent(s string) {
 	env.Limit = len(s)
 	env.LimitBackward = 0
 	env.Bra = 0
-	env.Ket = len(s)
+	env.Ket = 0
 }
 
 func (env *Env) ReplaceS(bra, ket int, s string) int32 {
@@ -463,8 +463,20 @@ func (env *Env) FindAmongB(amongs []*Among, ctx interface{}) int32 {
 	}
 }
 
-func (env *Env) Debug(count, lineNumber int) {
-	log.Printf("snowball debug, count: %d, line: %d", count, lineNumber)
+func (env *Env) Debug(n, line int) {
+	length := len(env.current)
+	fmt.Printf("%3d (line %4d): [%d]'", n, line, length)
+	for i := 0; i <= length; i++ {
+		if env.LimitBackward == i { fmt.Print("{") }
+		if env.Bra == i { fmt.Print("[") }
+		if env.Cursor == i { fmt.Print("|") }
+		if env.Ket == i { fmt.Print("]") }
+		if env.Limit == i { fmt.Print("}") }
+		if (i < length) {
+			if env.current[i] == 0 { fmt.Print("#") } else { fmt.Print(env.current[i : i + 1]) }
+		}
+	}
+	fmt.Println("'")
 }
 
 func (env *Env) Clone() *Env {

data/vendor/snowball/include/libstemmer.h

@@ -7,9 +7,6 @@ extern "C" {
 struct sb_stemmer;
 typedef unsigned char sb_symbol;
 
-/* FIXME - should be able to get a version number for each stemming
- * algorithm (which will be incremented each time the output changes). */
-
 /** Returns an array of the names of the available stemming algorithms.
  *  Note that these are the canonical names - aliases (ie, other names for
  *  the same algorithm) will not be included in the list.

data/vendor/snowball/java/org/tartarus/snowball/SnowballProgram.java

@@ -12,8 +12,8 @@ public class SnowballProgram {
         cursor = 0;
         length = limit = 0;
         limit_backward = 0;
-        bra = cursor;
-        ket = limit;
+        bra = 0;
+        ket = 0;
     }
 
     /**
@@ -42,8 +42,8 @@ public class SnowballProgram {
         cursor = 0;
         this.length = limit = length;
         limit_backward = 0;
-        bra = cursor;
-        ket = limit;
+        bra = 0;
+        ket = 0;
     }
 
     /**

data/vendor/snowball/javascript/base-stemmer.js

@@ -20,8 +20,8 @@ export default class BaseStemmer {
         this.c = 0;
         this.limit = this.current.length;
         this.limit_backward = 0;
-        this.bra = this.c;
-        this.ket = this.limit;
+        this.bra = 0;
+        this.ket = 0;
     }
 
     /**
@@ -228,7 +228,7 @@ export default class BaseStemmer {
 
     /**
      * @param {Array<Array<string|number>>} v
-     * @param {?function(): boolean} call_among_func
+     * @param {function(): boolean} [call_among_func]
      * @return {number}
      */
     find_among(v, call_among_func) {
@@ -250,15 +250,13 @@ export default class BaseStemmer {
             let common = common_i < common_j ? common_i : common_j; // smaller
             // w[0]: string, w[1]: result, w[2]: substring_i (optional), w[3]: function (optional)
             const w = v[k];
-            let i2;
-            // @ts-expect-error: w[0] always string.
-            for (i2 = common; i2 < w[0].length; i2++) {
+            const s = /**@type{string}*/ (w[0]);
+            for (let i2 = common; i2 < s.length; i2++) {
                 if (c + common === l) {
                     diff = -1;
                     break;
                 }
-                // @ts-expect-error: w[0] always string.
-                diff = this.current.charCodeAt(c + common) - w[0].charCodeAt(i2);
+                diff = this.current.charCodeAt(c + common) - s.charCodeAt(i2);
                 if (diff !== 0) break;
                 common++;
             }
@@ -283,33 +281,28 @@ export default class BaseStemmer {
         }
         while (true) {
             const w = v[i];
-            // @ts-expect-error: w[0] always string.
-            if (common_i >= w[0].length) {
-                // @ts-expect-error: w[0] always string.
-                this.c = c + w[0].length;
-                // @ts-expect-error: w[1] always number.
-                if (w.length < 4) return w[1];
-                // @ts-expect-error: w[3] always number.
-                this.af = w[3];
+            const len = /**@type {string}*/ (w[0]).length;
+            if (common_i >= len) {
+                this.c = c + len;
+                if (w.length < 4) return /**@type {number}*/ (w[1]);
+                this.af = /**@type {number}*/ (w[3]);
                 // @ts-expect-error: call_among_func never null here.
                 if (call_among_func.call(this)) {
-                    // @ts-expect-error: w[0] always string.
-                    this.c = c + w[0].length;
-                    // @ts-expect-error: w[3] always number.
-                    return w[1];
+                    this.c = c + len;
+                    return /**@type {number}*/ (w[1]);
                 }
             }
             // Tests for undefined (if w.length < 2) or 0.
             if (!w[2]) return 0;
-            // @ts-expect-error: w[2] always number.
-            i -= w[2];
+            i -= /**@type {number}*/ (w[2]);
         }
     }
 
     // find_among_b is for backwards processing. Same comments apply
     /**
      * @param {Array<Array<string|number>>} v
-     * @param {?function(): boolean} call_among_func
+     * @param {function(): boolean} [call_among_func]
+     * @return {number}
      */
     find_among_b(v, call_among_func) {
         /** @protected */
@@ -329,15 +322,13 @@ export default class BaseStemmer {
             let diff = 0;
             let common = common_i < common_j ? common_i : common_j;
             const w = v[k];
-            let i2;
-            // @ts-expect-error: w[0] always string.
-            for (i2 = w[0].length - 1 - common; i2 >= 0; i2--) {
+            const s = /**@type{string}*/ (w[0]);
+            for (let i2 = s.length - 1 - common; i2 >= 0; i2--) {
                 if (c - common === lb) {
                     diff = -1;
                     break;
                 }
-                // @ts-expect-error: w[0] always string.
-                diff = this.current.charCodeAt(c - 1 - common) - w[0].charCodeAt(i2);
+                diff = this.current.charCodeAt(c - 1 - common) - s.charCodeAt(i2);
                 if (diff !== 0) break;
                 common++;
             }
@@ -357,24 +348,20 @@ export default class BaseStemmer {
         }
         while (true) {
             const w = v[i];
-            // @ts-expect-error: w[0] always string.
-            if (common_i >= w[0].length) {
-                // @ts-expect-error: w[0] always string.
-                this.c = c - w[0].length;
-                if (w.length < 4) return w[1];
-                // @ts-expect-error: w[3] always number.
-                this.af = w[3];
+            const len = /**@type {string}*/ (w[0]).length;
+            if (common_i >= len) {
+                this.c = c - len;
+                if (w.length < 4) return /**@type {number}*/ (w[1]);
+                this.af = /**@type {number}*/ (w[3]);
                 // @ts-expect-error: call_among_func never null here.
                 if (call_among_func.call(this)) {
-                    // @ts-expect-error: w[0] always string.
-                    this.c = c - w[0].length;
-                    return w[1];
+                    this.c = c - len;
+                    return /**@type {number}*/ (w[1]);
                 }
             }
             // Tests for undefined (if w.length < 2) or 0.
             if (!w[2]) return 0;
-            // @ts-expect-error: w[2] always number.
-            i -= w[2];
+            i -= /**@type {number}*/ (w[2]);
         }
     }
 

data/vendor/snowball/javascript/stemwords.js

@@ -1,3 +1,4 @@
+import { Buffer } from 'node:buffer';
 import fs from 'node:fs';
 import process from 'node:process';
 import readline from 'node:readline';
@@ -19,8 +20,9 @@ The output file consists of the stemmed words, one per line.
 }
 
 {
-    let input = '';
-    let output = '';
+    let input;
+    let output;
+    /**@type {BufferEncoding}*/
     let encoding = 'utf8';
     let language = 'English';
     let usage_error = false;
@@ -45,7 +47,7 @@ The output file consists of the stemmed words, one per line.
                 usage_error = true;
                 break;
             }
-            language = argv.shift();
+            language = /**@type {string}*/ (argv.shift());
             break;
         case "-i":
             if (argv.length === 0)
@@ -69,7 +71,16 @@ The output file consists of the stemmed words, one per line.
                 usage_error = true;
                 break;
             }
-            encoding = argv.shift();
+            {
+                const enc = /**@type {string}*/ (argv.shift());
+                if (!Buffer.isEncoding(enc))
+                {
+                    console.log('Unknown character encoding: ' + enc + '\n');
+                    usage_error = true;
+                    break;
+                }
+                encoding = enc;
+            }
             break;
         default:
             console.log('Unknown command line option: ' + arg + '\n');
@@ -85,13 +96,13 @@ The output file consists of the stemmed words, one per line.
 
     const stemmer = await create(language);
     let istream, ostream;
-    if (input !== '') {
+    if (input !== undefined) {
        istream = fs.createReadStream(input, encoding);
     } else {
        istream = process.stdin;
        if (istream.setEncoding) istream.setEncoding(encoding);
     }
-    if (output !== '') {
+    if (output !== undefined) {
         ostream = fs.createWriteStream(output, encoding);
     } else {
         ostream = process.stdout;
@@ -101,8 +112,17 @@ The output file consists of the stemmed words, one per line.
     stemming(stemmer, istream, ostream);
 }
 
-// function stemming (stemmer : Stemmer, input : Stream, output : Stream) {
-function stemming (stemmer, input, output) {
+/**
+ * @typedef {object} Stemmer
+ * @property {function(string): string} stemWord
+ */
+
+/**
+ * @param {Stemmer} stemmer
+ * @param {NodeJS.ReadableStream} input
+ * @param {NodeJS.WritableStream} output
+ */
+function stemming(stemmer, input, output) {
     const lines = readline.createInterface({
         input: input,
         terminal: false
@@ -112,13 +132,16 @@ function stemming (stemmer, input, output) {
     });
 }
 
-async function create (name) {
+/**
+ * @param {string} name
+ * @return {Promise<Stemmer>} name
+ */
+async function create(name) {
     const lc_name = name.toLowerCase();
     if (/\W/.test(lc_name) || lc_name === 'base') {
         console.log('Unknown stemming language: ' + name + '\n');
         usage();
         process.exit(1);
-        return;
     }
     const filename = `../js_out/${lc_name}-stemmer.js`;
     try {
@@ -127,5 +150,6 @@ async function create (name) {
         return new stemmerModule.default();
     } catch (error) {
         console.error(error);
+        process.exit(1);
     }
 }