mittens 0.3.0 → 0.3.2

data/vendor/snowball/javascript/stemwords.js

@@ -1,103 +1,131 @@
-const fs = require('fs');
-const readline = require('readline');
+import fs from 'node:fs';
+import process from 'node:process';
+import readline from 'node:readline';
 
 function usage() {
-    console.log("usage: stemwords.js [-l <language>] -i <input file> -o <output file> [-c <character encoding>] [-h]\n");
-    console.log("The input file consists of a list of words to be stemmed, one per");
-    console.log("line. Words should be in lower case.\n");
-    console.log("If -c is given, the argument is the character encoding of the input");
-    console.log("and output files.  If it is omitted, the UTF-8 encoding is used.\n");
-    console.log("The output file consists of the stemmed words, one per line.\n");
-    console.log("-h displays this help");
-}
+    console.log(`usage: stemwords.js [-l <language>] [-i <input file>] [-o <output file>] [-c <character encoding>] [-h]
 
-if (process.argv.length < 5)
-{
-    usage();
+The input file consists of a list of words to be stemmed, one per line.
+Words should be in lower case.
+
+Language defaults to "English", input to stdin, and output to stdout.
+
+If -c is given, the argument is the character encoding of the input and
+output files.  If it is omitted, the UTF-8 encoding is used.
+
+The output file consists of the stemmed words, one per line.
+
+-h displays this help`);
 }
-else
+
 {
-    var input = '';
-    var output = '';
-    var encoding = 'utf8';
-    var language = 'English';
-    var show_help = false;
-    while (process.argv.length > 0)
+    let input = '';
+    let output = '';
+    let encoding = 'utf8';
+    let language = 'English';
+    let usage_error = false;
+    // Skip the first two entries of argv which are the interpreter
+    // and the script name.
+    //
+    // deno doesn't allow modifying process.argv so we need to make
+    // a copy here.
+    const argv = process.argv.slice(2);
+    while (argv.length > 0)
     {
-        var arg = process.argv.shift();
+        const arg = argv.shift();
         switch (arg)
         {
         case "-h":
-            show_help = true;
-            process.argv.length = 0;
+            usage();
+            process.exit(0);
             break;
         case "-l":
-            if (process.argv.length == 0)
+            if (argv.length === 0)
             {
-                show_help = true;
+                usage_error = true;
                 break;
             }
-            language = process.argv.shift();
+            language = argv.shift();
             break;
         case "-i":
-            if (process.argv.length == 0)
+            if (argv.length === 0)
             {
-                show_help = true;
+                usage_error = true;
                 break;
             }
-            input = process.argv.shift();
+            input = argv.shift();
             break;
         case "-o":
-            if (process.argv.length == 0)
+            if (argv.length === 0)
             {
-                show_help = true;
+                usage_error = true;
                 break;
             }
-            output = process.argv.shift();
+            output = argv.shift();
             break;
         case "-c":
-            if (process.argv.length == 0)
+            if (argv.length === 0)
             {
-                show_help = true;
+                usage_error = true;
                 break;
             }
-            encoding = process.argv.shift();
+            encoding = argv.shift();
             break;
+        default:
+            console.log('Unknown command line option: ' + arg + '\n');
+            usage_error = true;
+        }
+
+        if (usage_error)
+        {
+            usage();
+            process.exit(1);
         }
     }
-    if (show_help || input == '' || output == '')
-    {
-        usage();
+
+    const stemmer = await create(language);
+    let istream, ostream;
+    if (input !== '') {
+       istream = fs.createReadStream(input, encoding);
+    } else {
+       istream = process.stdin;
+       if (istream.setEncoding) istream.setEncoding(encoding);
     }
-    else
-    {
-        stemming(language, input, output, encoding);
+    if (output !== '') {
+        ostream = fs.createWriteStream(output, encoding);
+    } else {
+        ostream = process.stdout;
+        if (ostream.setEncoding) ostream.setEncoding(encoding);
     }
+
+    stemming(stemmer, istream, ostream);
 }
 
-// function stemming (lang : string, input : string, output : string, encoding : string) {
-function stemming (lang, input, output, encoding) {
+// function stemming (stemmer : Stemmer, input : Stream, output : Stream) {
+function stemming (stemmer, input, output) {
     const lines = readline.createInterface({
-        input: fs.createReadStream(input, encoding),
+        input: input,
         terminal: false
     });
-    var out = fs.createWriteStream(output, encoding);
-    var stemmer = create(lang);
     lines.on('line', (original) => {
-        out.write(stemmer.stemWord(original) + '\n');
+        output.write(stemmer.stemWord(original) + '\n');
     });
 }
 
-function create (name) {
-    var lc_name = name.toLowerCase();
-    if (!lc_name.match('\\W') && lc_name != 'base') {
-        try {
-            const Stemmer = require(lc_name + '-stemmer.js');
-            return new Stemmer();
-        } catch (error) {
-        }
+async function create (name) {
+    const lc_name = name.toLowerCase();
+    if (/\W/.test(lc_name) || lc_name === 'base') {
+        console.log('Unknown stemming language: ' + name + '\n');
+        usage();
+        process.exit(1);
+        return;
+    }
+    const filename = `../js_out/${lc_name}-stemmer.js`;
+    try {
+        // Load stemmer class from the module scope
+        const stemmerModule = await import(filename);
+        return new stemmerModule.default();
+    } catch (error) {
+        console.error(error);
     }
-    console.log('Unknown stemming language: ' + name + '\n');
-    usage();
-    process.exit(1);
 }

data/vendor/snowball/libstemmer/mkmodules.pl

@@ -252,7 +252,7 @@ EOS
     for $srcfile ('include/libstemmer.h',
                   "libstemmer/modules${extn}.h",
                   'runtime/api.h',
-                  'runtime/header.h') {
+                  'runtime/snowball_runtime.h') {
         print OUT " \\\n" if $need_sep;
         print OUT "  $srcfile";
         $need_sep = 1;

data/vendor/snowball/libstemmer/modules.txt

@@ -13,6 +13,7 @@ arabic          UTF_8                   arabic,ar,ara
 armenian        UTF_8                   armenian,hy,hye,arm
 basque          UTF_8,ISO_8859_1        basque,eu,eus,baq
 catalan         UTF_8,ISO_8859_1        catalan,ca,cat
+czech           UTF_8,ISO_8859_2        czech,cs,ces,cze
 danish          UTF_8,ISO_8859_1        danish,da,dan
 dutch           UTF_8,ISO_8859_1        dutch,nl,dut,nld,kraaij_pohlmann
 english         UTF_8,ISO_8859_1        english,en,eng
@@ -30,6 +31,8 @@ italian         UTF_8,ISO_8859_1        italian,it,ita
 lithuanian      UTF_8                   lithuanian,lt,lit
 nepali          UTF_8                   nepali,ne,nep
 norwegian       UTF_8,ISO_8859_1        norwegian,no,nor
+polish          UTF_8,ISO_8859_2        polish,pl,pol
+persian         UTF_8                   persian,fa,fas,pers
 portuguese      UTF_8,ISO_8859_1        portuguese,pt,por
 romanian        UTF_8                   romanian,ro,rum,ron
 russian         UTF_8,KOI8_R            russian,ru,rus
@@ -39,6 +42,7 @@ swedish         UTF_8,ISO_8859_1        swedish,sv,swe
 tamil           UTF_8                   tamil,ta,tam
 turkish         UTF_8                   turkish,tr,tur
 yiddish         UTF_8                   yiddish,yi,yid
+sesotho         UTF_8                   sesotho,st,sot
 
 # Also include the traditional porter algorithm for english.
 # The porter algorithm is included in the libstemmer distribution to assist

data/vendor/snowball/pascal/SnowballProgram.pas

@@ -36,11 +36,8 @@ Type
         Function OutGroupingBk(s : array of char; min, max : Integer) : Boolean;
         Function GoOutGroupingBk(s : array of char; min, max : Integer) : Boolean;
 
-        Function EqS(s_size : Integer; s : AnsiString) : Boolean;
-        Function EqSBk(s_size : Integer; s : AnsiString) : Boolean;
-
-        Function EqV(s : AnsiString) : Boolean;
-        Function EqVBk(s : AnsiString) : Boolean;
+        Function EqS(s : AnsiString) : Boolean;
+        Function EqSBk(s : AnsiString) : Boolean;
 
         Function FindAmong(v : array of TAmong; v_size : Integer) : Integer;
         Function FindAmongBk(v : array of TAmong; v_size : Integer) : Integer;
@@ -233,10 +230,11 @@ Begin
     Result := False;
 End;
 
-Function TSnowballProgram.EqS(s_size : Integer; s : AnsiString) : Boolean;
-Var I : Integer;
+Function TSnowballProgram.EqS(s : AnsiString) : Boolean;
+Var I, s_size : Integer;
 Begin
     Result := False;
+    s_size := Length(s);
 
     If (FLimit - FCursor) < s_size Then Exit;
 
@@ -248,10 +246,11 @@ Begin
     Result := True;
 End;
 
-Function TSnowballProgram.EqSBk(s_size : Integer; s : AnsiString) : Boolean;
-Var I : Integer;
+Function TSnowballProgram.EqSBk(s : AnsiString) : Boolean;
+Var I, s_size : Integer;
 Begin
     Result := False;
+    s_size := Length(s);
 
     if (FCursor - FBkLimit) < s_size Then Exit;
 
@@ -263,19 +262,9 @@ Begin
     Result := True;
 End;
 
-Function TSnowballProgram.EqV(s : AnsiString) : Boolean;
-Begin
-    Result := EqS(Length(s), s);
-End;
-
-Function TSnowballProgram.EqVBk(s : AnsiString) : Boolean;
-Begin
-    Result := EqSBk(Length(s), s);
-End;
-
 Function TSnowballProgram.FindAmong(v : array of TAmong; v_size : Integer) : Integer;
 Var i, i2, j, c, l, common_i, common_j, k, diff, common : Integer;
-    first_key_inspected, res : Boolean;
+    first_key_inspected : Boolean;
     w : TAmong;
 Begin
     i := 0;
@@ -347,10 +336,9 @@ Begin
                 Exit;
             End;
 
-            res := w.Method;
-
-            FCursor := c + Length(w.Str);
-            if (res) Then Begin
+            if w.Method Then
+            Begin
+                FCursor := c + Length(w.Str);
                 Result := w.Result;
                 Exit;
             End;
@@ -367,7 +355,7 @@ End;
 
 Function TSnowballProgram.FindAmongBk(v : array of TAmong; v_size : Integer) : Integer;
 Var i, j, c, lb, common_i, common_j, k, diff, common, i2 : Integer;
-    first_key_inspected, res : Boolean;
+    first_key_inspected : Boolean;
     w : TAmong;
 Begin
     i := 0;
@@ -433,11 +421,9 @@ Begin
                 Exit;
             End;
 
-            res := w.Method;
-
-            FCursor := c - Length(w.Str);
-            If Res Then
+            if w.Method Then
             Begin
+                FCursor := c - Length(w.Str);
                 Result := w.Result;
                 Exit;
             End;
@@ -446,8 +432,8 @@ Begin
         i := w.Index;
         If i < 0 Then
         Begin
-                Result := 0;
-                Exit;
+            Result := 0;
+            Exit;
         End;
     End;
 End;
@@ -502,6 +488,7 @@ Procedure TSnowballProgram.SliceFrom(s : AnsiString);
 Begin
     SliceCheck();
     ReplaceS(FBra, FKet, s);
+    FKet := FBra + Length(s);
 End;
 
 Function TSnowballProgram.AssignTo() : AnsiString;