mittens 0.3.1 → 0.3.2

data/vendor/snowball/cxx/stemwords.cxx

@@ -0,0 +1,176 @@
+/* This is a simple program which uses the generated C++ stemmers to provide a
+ * command line interface for stemming using any of the algorithms provided.
+ */
+
+#include <exception>
+#include <fstream>
+#include <iostream>
+
+#include <ctype.h>  /* for tolower */
+#include <string.h>  /* for strcmp */
+
+#include "stemmer.h"
+
+const char * progname;
+static int pretty = 1;
+
+static void
+stem_file(Snowball::Stemmer& stemmer, std::istream& f_in, std::ostream& f_out)
+{
+    std::string word;
+
+    while (true) {
+        int ch = f_in.get();
+        if (ch < 0) {
+            return;
+        }
+        {
+            word.clear();
+            int inlen = 0;
+            while (ch != '\n' && ch >= 0) {
+                /* Update count of utf-8 characters. */
+                if (ch < 0x80 || ch > 0xBF) inlen += 1;
+                /* force lower case: */
+                if (ch < 128) ch = tolower(ch);
+
+                word += ch;
+                ch = f_in.get();
+            }
+            if (pretty) {
+                f_out << word;
+            }
+
+            {
+                word = stemmer(word);
+                if (pretty == 1) {
+                    f_out << " -> ";
+                } else if (pretty == 2) {
+                    if (word.size() > 0) {
+                        int j;
+                        if (inlen < 30) {
+                            for (j = 30 - inlen; j > 0; j--)
+                                f_out << ' ';
+                        } else {
+                            f_out << '\n';
+                            for (j = 30; j > 0; j--)
+                                f_out << ' ';
+                        }
+                    }
+                }
+
+                f_out << word << '\n';
+            }
+        }
+    }
+    return;
+}
+
+/** Display the command line syntax, and then exit.
+ *  @param n The value to exit with.
+ */
+static void
+usage(int n)
+{
+    printf("usage: %s [-l <language>] [-i <input file>] [-o <output file>] [-c <character encoding>] [-p[2]] [-h]\n"
+          "\n"
+          "The input file consists of a list of words to be stemmed, one per\n"
+          "line. Words should be in lower case, but (for English) A-Z letters\n"
+          "are mapped to their a-z equivalents anyway. If omitted, stdin is\n"
+          "used.\n"
+          "\n",
+          progname);
+    printf(
+          "If -p is given the output file consists of each word of the input\n"
+          "file followed by \"->\" followed by its stemmed equivalent.\n"
+          "If -p2 is given the output file is a two column layout containing\n"
+          "the input words in the first column and the stemmed equivalents in\n"
+          "the second column.\n"
+          "Otherwise, the output file consists of the stemmed words, one per\n"
+          "line.\n"
+          "\n"
+          "-h displays this help\n");
+    exit(n);
+}
+
+int
+main(int argc, char * argv[])
+try {
+    const char * in = NULL;
+    const char * out = NULL;
+    const char * language = "english";
+
+    int i = 1;
+    pretty = 0;
+
+    progname = argv[0];
+
+    while (i < argc) {
+        const char * s = argv[i++];
+        if (s[0] == '-') {
+            if (strcmp(s, "-o") == 0) {
+                if (i >= argc) {
+                    fprintf(stderr, "%s requires an argument\n", s);
+                    exit(1);
+                }
+                out = argv[i++];
+            } else if (strcmp(s, "-i") == 0) {
+                if (i >= argc) {
+                    fprintf(stderr, "%s requires an argument\n", s);
+                    exit(1);
+                }
+                in = argv[i++];
+            } else if (strcmp(s, "-l") == 0) {
+                if (i >= argc) {
+                    fprintf(stderr, "%s requires an argument\n", s);
+                    exit(1);
+                }
+                language = argv[i++];
+            } else if (strcmp(s, "-p2") == 0) {
+                pretty = 2;
+            } else if (strcmp(s, "-p") == 0) {
+                pretty = 1;
+            } else if (strcmp(s, "-h") == 0) {
+                usage(0);
+            } else {
+                fprintf(stderr, "option %s unknown\n", s);
+                usage(1);
+            }
+        } else {
+            fprintf(stderr, "unexpected parameter %s\n", s);
+            usage(1);
+        }
+    }
+
+    Snowball::Stemmer* stemmer = Snowball::make_stemmer(language);
+    if (stemmer == NULL) {
+        fprintf(stderr, "language `%s' not available for stemming\n", language);
+        exit(1);
+    }
+
+    /* prepare the files */
+    std::ifstream f_in;
+    if (in) {
+        f_in.open(in);
+        if (!f_in.is_open()) {
+            fprintf(stderr, "file %s not found\n", in);
+            exit(1);
+        }
+    }
+    std::ofstream f_out;
+    if (out) {
+        f_out.open(out);
+        if (!f_out.is_open()) {
+            fprintf(stderr, "file %s cannot be opened\n", out);
+            exit(1);
+        }
+    }
+
+    /* do the stemming process: */
+    stem_file(*stemmer, in ? f_in : std::cin, out ? f_out : std::cout);
+
+    return 0;
+}
+catch (const std::exception& e) {
+    fprintf(stderr, "Exception: %s\n", e.what());
+    exit(1);
+}

data/vendor/snowball/cxx/utilities.cxx

@@ -0,0 +1,2 @@
+#define SNOWBALL_RUNTIME_THROW_EXCEPTIONS
+#include "../runtime/utilities.c"

data/vendor/snowball/dart/.gitignore

@@ -0,0 +1,4 @@
+.dart_tool/
+pubspec.lock
+
+.dart_deps

data/vendor/snowball/dart/analysis_options.yaml

@@ -0,0 +1 @@
+include: package:lints/recommended.yaml

data/vendor/snowball/dart/example/test_app.dart

@@ -0,0 +1,65 @@
+import 'dart:convert';
+import 'dart:io';
+
+import 'package:snowball/snowball.dart';
+
+void usage() {
+  print("Usage: test_app <algorithm> [<input file>] [-o <output file>]");
+}
+
+Future<void> main(List<String> args) async {
+  if (args.isEmpty) {
+    usage();
+    exit(1);
+  }
+
+  final algorithms = Algorithm.values.where((v) => v.name == args[0]);
+  if (algorithms.isEmpty) {
+    print('Stemmer for ${args[0]} not found');
+    exit(1);
+  }
+  final algorithm = algorithms.first;
+  final stemmer = SnowballStemmer(algorithm);
+
+  int arg = 1;
+
+  Stream<List<int>> inStream;
+  if (args.length > arg && args[arg] != '-o') {
+    inStream = File(args[arg++]).openRead();
+  } else {
+    inStream = stdin;
+  }
+
+  IOSink outStream;
+  if (args.length > arg) {
+    if (args.length != arg + 2 || args[arg] != '-o') {
+      usage();
+      exit(1);
+    }
+    outStream = File(args[arg + 1]).openWrite();
+  } else {
+    outStream = stdout.nonBlocking;
+  }
+
+  Stream<String> reader = inStream
+      .transform(utf8.decoder)
+      .transform(LineSplitter());
+
+  final outBuffer = StringBuffer();
+  await for (var line in reader) {
+    try {
+      final stem = stemmer.stem(line);
+      outBuffer.writeln(stem);
+      if (outBuffer.length > 8192) {
+        outStream.write(outBuffer);
+        outBuffer.clear();
+      }
+    } catch (e) {
+      print('Failed to stem word "$line"');
+      rethrow;
+    }
+  }
+  if (outBuffer.isNotEmpty) outStream.write(outBuffer);
+  await outStream.flush();
+  await outStream.close();
+}

data/vendor/snowball/dart/generate_algorithms.pl

@@ -0,0 +1,21 @@
+#!/usr/bin/env perl
+use strict;
+use warnings;
+
+my @algorithms = @ARGV;
+
+# Generate the lib/src/algorithms.dart file from modules.txt
+print("// ignore_for_file: constant_identifier_names\n\n");
+print("import 'package:snowball/src/snowball.dart';\n\n");
+foreach my $algorithm (@algorithms) {
+    print("import '../ext/${algorithm}_stemmer.dart';\n");
+}
+print("\nenum Algorithm {\n");
+foreach my $algorithm (@algorithms) {
+    print("  $algorithm,\n");
+}
+print("}\n\nfinal stemmers = <Algorithm, SnowballStemmer>{\n");
+foreach my $algorithm (@algorithms) {
+    print("  Algorithm.$algorithm: ${algorithm}_stemmer(),\n");
+}
+print("};\n");

data/vendor/snowball/dart/lib/snowball.dart

@@ -0,0 +1,18 @@
+// ignore_for_file: non_constant_identifier_names, curly_braces_in_flow_control_structures
+
+library;
+
+import 'src/algorithms.dart';
+export 'src/algorithms.dart';
+import 'src/snowball.dart' as impl;
+
+class SnowballStemmer {
+  final impl.SnowballStemmer _stemmer;
+
+  SnowballStemmer(Algorithm algorithm) : _stemmer = stemmers[algorithm]!;
+  String stem(String s) {
+    _stemmer.init(s);
+    _stemmer.stem();
+    return _stemmer.current;
+  }
+}

data/vendor/snowball/dart/lib/src/snowball.dart

@@ -0,0 +1,339 @@
+// ignore_for_file: non_constant_identifier_names, curly_braces_in_flow_control_structures
+
+library;
+
+/// Internal class used by Snowball stemmers
+class Among {
+  Among(this.s, this.substring_i, this.result, [this.function_index = 0]);
+
+  /// search string
+  final String s;
+
+  /// index to longest matching substring
+  final int substring_i;
+
+  /// result of the lookup
+  final int result;
+
+  /// function index, 0 if none
+  final int function_index;
+}
+
+class SnowballProgram {
+  String current = '';
+
+  int cursor = 0;
+  int limit = 0;
+  int limit_backward = 0;
+  int bra = 0;
+  int ket = 0;
+  int af = 0;
+
+  SnowballProgram();
+
+  SnowballProgram.from(SnowballProgram other)
+    : current = other.current,
+      cursor = other.cursor,
+      limit = other.limit,
+      limit_backward = other.limit_backward,
+      bra = other.bra,
+      ket = other.ket,
+      af = other.af;
+
+  void init(String s) {
+    current = s;
+    cursor = 0;
+    limit = current.length;
+    limit_backward = 0;
+    bra = cursor;
+    ket = limit;
+  }
+
+  void copy_from(SnowballProgram other) {
+    current = other.current;
+    cursor = other.cursor;
+    limit = other.limit;
+    limit_backward = other.limit_backward;
+    bra = other.bra;
+    ket = other.ket;
+  }
+
+  bool in_grouping(String s, int min, int max) {
+    if (cursor >= limit) return false;
+    int ch = current.codeUnitAt(cursor);
+    if (ch > max || ch < min) return false;
+    ch -= min;
+    if ((s.codeUnitAt(ch >> 3) & (0X1 << (ch & 0X7))) == 0) return false;
+    cursor++;
+    return true;
+  }
+
+  bool go_in_grouping(String s, int min, int max) {
+    while (cursor < limit) {
+      int ch = current.codeUnitAt(cursor);
+      if (ch > max || ch < min) return true;
+      ch -= min;
+      if ((s.codeUnitAt(ch >> 3) & (0X1 << (ch & 0X7))) == 0) return true;
+      cursor++;
+    }
+    return false;
+  }
+
+  bool in_grouping_b(String s, int min, int max) {
+    if (cursor <= limit_backward) return false;
+    int ch = current.codeUnitAt(cursor - 1);
+    if (ch > max || ch < min) return false;
+    ch -= min;
+    if ((s.codeUnitAt(ch >> 3) & (0X1 << (ch & 0X7))) == 0) return false;
+    cursor--;
+    return true;
+  }
+
+  bool go_in_grouping_b(String s, int min, int max) {
+    while (cursor > limit_backward) {
+      int ch = current.codeUnitAt(cursor - 1);
+      if (ch > max || ch < min) return true;
+      ch -= min;
+      if ((s.codeUnitAt(ch >> 3) & (0X1 << (ch & 0X7))) == 0) return true;
+      cursor--;
+    }
+    return false;
+  }
+
+  bool out_grouping(String s, int min, int max) {
+    if (cursor >= limit) return false;
+    int ch = current.codeUnitAt(cursor);
+    if (ch > max || ch < min) {
+      cursor++;
+      return true;
+    }
+    ch -= min;
+    if ((s.codeUnitAt(ch >> 3) & (0X1 << (ch & 0X7))) == 0) {
+      cursor++;
+      return true;
+    }
+    return false;
+  }
+
+  bool go_out_grouping(String s, int min, int max) {
+    while (cursor < limit) {
+      int ch = current.codeUnitAt(cursor);
+      if (ch <= max && ch >= min) {
+        ch -= min;
+        if ((s.codeUnitAt(ch >> 3) & (0X1 << (ch & 0X7))) != 0) {
+          return true;
+        }
+      }
+      cursor++;
+    }
+    return false;
+  }
+
+  bool out_grouping_b(String s, int min, int max) {
+    if (cursor <= limit_backward) return false;
+    int ch = current.codeUnitAt(cursor - 1);
+    if (ch > max || ch < min) {
+      cursor--;
+      return true;
+    }
+    ch -= min;
+    if ((s.codeUnitAt(ch >> 3) & (0X1 << (ch & 0X7))) == 0) {
+      cursor--;
+      return true;
+    }
+    return false;
+  }
+
+  bool go_out_grouping_b(String s, int min, int max) {
+    while (cursor > limit_backward) {
+      int ch = current.codeUnitAt(cursor - 1);
+      if (ch <= max && ch >= min) {
+        ch -= min;
+        if ((s.codeUnitAt(ch >> 3) & (0X1 << (ch & 0X7))) != 0) {
+          return true;
+        }
+      }
+      cursor--;
+    }
+    return false;
+  }
+
+  bool eq_s(String s) {
+    if (limit - cursor < s.length) return false;
+    if (!current.startsWith(s, cursor)) return false;
+    cursor += s.length;
+    return true;
+  }
+
+  bool eq_s_b(String s) {
+    if (cursor - limit_backward < s.length) return false;
+    if (!current.substring(0, cursor).endsWith(s)) return false;
+    cursor -= s.length;
+    return true;
+  }
+
+  int find_among(List<Among> v, bool Function()? call_among_func) {
+    int i = 0;
+    int j = v.length;
+
+    int c = cursor;
+    int l = limit;
+
+    int common_i = 0;
+    int common_j = 0;
+
+    bool first_key_inspected = false;
+
+    while (true) {
+      int k = i + ((j - i) >> 1);
+      int diff = 0;
+      int common = common_i < common_j ? common_i : common_j; // smaller
+      Among w = v[k];
+      for (int i2 = common; i2 < w.s.length; i2++) {
+        if (c + common == l) {
+          diff = -1;
+          break;
+        }
+        diff = current.codeUnitAt(c + common) - w.s.codeUnitAt(i2);
+        if (diff != 0) break;
+        common++;
+      }
+      if (diff < 0) {
+        j = k;
+        common_j = common;
+      } else {
+        i = k;
+        common_i = common;
+      }
+      if (j - i <= 1) {
+        if (i > 0) break; // v->s has been inspected
+        if (j == i) break; // only one item in v
+
+        // - but now we need to go round once more to get
+        // v->s inspected. This looks messy, but is actually
+        // the optimal approach.
+
+        if (first_key_inspected) break;
+        first_key_inspected = true;
+      }
+    }
+    while (true) {
+      Among w = v[i];
+      if (common_i >= w.s.length) {
+        cursor = c + w.s.length;
+        if (w.function_index == 0) return w.result;
+        af = w.function_index;
+        if (call_among_func?.call() ?? false) {
+          cursor = c + w.s.length;
+          return w.result;
+        }
+      }
+      i = w.substring_i;
+      if (i < 0) return 0;
+    }
+  }
+
+  int find_among_b(List<Among> v, bool Function()? call_among_func) {
+    int i = 0;
+    int j = v.length;
+
+    int c = cursor;
+    int lb = limit_backward;
+
+    int common_i = 0;
+    int common_j = 0;
+
+    bool first_key_inspected = false;
+
+    while (true) {
+      int k = i + ((j - i) >> 1);
+      int diff = 0;
+      int common = common_i < common_j ? common_i : common_j;
+      Among w = v[k];
+      for (int i2 = w.s.length - 1 - common; i2 >= 0; i2--) {
+        if (c - common == lb) {
+          diff = -1;
+          break;
+        }
+        diff = current.codeUnitAt(c - 1 - common) - w.s.codeUnitAt(i2);
+        if (diff != 0) break;
+        common++;
+      }
+      if (diff < 0) {
+        j = k;
+        common_j = common;
+      } else {
+        i = k;
+        common_i = common;
+      }
+      if (j - i <= 1) {
+        if (i > 0) break;
+        if (j == i) break;
+        if (first_key_inspected) break;
+        first_key_inspected = true;
+      }
+    }
+    while (true) {
+      Among w = v[i];
+      if (common_i >= w.s.length) {
+        cursor = c - w.s.length;
+        if (w.function_index == 0) return w.result;
+        af = w.function_index;
+        if (call_among_func?.call() ?? false) {
+          cursor = c - w.s.length;
+          return w.result;
+        }
+      }
+      i = w.substring_i;
+      if (i < 0) return 0;
+    }
+  }
+
+  int replace_s(int c_bra, int c_ket, String s) {
+    final adjustment = s.length - (c_ket - c_bra);
+    current = current.replaceRange(c_bra, c_ket, s);
+    limit += adjustment;
+    if (cursor >= c_ket) {
+      cursor += adjustment;
+    } else if (cursor > c_bra) {
+      cursor = c_bra;
+    }
+    return adjustment;
+  }
+
+  void slice_check() {
+    assert(bra >= 0, 'bra=$bra');
+    assert(bra <= ket, 'bra=$bra,ket=$ket');
+    assert(ket <= limit, 'ket=$ket,limit=$limit');
+    assert(limit <= current.length, 'limit=$limit,length=${current.length}');
+  }
+
+  void slice_from(String s) {
+    slice_check();
+    replace_s(bra, ket, s);
+    ket = bra + s.length;
+  }
+
+  void slice_del() {
+    slice_from("");
+  }
+
+  void insert(int c_bra, int c_ket, String s) {
+    int adjustment = replace_s(c_bra, c_ket, s);
+    if (c_bra <= bra) bra += adjustment;
+    if (c_bra <= ket) ket += adjustment;
+  }
+
+  String slice_to() {
+    slice_check();
+    return current.substring(bra, ket);
+  }
+
+  String assign_to() {
+    return current.substring(0, limit);
+  }
+}
+
+abstract class SnowballStemmer extends SnowballProgram {
+  bool stem();
+}

data/vendor/snowball/dart/pubspec.yaml

@@ -0,0 +1,17 @@
+name: snowball
+description: This package provides stemmers generated from Snowball algorithms.
+version: 3.1.1 # SNOWBALL_VERSION
+repository: https://github.com/snowballstem/snowball
+issue_tracker: https://github.com/snowballstem/snowball/issues
+topics:
+  - stemmer
+
+environment:
+  sdk: ^3.8.1
+
+dependencies:
+  # path: ^1.8.0
+
+dev_dependencies:
+  lints: ^5.0.0
+  test: ^1.24.0

data/vendor/snowball/doc/libstemmer_dart_README

@@ -0,0 +1,37 @@
+# snowball
+
+This package provides stemmers generated from Snowball algorithms.
+
+### What is Stemming?
+
+Stemming maps different forms of the same word to a common "stem" - for
+example, the English stemmer maps *connection*, *connections*, *connective*,
+*connected*, and *connecting* to *connect*.  So a search for *connected*
+would also find documents which only have the other forms.
+
+This stem form is often a word itself, but this is not always the case as this
+is not a requirement for text search systems, which are the intended field of
+use.  We also aim to conflate words with the same meaning, rather than all
+words with a common linguistic root (so *awe* and *awful* don't have the same
+stem), and over-stemming is more problematic than under-stemming so we tend not
+to stem in cases that are hard to resolve.  If you want to always reduce words
+to a root form and/or get a root form which is itself a word then Snowball's
+stemming algorithms likely aren't the right answer.
+
+### How to use library
+
+The stemming algorithms generally expect the input text to use composed accents
+(Unicode NFC or NFKC) and to have been folded to lower case already.
+
+The ``snowball`` package has two functions.
+
+```dart
+import 'package:snowball/snowball.dart';
+
+final stemmer = SnowballStemmer(Algorithm.english);
+print(stemmer.stem('walking'));
+```
+
+Generally you should create a stemmer object and reuse it rather than creating
+a fresh object for each word stemmed (since there's some cost to creating and
+destroying the object).

data/vendor/snowball/doc/libstemmer_js_README

@@ -30,9 +30,9 @@ with node:
 
 .. code-block:: javascript
 
-    var EnglishStemmer = require('english-stemmer.js');
+    let EnglishStemmer = require('english-stemmer.js');
 
-    var stemmer = new EnglishStemmer();
+    let stemmer = new EnglishStemmer();
     console.log(stemmer.stemWord("testing"));
 
 You'll need to bundle ``base-stemmer.js`` and whichever languages you want