mittens 0.1.0

data/vendor/snowball/pascal/SnowballProgram.pas

@@ -0,0 +1,430 @@
+unit SnowballProgram;
+
+interface
+
+Type
+    TAmongHandler = Function : Boolean of Object;
+
+Type
+    TAmong = record
+        Str    : AnsiString;    // search string
+        Index  : Integer;       // index to longest matching substring
+        Result : Integer;       // result of the lookup
+        Method : TAmongHandler; // method to use if substring matches
+    End;
+
+Type
+    {$M+}
+    TSnowballProgram = Class
+    Protected
+        FCurrent : AnsiString;
+        FCursor  : Integer;
+        FLimit   : Integer;
+        FBkLimit : Integer;
+        FBra     : Integer;
+        FKet     : Integer;
+
+        Procedure SetCurrent(Current: AnsiString);
+
+    Protected
+        Function InGrouping(s : array of char; min, max : Integer) : Boolean;
+        Function InGroupingBk(s : array of char; min, max : Integer) : Boolean;
+        Function OutGrouping(s : array of char; min, max : Integer) : Boolean;
+        Function OutGroupingBk(s : array of char; min, max : Integer) : Boolean;
+
+        Function EqS(s_size : Integer; s : AnsiString) : Boolean;
+        Function EqSBk(s_size : Integer; s : AnsiString) : Boolean;
+
+        Function EqV(s : AnsiString) : Boolean;
+        Function EqVBk(s : AnsiString) : Boolean;
+
+        Function FindAmong(v : array of TAmong; v_size : Integer) : Integer;
+        Function FindAmongBk(v : array of TAmong; v_size : Integer) : Integer;
+
+        Procedure SliceDel;
+        Procedure SliceCheck;
+        Procedure SliceFrom(s : AnsiString);
+
+        Function  ReplaceS(bra, ket : Integer; s : AnsiString) : Integer;        
+        Procedure Insert(bra, ket : Integer; s : AnsiString);
+
+        Function SliceTo : AnsiString;
+        Function AssignTo : AnsiString;
+
+    Public
+        { Set & Retrieve current string }
+        Property Current: AnsiString Read FCurrent Write SetCurrent;
+
+        { Method subclasses need to implement }
+        Function stem : Boolean; Virtual; Abstract;
+    End;
+
+Implementation
+
+Uses Math;
+
+Procedure TSnowballProgram.SetCurrent(Current: AnsiString);
+Begin
+    FCurrent := Current;
+    FCursor  := 0;
+    FLimit   := Length(Current);
+    FBkLimit := 0;
+    FBra     := FCursor;
+    FKet     := FLimit;
+End;
+
+Function TSnowballProgram.InGrouping(s : array of char; min, max : Integer) : Boolean;
+Var ch : Integer;
+Begin
+    Result := False;
+    If (FCursor >= FLimit) Then Exit;
+
+    ch := Ord(FCurrent[FCursor + 1]);
+    If (ch > max) Or (ch < min) Then Exit;
+
+    ch := ch - min;
+    If (Ord(s[ch Shr 3]) And Ord(1 Shl (ch And $7))) = 0 Then Exit;
+
+    Inc(FCursor);
+    Result := True;
+End;
+
+Function TSnowballProgram.InGroupingBk(s : array of char; min, max : Integer) : Boolean;
+Var ch : Integer;
+Begin
+    Result := False;
+    If (FCursor <= FBkLimit) Then Exit;
+
+    ch := Ord(FCurrent[FCursor]);
+    If (ch > max) Or (ch < min) Then Exit;
+
+    ch := ch - min;
+    If (Ord(s[ch Shr 3]) And Ord(1 Shl (ch And $7))) = 0 Then Exit;
+
+    Dec(FCursor);
+    Result := True;
+End;
+
+Function TSnowballProgram.OutGrouping(s : array of char; min, max : Integer) : Boolean;
+Var ch : Integer;
+Begin
+    Result := False;
+    If (FCursor >= FLimit) Then Exit;
+
+    ch := Ord(FCurrent[FCursor + 1]);
+
+    If (ch > max) Or (ch < min) Then
+    Begin
+        Inc(FCursor);
+        Result := True;
+        Exit;
+    End;
+
+    ch := ch - min;
+    If (Ord(s[ch Shr 3]) And Ord(1 Shl (ch And $7))) = 0 Then
+    Begin
+        Inc(FCursor);
+        Result := True;
+    End;
+End;
+
+Function TSnowballProgram.OutGroupingBk(s : array of char; min, max : Integer) : Boolean;
+Var ch : Integer;
+Begin
+    Result := False;
+
+    If (FCursor <= FBkLimit) Then Exit;
+
+    ch := Ord(FCurrent[FCursor]);
+    If (ch > max) Or (ch < min) Then
+    Begin
+        Dec(FCursor);
+        Result := True;
+        Exit;
+    End;
+
+    ch := ch - min;
+    If (Ord(s[ch Shr 3]) And Ord(1 Shl (ch And $7))) = 0 Then
+    Begin
+        Dec(FCursor);
+        Result := True;
+    End;
+End;
+
+Function TSnowballProgram.EqS(s_size : Integer; s : AnsiString) : Boolean;
+Var I : Integer;
+Begin
+    Result := False;
+
+    If (FLimit - FCursor) < s_size Then Exit;
+
+    For I := 1 To s_size Do
+        If FCurrent[FCursor + I] <> s[I] Then Exit;
+
+    FCursor := FCursor + s_size;
+
+    Result := True;
+End;
+
+Function TSnowballProgram.EqSBk(s_size : Integer; s : AnsiString) : Boolean;
+Var I : Integer;
+Begin
+    Result := False;
+
+    if (FCursor - FBkLimit) < s_size Then Exit;
+
+    For I := 1 To s_size Do
+        If FCurrent[FCursor - s_size + I] <> s[i] Then Exit;
+
+    FCursor := FCursor - s_size;
+
+    Result := True;
+End;
+
+Function TSnowballProgram.EqV(s : AnsiString) : Boolean;
+Begin
+    Result := EqS(Length(s), s);
+End;
+
+Function TSnowballProgram.EqVBk(s : AnsiString) : Boolean;
+Begin
+    Result := EqSBk(Length(s), s);
+End;
+
+Function TSnowballProgram.FindAmong(v : array of TAmong; v_size : Integer) : Integer;
+Var i, i2, j, c, l, common_i, common_j, k, diff, common : Integer;
+    first_key_inspected, res : Boolean;
+    w : TAmong;
+Begin
+    i := 0;
+    j := v_size;
+
+    c := FCursor;
+    l := FLimit;
+
+    common_i := 0;
+    common_j := 0;
+
+    first_key_inspected := false;
+
+    While True Do
+    Begin
+        k := i + ((j - i) Shr 1);
+        diff := 0;
+        common := Min(common_i, common_j); // smaller
+        w := v[k];
+
+        For i2 := common To Length(w.Str) - 1 Do
+        Begin
+            if (c + common) = l Then
+            Begin
+                diff := -1;
+                Break;
+            End;
+
+            diff := Ord(FCurrent[c + common + 1]) - Ord(w.Str[i2 + 1]);
+            if diff <> 0 Then Break;
+
+            Inc(common);
+        End;
+
+        if diff < 0 Then
+        Begin
+            j := k;
+            common_j := common;
+        End
+        Else
+        Begin
+            i := k;
+            common_i := common;
+        End;
+
+        If (j - i) <= 1 Then
+        Begin
+            If (i > 0) Then Break; // v->s has been inspected
+            if (j = i) Then Break; // only one item in v
+
+            // - but now we need to go round once more to get
+            // v->s inspected. This looks messy, but is actually
+            // the optimal approach.
+
+            if (first_key_inspected) Then Break;
+            first_key_inspected := True;
+        End;
+    End;
+
+    While True Do
+    Begin
+        w := v[i];
+        If (common_i >= Length(w.Str)) Then
+        Begin
+            FCursor := c + Length(w.Str);
+            If Not Assigned(w.Method) Then
+            Begin
+                Result := w.Result;
+                Exit;   
+            End;
+
+            res := w.Method;
+
+            FCursor := c + Length(w.Str);
+            if (res) Then Begin
+                Result := w.Result;
+                Exit;
+            End;
+        End;
+
+        i := w.Index;
+        if i < 0 Then
+        Begin
+            Result := 0;
+            Exit;
+        End;
+    End;
+End;
+
+Function TSnowballProgram.FindAmongBk(v : array of TAmong; v_size : Integer) : Integer;
+Var i, j, c, lb, common_i, common_j, k, diff, common, i2 : Integer;
+    first_key_inspected, res : Boolean;
+    w : TAmong;
+Begin
+    i := 0;
+    j := v_size;
+
+    c := FCursor;
+    lb := FBkLimit;
+
+    common_i := 0;
+    common_j := 0;
+
+    first_key_inspected := false;
+
+    While True Do
+    Begin
+        k := i + ((j - i) Shr 1);
+        diff := 0;
+        common := Min(common_i, common_j);
+        w := v[k];
+
+        For i2 := Length(w.Str) - 1 - common DownTo 0 Do
+        Begin
+            If (c - common) = lb Then
+            Begin
+                diff := -1;
+                Break;
+            End;
+
+            diff := Ord(FCurrent[c - common]) - Ord(w.Str[i2 + 1]);
+            if diff <> 0 Then Break;
+            Inc(common);
+        End;
+
+        If diff < 0 Then
+        Begin
+            j := k;
+            common_j := common;
+        End
+        Else
+        Begin
+            i := k;
+            common_i := common;
+        End;
+
+        If (j - i) <= 1 Then
+        Begin
+            if i > 0 Then Break;
+            if j = i Then Break;
+            if first_key_inspected Then Break;
+            first_key_inspected := True;
+        End;
+    End;
+
+    While True Do
+    Begin
+        w := v[i];
+        if common_i >= Length(w.Str) Then
+        Begin
+            FCursor := c - Length(w.Str);
+            If Not Assigned(w.Method) Then
+            Begin
+                Result := w.Result;
+                Exit;
+            End;
+
+            res := w.Method;
+
+            FCursor := c - Length(w.Str);
+            If Res Then
+            Begin
+                Result := w.Result;
+                Exit;
+            End;
+        End;
+
+        i := w.Index;
+        If i < 0 Then
+        Begin
+                Result := 0;
+                Exit;
+        End;
+    End;
+End;
+
+Procedure TSnowballProgram.SliceCheck;
+Begin
+    if (FBra < 0) Or (FBra > FKet) Or (FKet > FLimit) Or (FLimit > Length(FCurrent)) Then
+    Begin
+        WriteLn('Faulty slice operation.');
+        Halt;
+    End;
+End;
+
+Procedure TSnowballProgram.SliceDel;
+Begin
+    SliceFrom('');
+End;
+
+Function TSnowballProgram.ReplaceS(bra, ket : Integer; s : AnsiString) : Integer;
+Var adjustment : Integer;
+Begin
+    adjustment := Length(s) - (ket - bra);
+
+    Delete(FCurrent, bra + 1, ket - bra);
+    System.Insert(s, FCurrent, bra + 1);
+
+    FLimit := FLimit + adjustment;
+
+    if (FCursor >= ket) Then
+        FCursor := FCursor + adjustment
+    Else If (FCursor > bra) Then
+        FCursor := bra;
+
+    Result := adjustment;
+End;
+
+Procedure TSnowballProgram.Insert(bra, ket : Integer; s : AnsiString);
+Var adjustment : Integer;
+Begin
+    adjustment := ReplaceS(bra, ket, s);
+    If (bra <= FBra) Then FBra := FBra + adjustment;
+    If (bra <= FKet) Then FKet := FKet + adjustment;
+End;
+
+Function TSnowballProgram.SliceTo() : AnsiString;
+Begin
+    SliceCheck();
+    Result := Copy(FCurrent, FBra + 1, FKet - FBra);
+End;
+
+Procedure TSnowballProgram.SliceFrom(s : AnsiString);
+Begin
+    SliceCheck();
+    ReplaceS(FBra, FKet, s);
+End;
+
+Function TSnowballProgram.AssignTo() : AnsiString;
+Begin
+    Result := Copy(FCurrent, 1, FLimit);
+End;
+
+End.

data/vendor/snowball/pascal/generate.pl

@@ -0,0 +1,23 @@
+#!/usr/bin/env perl
+use strict;
+use warnings;
+
+# Generate Pascal stemwords source.
+
+my @sources = @ARGV;
+
+while (defined(my $line = <STDIN>)) {
+	if ($line =~ /\{\s*BEGIN TEMPLATE\s*\}/) {
+		my $template = '';
+		while (defined($line = <STDIN>) && $line !~ /\{\s*END TEMPLATE\s*\}/) {
+			$template .= $line;
+		}
+		foreach my $source(@sources) {
+			my $out = $template;
+			$out =~ s/%STEMMER%/$source/g;
+			print $out;
+		}
+		next;
+	}
+	print $line;
+}

data/vendor/snowball/pascal/stemwords-template.dpr

@@ -0,0 +1,78 @@
+program stemwords;
+
+{$ifdef windows}
+{$APPTYPE CONSOLE}
+{$endif}
+
+uses
+  SnowballProgram,
+  { BEGIN TEMPLATE }
+  %STEMMER%Stemmer in '%STEMMER%Stemmer.pas',
+  { END TEMPLATE }
+  SysUtils;
+
+Var
+    Stemmer : TSnowballProgram;
+    CurWord : AnsiString;
+    i : Integer;
+    language : AnsiString;
+
+Const
+    Delimiters : Set Of Char = [#10, #13];
+
+Function NextWord : Boolean;
+Var C : Char;
+Begin
+    CurWord := '';
+
+    Result := Not Eof;
+
+    While Not Eof Do
+    Begin
+        Read(C);
+        If IOResult <> 0 Then Break;
+        If C In Delimiters Then Break;
+        CurWord := CurWord + C;
+    End;
+End;
+
+begin
+    language := 'english';
+    i := 0;
+    while i < ParamCount do
+    begin
+        i := i + 1;
+        if ParamStr(i) = '-l' then
+        begin
+            i := i + 1;
+            language := ParamStr(i);
+            continue;
+        end;
+        WriteLn('option '+ParamStr(i)+' unknown');
+        Exit;
+    end;
+    if False then
+    { BEGIN TEMPLATE }
+    else if language = '%STEMMER%' then
+        Stemmer := T%STEMMER%Stemmer.Create
+    { END TEMPLATE }
+    else
+    begin
+        WriteLn('Stemming language '+language+' unknown');
+        Exit;
+    end;
+
+    Try
+        While Not Eof Do
+        Begin
+            While NextWord Do
+            Begin
+                Stemmer.Current := CurWord;
+                Stemmer.Stem;
+                WriteLn(Stemmer.Current);
+            End;
+        End;
+    Finally
+        Stemmer.Free;
+    End;
+end.

data/vendor/snowball/python/MANIFEST.in

@@ -0,0 +1,7 @@
+include *.rst
+include modules.txt
+include setup.*
+recursive-include src *.py
+include MANIFEST.in
+include COPYING
+include NEWS

data/vendor/snowball/python/create_init.py

@@ -0,0 +1,54 @@
+#! /bin/sh/env python
+
+import sys
+import re
+import os
+
+python_out_folder = sys.argv[1]
+
+filematch = re.compile(r"(\w+)_stemmer\.py$")
+
+imports = []
+languages = []
+
+for pyscript in os.listdir(python_out_folder):
+    match = filematch.match(pyscript)
+    if (match):
+        langname = match.group(1)
+        titlecase = langname.title()
+        languages.append("    '%(lang)s': %(title)sStemmer," % {'lang': langname, 'title': titlecase})
+        imports.append('from .%(lang)s_stemmer import %(title)sStemmer' % {'lang': langname, 'title': titlecase})
+imports.sort()
+languages.sort()
+
+src = '''__all__ = ('language', 'stemmer')
+
+%(imports)s
+
+_languages = {
+%(languages)s
+}
+
+try:
+    import Stemmer
+    cext_available = True
+except ImportError:
+    cext_available = False
+
+def algorithms():
+    if cext_available:
+        return Stemmer.language()
+    else:
+        return list(_languages.keys())
+
+def stemmer(lang):
+    if cext_available:
+        return Stemmer.Stemmer(lang)
+    if lang.lower() in _languages:
+        return _languages[lang.lower()]()
+    else:
+        raise KeyError("Stemming algorithm '%%s' not found" %% lang)
+''' % {'imports': '\n'.join(imports), 'languages': '\n'.join(languages)}
+
+with open(os.path.join(python_out_folder, '__init__.py'), 'w') as out:
+    out.write(src)

data/vendor/snowball/python/setup.cfg

@@ -0,0 +1,6 @@
+[metadata]
+long_description = file: README.rst
+long_description_content_type = text/x-rst
+
+[bdist_wheel]
+universal=1

data/vendor/snowball/python/setup.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python
+
+from setuptools import setup
+import re
+
+SNOWBALL_VERSION = '2.2.0'
+
+n_stemmers = 0
+
+langs = []
+variants = {}
+with open('modules.txt') as fp:
+    for line in fp.readlines():
+        if len(line) <= 1 or line[0] == '#':
+            continue
+        if line[-1:] == '\n':
+            line = line[:-1]
+        tokens = re.split(r'\s+', line)
+        if len(tokens) < 3:
+            print("Bad modules.txt line: " + line)
+            continue
+        (name, encs, codes) = tokens[:3]
+        if len(tokens) > 3:
+            variant_of = tokens[3]
+            if variant_of in variants:
+                variants[variant_of].append(name)
+            else:
+                variants[variant_of] = [name]
+        else:
+            langs.append(name)
+        n_stemmers += 1
+
+desc = 'This package provides ' + str(n_stemmers) + ' stemmers for ' + \
+    str(len(langs)) + ' languages generated from Snowball algorithms.'
+
+classifiers = [
+    'Development Status :: 5 - Production/Stable',
+    'Intended Audience :: Developers',
+    'License :: OSI Approved :: BSD License'
+]
+
+for lang in langs:
+    lang_titlecase = lang.title()
+    # Only classifiers listed in https://pypi.org/classifiers/ are allowed
+    if lang_titlecase not in ('Armenian', 'Yiddish'):
+        classifiers.append('Natural Language :: ' + lang_titlecase)
+
+classifiers.extend([
+    'Operating System :: OS Independent',
+    'Programming Language :: Python',
+    'Programming Language :: Python :: 2',
+    'Programming Language :: Python :: 2.6',
+    'Programming Language :: Python :: 2.7',
+    'Programming Language :: Python :: 3',
+    'Programming Language :: Python :: 3.4',
+    'Programming Language :: Python :: 3.5',
+    'Programming Language :: Python :: 3.6',
+    'Programming Language :: Python :: 3.7',
+    'Programming Language :: Python :: 3.8',
+    'Programming Language :: Python :: 3.9',
+    'Programming Language :: Python :: 3.10',
+    'Programming Language :: Python :: Implementation :: CPython',
+    'Programming Language :: Python :: Implementation :: PyPy',
+    'Topic :: Database',
+    'Topic :: Internet :: WWW/HTTP :: Indexing/Search',
+    'Topic :: Text Processing :: Indexing',
+    'Topic :: Text Processing :: Linguistic'
+])
+
+setup(name='snowballstemmer',
+      version=SNOWBALL_VERSION,
+      description=desc,
+      author='Snowball Developers',
+      author_email='snowball-discuss@lists.tartarus.org',
+      url='https://github.com/snowballstem/snowball',
+      keywords="stemmer",
+      license="BSD-3-Clause",
+      packages=['snowballstemmer'],
+      package_dir={"snowballstemmer": "src/snowballstemmer"},
+      classifiers = classifiers
+)

data/vendor/snowball/python/snowballstemmer/among.py

@@ -0,0 +1,13 @@
+
+class Among(object):
+    def __init__(self, s, substring_i, result, method=None):
+        """
+        @ivar s search string
+        @ivar substring index to longest matching substring
+        @ivar result of the lookup
+        @ivar method method to use if substring matches
+        """
+        self.s = s
+        self.substring_i = substring_i
+        self.result = result
+        self.method = method