mittens 0.1.0

data/vendor/snowball/libstemmer/mkmodules.pl

@@ -0,0 +1,267 @@
+#!/usr/bin/env perl
+use strict;
+use 5.006;
+use warnings;
+
+my $progname = $0;
+
+if (scalar @ARGV < 4 || scalar @ARGV > 5) {
+  print "Usage: $progname <outfile> <C source directory> <modules description file> <source list file> [<enc>]\n";
+  exit 1;
+}
+
+my $outname = shift(@ARGV);
+my $c_src_dir = shift(@ARGV);
+my $descfile = shift(@ARGV);
+my $srclistfile = shift(@ARGV);
+my $enc_only;
+my $extn = '';
+if (@ARGV) {
+  $enc_only = shift(@ARGV);
+  $extn = '_'.$enc_only;
+}
+
+my %aliases = ();
+my %algorithms = ();
+my %algorithm_encs = ();
+
+my %encs = ();
+
+sub addalgenc($$) {
+  my $alg = shift();
+  my $enc = shift();
+
+  if (defined $enc_only) {
+      my $norm_enc = lc $enc;
+      $norm_enc =~ s/_//g;
+      if ($norm_enc ne $enc_only) {
+	  return;
+      }
+  }
+
+  if (defined $algorithm_encs{$alg}) {
+      my $hashref = $algorithm_encs{$alg};
+      $$hashref{$enc}=1;
+  } else {
+      my %newhash = ($enc => 1);
+      $algorithm_encs{$alg}=\%newhash;
+  }
+
+  $encs{$enc} = 1;
+}
+
+sub readinput()
+{
+    open DESCFILE, $descfile;
+    my $line;
+    while ($line = <DESCFILE>)
+    {
+        next if $line =~ m/^\s*#/;
+        next if $line =~ m/^\s*$/;
+        my ($alg,$encstr,$aliases) = split(/\s+/, $line);
+        my $enc;
+        my $alias;
+
+        $algorithms{$alg} = 1;
+        foreach $alias (split(/,/, $aliases)) {
+            foreach $enc (split(/,/, $encstr)) {
+                # print "$alias, $enc\n";
+                $aliases{$alias} = $alg;
+                addalgenc($alg, $enc);
+            }
+        }
+    }
+}
+
+sub printoutput()
+{
+    open (OUT, ">$outname") or die "Can't open output file `$outname': $!\n";
+
+    print OUT <<EOS;
+/* $outname: List of stemming modules.
+ *
+ * This file is generated by mkmodules.pl from a list of module names.
+ * Do not edit manually.
+ *
+EOS
+
+    my $line = " * Modules included by this file are: ";
+    print OUT $line;
+    my $linelen = length($line);
+
+    my $need_sep = 0;
+    my $lang;
+    my $enc;
+    my @algorithms = sort keys(%algorithms);
+    foreach $lang (@algorithms) {
+        if ($need_sep) {
+            if (($linelen + 2 + length($lang)) > 77) {
+                print OUT ",\n * ";
+                $linelen = 3;
+            } else {
+                print OUT ', ';
+                $linelen += 2;
+            }
+        }
+        print OUT $lang;
+        $linelen += length($lang);
+        $need_sep = 1;
+    }
+    print OUT "\n */\n\n";
+
+    foreach $lang (@algorithms) {
+        my $hashref = $algorithm_encs{$lang};
+        foreach $enc (sort keys (%$hashref)) {
+            print OUT "#include \"../$c_src_dir/stem_${enc}_$lang.h\"\n";
+        }
+    }
+
+    print OUT <<EOS;
+
+typedef enum {
+  ENC_UNKNOWN=0,
+EOS
+    my $neednl = 0;
+    for $enc (sort keys %encs) {
+        print OUT ",\n" if $neednl;
+        print OUT "  ENC_${enc}";
+        $neednl = 1;
+    }
+    print OUT <<EOS;
+
+} stemmer_encoding_t;
+
+struct stemmer_encoding {
+  const char * name;
+  stemmer_encoding_t enc;
+};
+static const struct stemmer_encoding encodings[] = {
+EOS
+    for $enc (sort keys %encs) {
+        print OUT "  {\"${enc}\", ENC_${enc}},\n";
+    }
+    print OUT <<EOS;
+  {0,ENC_UNKNOWN}
+};
+
+struct stemmer_modules {
+  const char * name;
+  stemmer_encoding_t enc; 
+  struct SN_env * (*create)(void);
+  void (*close)(struct SN_env *);
+  int (*stem)(struct SN_env *);
+};
+static const struct stemmer_modules modules[] = {
+EOS
+
+    for $lang (sort keys %aliases) {
+        my $l = $aliases{$lang};
+        my $hashref = $algorithm_encs{$l};
+        my $enc;
+        foreach $enc (sort keys (%$hashref)) {
+            my $p = "${l}_${enc}";
+            print OUT "  {\"$lang\", ENC_$enc, ${p}_create_env, ${p}_close_env, ${p}_stem},\n";
+        }
+    }
+
+    print OUT <<EOS;
+  {0,ENC_UNKNOWN,0,0,0}
+};
+EOS
+
+    print OUT <<EOS;
+static const char * algorithm_names[] = {
+EOS
+
+    for $lang (@algorithms) {
+        print OUT "  \"$lang\", \n";
+    }
+
+    print OUT <<EOS;
+  0
+};
+EOS
+    close OUT or die "Can't close ${outname}: $!\n";
+}
+
+sub printsrclist()
+{
+    open (OUT, ">$srclistfile") or die "Can't open output file `$srclistfile': $!\n";
+
+    print OUT <<EOS;
+# $srclistfile: List of stemming module source files
+#
+# This file is generated by mkmodules.pl from a list of module names.
+# Do not edit manually.
+#
+EOS
+
+    my $line = "# Modules included by this file are: ";
+    print OUT $line;
+    my $linelen = length($line);
+
+    my $need_sep = 0;
+    my $lang;
+    my $srcfile;
+    my $enc;
+    my @algorithms = sort keys(%algorithms);
+    foreach $lang (@algorithms) {
+        if ($need_sep) {
+            if (($linelen + 2 + length($lang)) > 77) {
+                print OUT ",\n# ";
+                $linelen = 3;
+            } else {
+                print OUT ', ';
+                $linelen += 2;
+            }
+        }
+        print OUT $lang;
+        $linelen += length($lang);
+        $need_sep = 1;
+    }
+
+    print OUT "\n\nsnowball_sources= \\\n";
+    for $lang (sort keys %aliases) {
+        my $hashref = $algorithm_encs{$lang};
+        my $enc;
+        foreach $enc (sort keys (%$hashref)) {
+            print OUT "  src_c/stem_${enc}_${lang}.c \\\n";
+        }
+    }
+
+    $need_sep = 0;
+    for $srcfile ('runtime/api.c',
+                  'runtime/utilities.c',
+                  "libstemmer/libstemmer${extn}.c") {
+        print OUT " \\\n" if $need_sep;
+        print OUT "  $srcfile";
+        $need_sep = 1;
+    }
+
+    print OUT "\n\nsnowball_headers= \\\n";
+    for $lang (sort keys %aliases) {
+        my $hashref = $algorithm_encs{$lang};
+        my $enc;
+        foreach $enc (sort keys (%$hashref)) {
+            my $p = "${lang}_${enc}";
+            print OUT "  src_c/stem_${enc}_${lang}.h \\\n";
+        }
+    }
+
+    $need_sep = 0;
+    for $srcfile ('include/libstemmer.h',
+                  "libstemmer/modules${extn}.h",
+                  'runtime/api.h',
+                  'runtime/header.h') {
+        print OUT " \\\n" if $need_sep;
+        print OUT "  $srcfile";
+        $need_sep = 1;
+    }
+
+    print OUT "\n\n";
+    close OUT or die "Can't close ${srclistfile}: $!\n";
+}
+
+readinput();
+printoutput();
+printsrclist();

data/vendor/snowball/libstemmer/modules.txt

@@ -0,0 +1,63 @@
+# This file contains a list of stemmers to include in the distribution.
+# The format is a set of space separated lines - on each line:
+#  First item is name of stemmer.
+#  Second item is comma separated list of character sets.
+#  Third item is comma separated list of names to refer to the stemmer by.
+#
+# Lines starting with a #, or blank lines, are ignored.
+
+# List all the main algorithms for each language, in UTF-8, and also with
+# the most commonly used encoding.
+
+arabic          UTF_8                   arabic,ar,ara
+armenian        UTF_8                   armenian,hy,hye,arm
+basque          UTF_8,ISO_8859_1        basque,eu,eus,baq
+catalan         UTF_8,ISO_8859_1        catalan,ca,cat
+danish          UTF_8,ISO_8859_1        danish,da,dan
+dutch           UTF_8,ISO_8859_1        dutch,nl,dut,nld
+english         UTF_8,ISO_8859_1        english,en,eng
+finnish         UTF_8,ISO_8859_1        finnish,fi,fin
+french          UTF_8,ISO_8859_1        french,fr,fre,fra
+german          UTF_8,ISO_8859_1        german,de,ger,deu
+greek           UTF_8                   greek,el,gre,ell
+hindi           UTF_8                   hindi,hi,hin
+hungarian       UTF_8,ISO_8859_2        hungarian,hu,hun
+indonesian      UTF_8,ISO_8859_1        indonesian,id,ind
+irish           UTF_8,ISO_8859_1        irish,ga,gle
+italian         UTF_8,ISO_8859_1        italian,it,ita
+lithuanian      UTF_8                   lithuanian,lt,lit
+nepali          UTF_8                   nepali,ne,nep
+norwegian       UTF_8,ISO_8859_1        norwegian,no,nor
+portuguese      UTF_8,ISO_8859_1        portuguese,pt,por
+romanian        UTF_8,ISO_8859_2        romanian,ro,rum,ron
+russian         UTF_8,KOI8_R            russian,ru,rus
+serbian         UTF_8                   serbian,sr,srp
+spanish         UTF_8,ISO_8859_1        spanish,es,esl,spa
+swedish         UTF_8,ISO_8859_1        swedish,sv,swe
+tamil           UTF_8                   tamil,ta,tam
+turkish         UTF_8                   turkish,tr,tur
+yiddish         UTF_8                   yiddish,yi,yid
+
+# Also include the traditional porter algorithm for english.
+# The porter algorithm is included in the libstemmer distribution to assist
+# with backwards compatibility, but for new systems the english algorithm
+# should be used in preference.
+porter          UTF_8,ISO_8859_1        porter			english
+
+# Some other stemmers in the snowball project are not included in the standard
+# distribution. To compile a libstemmer with them in, add them to this list,
+# and regenerate the distribution. (You will need a full source checkout for
+# this.) They are included in the snowball website as curiosities, but are not
+# intended for general use, and use of them is is not fully supported.  These
+# algorithms are:
+#
+# german2          - This is a slight modification of the german stemmer.
+#german2          UTF_8,ISO_8859_1        german2		german
+#
+# kraaij_pohlmann  - This is a different dutch stemmer.
+#kraaij_pohlmann  UTF_8,ISO_8859_1        kraaij_pohlmann	dutch
+#
+# lovins           - This is an english stemmer, but fairly outdated, and
+#                    only really applicable to a restricted type of input text
+#                    (keywords in academic publications).
+#lovins           UTF_8,ISO_8859_1        lovins		english

data/vendor/snowball/libstemmer/test.c

@@ -0,0 +1,34 @@
+
+#include "libstemmer.h"
+
+/* test code */
+void error(const char * err) {
+    printf("%s\n", err);
+    exit(1);
+}
+
+int main () {
+    const char * stemmed;
+    const char * unstemmed;
+    struct sb_stemmer * s;
+    const char ** list = sb_stemmer_list();
+    if (*list == 0) error("TEST FAIL: empty list of stemmers");
+    
+    s = sb_stemmer_new("e");
+    if (s != 0) error("TEST FAIL: non zero return for unrecognised language");
+    s = sb_stemmer_new("english");
+    if (s == 0) error("TEST FAIL: zero return for recognised language");
+    sb_stemmer_delete(s);
+    s = sb_stemmer_new("en");
+    if (s == 0) error("TEST FAIL: zero return for recognised language");
+    unstemmed = "recognised";
+    stemmed = sb_stemmer_stem(s, unstemmed, 10);
+    printf("%s -> %s\n", unstemmed, stemmed);
+    if (sb_stemmer_length(s) != strlen(stemmed))
+        error("TEST FAIL: length not correct");
+    unstemmed = "recognized";
+    printf("%s -> %s\n", unstemmed, stemmed);
+    sb_stemmer_delete(s);
+    printf("Success\n");
+    return 0;
+}

data/vendor/snowball/pascal/.gitignore

@@ -0,0 +1,4 @@
+/*.ppu
+/*Stemmer.pas
+/stemwords.dpr
+/stemwords