mittens 0.1.0

data/vendor/snowball/csharp/Stemwords/Program.cs

@@ -0,0 +1,114 @@
+// Copyright (c) 2001, Dr Martin Porter
+// Copyright (c) 2002, Richard Boulton
+// Copyright (c) 2015, Cesar Souza
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+// 
+//     * Redistributions of source code must retain the above copyright notice,
+//     * this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above copyright
+//     * notice, this list of conditions and the following disclaimer in the
+//     * documentation and/or other materials provided with the distribution.
+//     * Neither the name of the copyright holders nor the names of its contributors
+//     * may be used to endorse or promote products derived from this software
+//     * without specific prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+namespace Snowball
+{
+    using System;
+    using System.IO;
+    using System.Reflection;
+    using System.Linq;
+    using System.Text;
+
+    /// <summary>
+    ///   Snowball's Stemmer program.
+    /// </summary>
+    /// 
+    public static class Program
+    {
+
+        private static void usage()
+        {
+            Console.WriteLine("Usage: stemwords.exe -l <language> -i <input file> [-o <output file>]");
+        }
+
+        /// <summary>
+        ///   Main program entrypoint.
+        /// </summary>
+        /// 
+        public static void Main(String[] args)
+        {
+            string language = null;
+            string inputName = null;
+            string outputName = null;
+
+            for (int i = 0; i < args.Length; i++)
+            {
+                if (args[i] == "-l")
+                    language = args[i + 1];
+                else if (args[i] == "-i")
+                    inputName = args[i + 1];
+                if (args[i] == "-o")
+                    outputName = args[i + 1];
+            }
+
+            if (language == null || inputName == null)
+            {
+                usage();
+                return;
+            }
+
+
+
+            Stemmer stemmer =
+                typeof(Stemmer).Assembly.GetTypes()
+                    .Where(t => t.IsSubclassOf(typeof(Stemmer)) && !t.IsAbstract)
+                    .Where(t => match(t.Name, language))
+                    .Select(t => (Stemmer)Activator.CreateInstance(t)).FirstOrDefault();
+
+            if (stemmer == null)
+            {
+                Console.WriteLine("Language not found.");
+                return;
+            }
+
+            Console.WriteLine("Using " + stemmer.GetType());
+
+            TextWriter output = System.Console.Out;
+
+            if (outputName != null)
+                output = new StreamWriter(outputName);
+
+
+            foreach (var line in File.ReadAllLines(inputName))
+            {
+                var o = stemmer.Stem(line);
+                output.WriteLine(o);
+            }
+
+            output.Flush();
+        }
+
+        private static bool match(string stemmerName, string language)
+        {
+            string expectedName = language.Replace("_", "") + "Stemmer";
+
+            return stemmerName.StartsWith(expectedName,
+                StringComparison.CurrentCultureIgnoreCase);
+        }
+    }
+}

data/vendor/snowball/doc/TODO

@@ -0,0 +1,12 @@
+Things to do:
+
+ - Write documentation for how to use libstemmer (as opposed to how stemming
+   algorithms themselves work).
+   Currently, the documentation in the include/libstemmer.h header file is
+   pretty clear and comprehensive, but an overview document wouldn't go amiss.
+
+Things that would be nice to include at some point.
+
+ - Add version numbers to each stemming algorithm, and allow the interface to
+   request a specific version of the stemming algorithms.  Default to providing
+   the latest version of the algorithm.

data/vendor/snowball/doc/libstemmer_c_README

@@ -0,0 +1,148 @@
+libstemmer_c
+============
+
+This document pertains to the C version of the libstemmer distribution,
+available for download from:
+
+https://snowballstem.org/download.html
+
+
+What is Stemming?
+-----------------
+
+Stemming maps different forms of the same word to a common "stem" - for
+example, the English stemmer maps *connection*, *connections*, *connective*,
+*connected*, and *connecting* to *connect*.  So a searching for *connected*
+would also find documents which only have the other forms.
+
+This stem form is often a word itself, but this is not always the case as this
+is not a requirement for text search systems, which are the intended field of
+use.  We also aim to conflate words with the same meaning, rather than all
+words with a common linguistic root (so *awe* and *awful* don't have the same
+stem), and over-stemming is more problematic than under-stemming so we tend not
+to stem in cases that are hard to resolve.  If you want to always reduce words
+to a root form and/or get a root form which is itself a word then Snowball's
+stemming algorithms likely aren't the right answer.
+
+
+Compiling the library
+=====================
+
+A simple makefile is provided for Unix style systems.  On such systems, it
+should be possible simply to run "make", and the file "libstemmer.o"
+and the example program "stemwords" will be generated.
+
+If this doesn't work on your system, you need to write your own build
+system (or call the compiler directly).  The files to compile are
+all contained in the "libstemmer", "runtime" and "src_c" directories,
+and the public header file is contained in the "include" directory.
+
+The library comes in two flavours; UTF-8 only, and UTF-8 plus other character
+sets.  To use the utf-8 only flavour, compile "libstemmer_utf8.c" instead of
+"libstemmer.c".
+
+For convenience "mkinc.mak" is a makefile fragment listing the source files and
+header files used to compile the standard version of the library.
+"mkinc_utf8.mak" is a comparable makefile fragment listing just the source
+files for the UTF-8 only version of the library.
+
+
+Using the library
+=================
+
+The library provides a simple C API.  Essentially, a new stemmer can
+be obtained by using "sb_stemmer_new".  "sb_stemmer_stem" is then
+used to stem a word, "sb_stemmer_length" returns the stemmed
+length of the last word processed, and "sb_stemmer_delete" is
+used to delete a stemmer.
+
+Generally you should create a stemmer object and reuse it rather than creating
+a fresh object for each word stemmed (since there's some cost to creating and
+destroying the object).
+
+The stemmer code is re-entrant, but not thread-safe if the same stemmer object
+is used concurrently in different threads.
+
+If you want to perform stemming concurrently in different threads, we suggest
+creating a new stemmer object for each thread.  The alternative is to share
+stemmer objects between threads and protect access using a mutex or similar
+but that's liable to slow your program down as threads can end up waiting for
+the lock.
+
+libstemmer does not currently incorporate any mechanism for caching the results
+of stemming operations.  Such caching can greatly increase the performance of a
+stemmer under certain situations, so suitable patches will be considered for
+inclusion.
+
+The standard libstemmer sources contain an algorithm for each of the supported
+languages.  The algorithm may be selected using the english name of the
+language, or using the 2 or 3 letter ISO 639 language codes.  In addition,
+the traditional "Porter" stemming algorithm for english is included for
+backwards compatibility purposes, but we recommend use of the "English"
+stemmer in preference for new projects.
+
+(Some minor algorithms which are included only as curiosities in the snowball
+website, such as the Lovins stemmer and the Kraaij Pohlmann stemmer, are not
+included in the standard libstemmer sources.  These are not really supported by
+the snowball project, but it would be possible to compile a modified libstemmer
+library containing these if desired.)
+
+
+The stemwords example
+=====================
+
+The stemwords example program allows you to run any of the stemmers
+compiled into the libstemmer library on a sample vocabulary.  For
+details on how to use it, run it with the "-h" command line option.
+
+
+Using the library in a larger system
+====================================
+
+If you are incorporating the library into the build system of a larger
+program, I recommend copying the unpacked tarball without modification into
+a subdirectory of the sources of your program.  Future versions of the
+library are intended to keep the same structure, so this will keep the
+work required to move to a new version of the library to a minimum.
+
+As an additional convenience, the list of source and header files used
+in the library is detailed in mkinc.mak - a file which is in a suitable
+format for inclusion by a Makefile.  By including this file in your build
+system, you can link the snowball system into your program with a few
+extra rules.
+
+Using the library in a system using GNU autotools
+=================================================
+
+The libstemmer_c library can be integrated into a larger system which uses the
+GNU autotool framework (and in particular, automake and autoconf) as follows:
+
+1) Unpack libstemmer_c-*.tar.gz in the top level project directory and rename
+   the resulting directory to remove the version number so that there is a
+   libstemmer_c subdirectory of the top level directory of the project.
+
+2) Add a file "Makefile.am" to the unpacked libstemmer_c folder, containing:
+   
+noinst_LTLIBRARIES = libstemmer.la
+include $(srcdir)/mkinc.mak
+noinst_HEADERS = $(snowball_headers)
+libstemmer_la_SOURCES = $(snowball_sources) 
+
+(You may also need to add other lines to this, for example, if you are using
+compiler options which are not compatible with compiling the libstemmer
+library.)
+
+3) Add libstemmer_c to the AC_CONFIG_FILES declaration in the project's
+   configure.ac file.
+
+4) Add to the top level makefile the following lines (or modify existing
+   assignments to these variables appropriately):
+
+AUTOMAKE_OPTIONS = subdir-objects
+AM_CPPFLAGS = -I$(top_srcdir)/libstemmer_c/include
+SUBDIRS=libstemmer_c
+<name>_LIBADD = libstemmer_c/libstemmer.la
+
+(Where <name> is the name of the library or executable which links against
+libstemmer.) 
+

data/vendor/snowball/doc/libstemmer_csharp_README

@@ -0,0 +1,53 @@
+libstemmer_csharp
+=================
+
+This document pertains to the C# version of the libstemmer distribution,
+available for download from:
+
+https://snowballstem.org/download.html
+
+
+What is Stemming?
+-----------------
+
+Stemming maps different forms of the same word to a common "stem" - for
+example, the English stemmer maps *connection*, *connections*, *connective*,
+*connected*, and *connecting* to *connect*.  So a searching for *connected*
+would also find documents which only have the other forms.
+
+This stem form is often a word itself, but this is not always the case as this
+is not a requirement for text search systems, which are the intended field of
+use.  We also aim to conflate words with the same meaning, rather than all
+words with a common linguistic root (so *awe* and *awful* don't have the same
+stem), and over-stemming is more problematic than under-stemming so we tend not
+to stem in cases that are hard to resolve.  If you want to always reduce words
+to a root form and/or get a root form which is itself a word then Snowball's
+stemming algorithms likely aren't the right answer.
+
+
+Compiling the library
+=====================
+
+To build a library::
+
+mcs -target:library -out:snowballstemmer.dll csharp/Snowball/*.cs csharp/Snowball/Algorithms/*cs
+
+And to build the example program using that library::
+
+mcs -target:exe -out:stemwords.exe -r:snowballstemmer.dll csharp/Stemwords/Program.cs
+
+Using the library
+=================
+
+There is currently no formal documentation on the use of the C# version
+of the library. Additionally, its interface is not guaranteed to be
+stable.
+
+The stemmer code is re-entrant, but not thread-safe if the same stemmer object
+is used concurrently in different threads.
+
+If you want to perform stemming concurrently in different threads, we suggest
+creating a new stemmer object for each thread.  The alternative is to share
+stemmer objects between threads and protect access using a mutex or similar
+but that's liable to slow your program down as threads can end up waiting for
+the lock.

data/vendor/snowball/doc/libstemmer_java_README

@@ -0,0 +1,67 @@
+libstemmer_java
+===============
+
+This document pertains to the Java version of the libstemmer distribution,
+available for download from:
+
+https://snowballstem.org/download.html
+
+
+What is Stemming?
+-----------------
+
+Stemming maps different forms of the same word to a common "stem" - for
+example, the English stemmer maps *connection*, *connections*, *connective*,
+*connected*, and *connecting* to *connect*.  So a searching for *connected*
+would also find documents which only have the other forms.
+
+This stem form is often a word itself, but this is not always the case as this
+is not a requirement for text search systems, which are the intended field of
+use.  We also aim to conflate words with the same meaning, rather than all
+words with a common linguistic root (so *awe* and *awful* don't have the same
+stem), and over-stemming is more problematic than under-stemming so we tend not
+to stem in cases that are hard to resolve.  If you want to always reduce words
+to a root form and/or get a root form which is itself a word then Snowball's
+stemming algorithms likely aren't the right answer.
+
+
+Compiling the library
+=====================
+
+Simply run the java compiler on all the java source files under the java
+directory.  For example, this can be done under unix by changing directory into
+the java directory, and running:
+
+ javac org/tartarus/snowball/*.java org/tartarus/snowball/ext/*.java
+
+This will compile the library and also an example program "TestApp" which
+provides a command line interface to the library.
+
+
+Using the library
+=================
+
+There is currently no formal documentation on the use of the Java version
+of the library.  Additionally, its interface is not guaranteed to be
+stable.
+
+The best documentation of the library is the source of the TestApp example
+program.
+
+The stemmer code is re-entrant, but not thread-safe if the same stemmer object
+is used concurrently in different threads.
+
+If you want to perform stemming concurrently in different threads, we suggest
+creating a new stemmer object for each thread.  The alternative is to share
+stemmer objects between threads and protect access using a mutex or similar
+but that's liable to slow your program down as threads can end up waiting for
+the lock.
+
+
+The TestApp example
+===================
+
+The TestApp example program allows you to run any of the stemmers
+compiled into the libstemmer library on a sample vocabulary.  For
+details on how to use it, run it with no command line parameters.
+

data/vendor/snowball/doc/libstemmer_js_README

@@ -0,0 +1,48 @@
+Snowball stemming library collection for Javascript
+===================================================
+
+What is Stemming?
+-----------------
+
+Stemming maps different forms of the same word to a common "stem" - for
+example, the English stemmer maps *connection*, *connections*, *connective*,
+*connected*, and *connecting* to *connect*.  So a searching for *connected*
+would also find documents which only have the other forms.
+
+This stem form is often a word itself, but this is not always the case as this
+is not a requirement for text search systems, which are the intended field of
+use.  We also aim to conflate words with the same meaning, rather than all
+words with a common linguistic root (so *awe* and *awful* don't have the same
+stem), and over-stemming is more problematic than under-stemming so we tend not
+to stem in cases that are hard to resolve.  If you want to always reduce words
+to a root form and/or get a root form which is itself a word then Snowball's
+stemming algorithms likely aren't the right answer.
+
+
+How to use library
+------------------
+
+You can use each stemming modules from Javascript code - e.g to use them
+with node:
+
+.. code-block:: javascript
+
+    const stemmer = require('base-stemmer.js');
+    const english_stemmer = require('english-stemmer.js');
+
+    var stemmer = new EnglishStemmer();
+    alert(stemmer.stemWord("testing"));
+
+You'll need to bundle ``base-stemmer.js`` and whichever languages you want
+stemmers for (e.g. ``english-stemmer.js`` for English).
+
+FIXME: Document how to use in a web browser.
+
+The stemmer code is re-entrant, but not thread-safe if the same stemmer object
+is used concurrently in different threads.
+
+If you want to perform stemming concurrently in different threads, we suggest
+creating a new stemmer object for each thread.  The alternative is to share
+stemmer objects between threads and protect access using a mutex or similar
+but that's liable to slow your program down as threads can end up waiting for
+the lock.

data/vendor/snowball/doc/libstemmer_python_README

@@ -0,0 +1,113 @@
+Snowball stemming library collection for Python
+===============================================
+
+Python 3 (>= 3.3) is supported.  We no longer actively support Python 2 as
+the Python developers stopped supporting it at the start of 2020.  Snowball
+2.1.0 was the last release to officially support Python 2.
+
+What is Stemming?
+-----------------
+
+Stemming maps different forms of the same word to a common "stem" - for
+example, the English stemmer maps *connection*, *connections*, *connective*,
+*connected*, and *connecting* to *connect*.  So a searching for *connected*
+would also find documents which only have the other forms.
+
+This stem form is often a word itself, but this is not always the case as this
+is not a requirement for text search systems, which are the intended field of
+use.  We also aim to conflate words with the same meaning, rather than all
+words with a common linguistic root (so *awe* and *awful* don't have the same
+stem), and over-stemming is more problematic than under-stemming so we tend not
+to stem in cases that are hard to resolve.  If you want to always reduce words
+to a root form and/or get a root form which is itself a word then Snowball's
+stemming algorithms likely aren't the right answer.
+
+How to use library
+------------------
+
+The ``snowballstemmer`` module has two functions.
+
+The ``snowballstemmer.algorithms`` function returns a list of available
+algorithm names.
+
+The ``snowballstemmer.stemmer`` function takes an algorithm name and returns a
+``Stemmer`` object.
+
+``Stemmer`` objects have a ``Stemmer.stemWord(word)`` method and a
+``Stemmer.stemWords(word[])`` method.
+
+.. code-block:: python
+
+   import snowballstemmer
+
+   stemmer = snowballstemmer.stemmer('english');
+   print(stemmer.stemWords("We are the world".split()));
+
+Generally you should create a stemmer object and reuse it rather than creating
+a fresh object for each word stemmed (since there's some cost to creating and
+destroying the object).
+
+The stemmer code is re-entrant, but not thread-safe if the same stemmer object
+is used concurrently in different threads.
+
+If you want to perform stemming concurrently in different threads, we suggest
+creating a new stemmer object for each thread.  The alternative is to share
+stemmer objects between threads and protect access using a mutex or similar
+(e.g. `threading.Lock` in Python) but that's liable to slow your program down
+as threads can end up waiting for the lock.
+
+Automatic Acceleration
+----------------------
+
+`PyStemmer <https://pypi.org/project/PyStemmer/>`_ is a wrapper module for
+Snowball's ``libstemmer_c`` and should provide results 100% compatible to
+**snowballstemmer**.
+
+**PyStemmer** is faster because it wraps generated C versions of the stemmers;
+**snowballstemmer** uses generate Python code and is slower but offers a pure
+Python solution.
+
+If PyStemmer is installed, ``snowballstemmer.stemmer`` returns a ``PyStemmer``
+``Stemmer`` object which provides the same ``Stemmer.stemWord()`` and
+``Stemmer.stemWords()`` methods.
+
+Benchmark
+~~~~~~~~~
+
+This is a crude benchmark which measures the time for running each stemmer on
+every word in its sample vocabulary (10,787,583 words over 26 languages).  It's
+not a realistic test of normal use as a real application would do much more
+than just stemming.  It's also skewed towards the stemmers which do more work
+per word and towards those with larger sample vocabularies.
+
+* Python 2.7 + **snowballstemmer** : 13m00s (15.0 * PyStemmer)
+* Python 3.7 + **snowballstemmer** : 12m19s (14.2 * PyStemmer)
+* PyPy 7.1.1 (Python 2.7.13) + **snowballstemmer** : 2m14s (2.6 * PyStemmer)
+* PyPy 7.1.1 (Python 3.6.1) + **snowballstemmer** : 1m46s (2.0 * PyStemmer)
+* Python 2.7 + **PyStemmer** : 52s
+
+For reference the equivalent test for C runs in 9 seconds.
+
+These results are for Snowball 2.0.0.  They're likely to evolve over time as
+the code Snowball generates for both Python and C continues to improve (for
+a much older test over a different set of stemmers using Python 2.7,
+**snowballstemmer** was 30 times slower than **PyStemmer**, or 9 times slower
+with **PyPy**).
+
+The message to take away is that if you're stemming a lot of words you should
+either install **PyStemmer** (which **snowballstemmer** will then automatically
+use for you as described above) or use PyPy.
+
+The TestApp example
+-------------------
+
+The ``testapp.py`` example program allows you to run any of the stemmers
+on a sample vocabulary.
+
+Usage::
+
+   testapp.py <algorithm> "sentences ... "
+
+.. code-block:: bash
+
+   $ python testapp.py English "sentences... "