mittens 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 5685ca729c53bcfeadf2241ffecd3eb536197dcf3863efd33e9b07aac114e4dd
+  data.tar.gz: a2a07b4b5d16a691695d1a420ef4acde948276077c9bce7cb98111b172bb71ab
+SHA512:
+  metadata.gz: 2b4f3abde51e786665b83c3846cf5aa0409cae5cbf91dd8f715d3aa850a4c967c951dcd1475d9c6887aa991fc7a460be6f9205196ed7c8f4389c301de38b4931
+  data.tar.gz: b9ea741946ac5af5a27ae6685297a7416d1b1e64b0c780c8e0871332393990832a59821bed08ff21428a8c703dfd089ec89e0b376e34514f25f3f4c151620d00

data/CHANGELOG.md

@@ -0,0 +1,3 @@
+## 0.1.0 (2023-03-20)
+
+- First release

data/Gemfile

@@ -0,0 +1,7 @@
+source "https://rubygems.org"
+
+gemspec
+
+gem "rake"
+gem "rake-compiler"
+gem "minitest", ">= 5"

data/LICENSE.txt

@@ -0,0 +1,30 @@
+Copyright (c) 2001, Dr Martin Porter
+Copyright (c) 2004,2005, Richard Boulton
+Copyright (c) 2013, Yoshiki Shibukawa
+Copyright (c) 2006,2007,2009,2010,2011,2014-2019, Olly Betts
+Copyright (c) 2022, Andrew Kane
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+  1. Redistributions of source code must retain the above copyright notice,
+     this list of conditions and the following disclaimer.
+  2. Redistributions in binary form must reproduce the above copyright notice,
+     this list of conditions and the following disclaimer in the documentation
+     and/or other materials provided with the distribution.
+  3. Neither the name of the Mittens project nor the names of its contributors
+     may be used to endorse or promote products derived from this software
+     without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

data/README.md

@@ -0,0 +1,62 @@
+# Mittens
+
+Stemming for Ruby, powered by [Snowball](https://github.com/snowballstem/snowball)
+
+:snowflake: Supports 28 languages
+
+[![Build Status](https://github.com/ankane/mittens/workflows/build/badge.svg?branch=master)](https://github.com/ankane/mittens/actions)
+
+## Installation
+
+Add this line to your application’s Gemfile:
+
+```ruby
+gem "mittens"
+```
+
+## Getting Started
+
+Create a stemmer
+
+```ruby
+stemmer = Mittens::Stemmer.new
+```
+
+Stem a word
+
+```ruby
+stemmer.stem("tomatos")
+```
+
+## Languages
+
+Specify the language
+
+```ruby
+stemmer = Mittens::Stemmer.new(language: "french")
+```
+
+Supports `arabic`, `armenian`, `basque`, `catalan`, `danish`, `dutch`, `english`, `finnish`, `french`, `german`, `greek`, `hindi`, `hungarian`, `indonesian`, `irish`, `italian`, `lithuanian`, `nepali`, `norwegian`, `porter`, `portuguese`, `romanian`, `russian`, `serbian`, `spanish`, `swedish`, `tamil`, `turkish`, and `yiddish`
+
+## History
+
+View the [changelog](https://github.com/ankane/mittens/blob/master/CHANGELOG.md)
+
+## Contributing
+
+Everyone is encouraged to help improve this project. Here are a few ways you can help:
+
+- [Report bugs](https://github.com/ankane/mittens/issues)
+- Fix bugs and [submit pull requests](https://github.com/ankane/mittens/pulls)
+- Write, clarify, or fix documentation
+- Suggest or add new features
+
+To get started with development:
+
+```sh
+git clone --recursive https://github.com/ankane/mittens.git
+cd mittens
+bundle install
+bundle exec rake compile
+bundle exec rake test
+```

data/Rakefile

@@ -0,0 +1,21 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+require "rake/extensiontask"
+
+task default: :test
+Rake::TestTask.new do |t|
+  t.libs << "test"
+  t.pattern = "test/**/*_test.rb"
+end
+
+Rake::ExtensionTask.new("mittens") do |ext|
+  ext.name = "ext"
+  ext.lib_dir = "lib/mittens"
+end
+
+task :remove_ext do
+  path = "lib/mittens/ext.bundle"
+  File.unlink(path) if File.exist?(path)
+end
+
+Rake::Task["build"].enhance [:remove_ext]

data/ext/mittens/ext.c

@@ -0,0 +1,96 @@
+#include "libstemmer.h"
+#include "ruby/ruby.h"
+
+typedef struct stemmer {
+    struct sb_stemmer * stemmer;
+} stemmer_t;
+
+static void stemmer_free(void *ptr)
+{
+    stemmer_t *stemmer = (stemmer_t *)ptr;
+    // safe to pass null pointer according to docs
+    sb_stemmer_delete(stemmer->stemmer);
+    xfree(ptr);
+}
+
+const rb_data_type_t stemmer_data_type = {
+    .wrap_struct_name = "stemmer",
+    .function = {
+        .dfree = stemmer_free,
+    },
+    .flags = RUBY_TYPED_FREE_IMMEDIATELY
+};
+
+static VALUE stemmer_allocate(VALUE klass)
+{
+    stemmer_t *stemmer;
+    VALUE obj = TypedData_Make_Struct(klass, stemmer_t, &stemmer_data_type, stemmer);
+    stemmer->stemmer = NULL;
+    return obj;
+}
+
+static VALUE stemmer_initialize(int argc, VALUE* argv, VALUE self)
+{
+    VALUE opts;
+    rb_scan_args(argc, argv, ":", &opts);
+
+    const char * algorithm = "english";
+    if (!NIL_P(opts)) {
+        VALUE language = rb_hash_aref(opts, ID2SYM(rb_intern("language")));
+        if (!NIL_P(language)) {
+            Check_Type(language, T_STRING);
+            algorithm = RSTRING_PTR(language);
+        }
+    }
+
+    stemmer_t *stemmer;
+    TypedData_Get_Struct(self, stemmer_t, &stemmer_data_type, stemmer);
+
+    // in case called multiple times
+    sb_stemmer_delete(stemmer->stemmer);
+
+    // if adding support for encoding, may want to change encoding returned from stem
+    stemmer->stemmer = sb_stemmer_new(algorithm, NULL);
+    if (stemmer->stemmer == NULL) {
+        rb_raise(rb_eArgError, "unknown language: %s", algorithm);
+    }
+
+    return self;
+}
+
+static VALUE stemmer_stem(VALUE self, VALUE value)
+{
+    stemmer_t *stemmer;
+    TypedData_Get_Struct(self, stemmer_t, &stemmer_data_type, stemmer);
+
+    Check_Type(value, T_STRING);
+
+    const sb_symbol * word = (const sb_symbol *) RSTRING_PTR(value);
+    int size = (int) RSTRING_LEN(value);
+    const sb_symbol * pointer_out = sb_stemmer_stem(stemmer->stemmer, word, size);
+
+    return rb_utf8_str_new_cstr((char *) pointer_out);
+}
+
+static VALUE stemmer_languages(VALUE klass)
+{
+    VALUE out = rb_ary_new();
+
+    const char **language = sb_stemmer_list();
+    while (*language != NULL) {
+        rb_ary_push(out, rb_utf8_str_new_cstr(*language));
+        language++;
+    }
+
+    return out;
+}
+
+void Init_ext(void)
+{
+    VALUE rb_mMittens = rb_define_module("Mittens");
+    VALUE rb_cStemmer = rb_define_class_under(rb_mMittens, "Stemmer", rb_cObject);
+    rb_define_alloc_func(rb_cStemmer, stemmer_allocate);
+    rb_define_method(rb_cStemmer, "initialize", stemmer_initialize, -1);
+    rb_define_method(rb_cStemmer, "stem", stemmer_stem, 1);
+    rb_define_singleton_method(rb_cStemmer, "languages", stemmer_languages, 0);
+}

data/ext/mittens/extconf.rb

@@ -0,0 +1,12 @@
+require "mkmf"
+require "open3"
+
+vendor = File.expand_path("../../vendor/snowball", __dir__)
+output, status = Open3.capture2("make", chdir: vendor)
+puts output
+raise "Command failed" unless status.success?
+
+$INCFLAGS += " -I$(srcdir)/../../vendor/snowball/include"
+$LDFLAGS += " $(srcdir)/../../vendor/snowball/libstemmer.a"
+
+create_makefile("mittens/ext")

data/lib/mittens/version.rb

@@ -0,0 +1,3 @@
+module Mittens
+  VERSION = "0.1.0"
+end

data/lib/mittens.rb

@@ -0,0 +1,7 @@
+# modules
+require_relative "mittens/ext"
+require_relative "mittens/version"
+
+module Mittens
+  class Error < StandardError; end
+end

data/mittens.gemspec

@@ -0,0 +1,22 @@
+require_relative "lib/mittens/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "mittens"
+  spec.version       = Mittens::VERSION
+  spec.summary       = "Stemming for Ruby, powered by Snowball"
+  spec.homepage      = "https://github.com/ankane/mittens"
+  spec.license       = "BSD-3-Clause"
+
+  spec.author        = "Andrew Kane"
+  spec.email         = "andrew@ankane.org"
+
+  spec.files = Dir.chdir(__dir__) do
+    `git ls-files -z --recurse-submodules`.split("\x0").reject do |f|
+      (f == __FILE__) || f.match(%r{\A(?:(?:bin|test|spec|features)/|\.(?:git|travis|circleci)|appveyor)})
+    end
+  end
+  spec.require_path  = "lib"
+  spec.extensions    = ["ext/mittens/extconf.rb"]
+
+  spec.required_ruby_version = ">= 3"
+end

data/vendor/snowball/.gitignore

@@ -0,0 +1,26 @@
+*.o
+/ada/bin/
+/ada/obj/
+/algorithms.mk
+/libstemmer/libstemmer.c
+/libstemmer/libstemmer_utf8.c
+/libstemmer/mkinc.mak
+/libstemmer/mkinc_utf8.mak
+/libstemmer/modules.h
+/libstemmer/modules_utf8.h
+/libstemmer.a
+/snowball
+/src_c
+/stemtest
+/stemwords
+/dist
+/java/org/tartarus/snowball/ext/
+/js_out
+/python_check
+/python_out
+*.generated.cs
+/rust/Cargo.lock
+/rust/src/snowball/algorithms/*.rs
+/rust/target/
+/go/algorithms/
+/go/stemwords/algorithms.go

data/vendor/snowball/.travis.yml

@@ -0,0 +1,112 @@
+language: minimal
+arch: arm64
+dist: xenial
+env:
+  global: MAKE=make
+matrix:
+  include:
+    - name: "C distribution build"
+      language: c
+      dist: focal
+      compiler: gcc
+      env: CFLAGS_DIST_BUILD='-O2 -Wall -W -std=c90 -Wdeclaration-after-statement -Werror'
+    - name: "C distribution build (clang)"
+      language: c
+      dist: focal
+      compiler: clang
+      env: CFLAGS_DIST_BUILD='-O2 -Wall -W -std=c90 -Wdeclaration-after-statement -Werror'
+    - language: c
+      compiler: gcc
+      env: c_tests=y CFLAGS='-O2 -Wall -W -std=c99 -Werror'
+    - language: c
+      compiler: clang
+      env: c_tests=y CFLAGS='-O2 -Wall -W -std=c99 -Werror'
+    - language: java
+      env: JAVA=java JAVAC=javac
+    - language: go
+      go: "1.8"
+      env: GO=go
+    - language: go
+      dist: bionic
+      go: "1.17"
+      env: GO=go
+    - language: node_js
+      node_js: "node"
+      env: NODE=node
+      dist: focal # seems to be needed for working node
+    - language: rust
+      rust:
+        - stable
+        - beta
+      dist: bionic
+      env: RUST=rust
+    - language: csharp
+      arch: amd64 # csharp doesn't seem to work on arm64
+      env: MCS=mcs
+      dist: bionic
+    - name: Pascal
+      env: FPC=fpc
+      dist: bionic
+      addons:
+        apt:
+          packages:
+            - fpc
+    # The pure Python versions run slowly so we need to thin the testdata
+    # for languages such as Arabic where there's a lot, or else the build
+    # hits the travis time limit.  With pypy, it's enough faster than we
+    # can run the full tests.
+    - language: python
+      python: "3.9"
+      env: PYTHON=python THIN_FACTOR=10
+    - language: python
+      python: "3.7"
+      env: PYTHON=python THIN_FACTOR=10
+    - language: python
+      python: "3.6"
+      env: PYTHON=python THIN_FACTOR=10
+    - language: python
+      python: "pypy3.7-7.3.5"
+      env: PYTHON=python
+      dist: bionic
+    - name: "Ada"
+      env: gprbuild=gprbuild
+      dist: bionic
+      addons:
+        apt:
+          packages:
+            - gnat
+            - gprbuild
+    - os: windows
+      language: c
+      env: c_tests=y MAKE=mingw32-make
+    - os: windows
+      language: go
+      env: GO=go MAKE=mingw32-make
+
+before_install:
+  # Try to check out a branch of the same name from the snowball-data repo
+  # sibling of this snowball repo, so that PRs requiring changes to both can be
+  # CI tested easily.
+  #
+  # If that fails, just use the standard snowball-data repo's default branch.
+  - GH_BRANCH=${TRAVIS_PULL_REQUEST_BRANCH:-$TRAVIS_BRANCH}
+  - GH_REPO_SLUG=${TRAVIS_PULL_REQUEST_SLUG:-$TRAVIS_REPO_SLUG}
+  - GH_REPO_URL=https://github.com/${GH_REPO_SLUG%%/*}/snowball-data.git
+  - echo "Trying branch $GH_BRANCH from $GH_REPO_URL"
+  - git clone --depth=1 -b "$GH_BRANCH" "$GH_REPO_URL" || git clone --depth=1 https://github.com/snowballstem/snowball-data.git
+
+script:
+  # Ensure CC is set for building the compiler in non-C builds.
+  - test -n "$CC" || export CC=gcc
+  - $MAKE CC="$CC"
+  - test -z "$CFLAGS_DIST_BUILD" || { pip install setuptools && $MAKE dist && mkdir tmp && cd tmp && tar xf ../dist/libstemmer_c-*.tar.gz && cd libstemmer_c-* && $MAKE CFLAGS="$CFLAGS_DIST_BUILD" ; }
+  - test -z "$c_tests" || $MAKE check CC="$CC" STEMMING_DATA=snowball-data
+  - test -z "$PYTHON" || $MAKE check_python python="$PYTHON" STEMMING_DATA=snowball-data
+  - test -z "$JAVA" -o -z "$JAVAC" || $MAKE check_java STEMMING_DATA=snowball-data
+  - test -z "$MCS" || $MAKE check_csharp MCS="$MCS" STEMMING_DATA=snowball-data
+  - test -z "$NODE" || $MAKE check_js STEMMING_DATA=snowball-data
+  - test -z "$RUST" || $MAKE check_rust STEMMING_DATA=snowball-data
+  - test -z "$RUST" || $MAKE check_rust STEMMING_DATA=snowball-data
+  - test -z "$GO" || $MAKE check_go STEMMING_DATA=snowball-data
+  - test -z "$FPC" || $MAKE check_pascal STEMMING_DATA=snowball-data
+  - test -z "$gprbuild" || $MAKE check_ada STEMMING_DATA=snowball-data

data/vendor/snowball/AUTHORS

@@ -0,0 +1,27 @@
+Authors
+=======
+
+Martin Porter
+-------------
+
+ - Designed the snowball language.
+ - Implemented the snowball to C compiler.
+ - Implemented the stemming algorithms in C.
+ - Wrote the documentation.
+
+Richard Boulton
+---------------
+
+ - Implemented Java backend of the snowball compiler.
+ - Developed build system.
+ - Assisted with website maintenance.
+
+
+Assistance from
+---------------
+
+Olivier Bornet - fixes to java packaging and build system.
+Andreas Jung - useful bug reports on the libstemmer library.
+Olly Betts - several patches, bug reports, and performance improvements.
+Sebastiano Vigna and Oerd Cukalla - patches for the Java stemming algorithms.
+Ralf Junker - fix a potential memory leak in sb_stemmer_new().

data/vendor/snowball/CONTRIBUTING.rst

@@ -0,0 +1,216 @@
+Adding a new stemming algorithm
+===============================
+
+This needs PRs against three repositories.  Name the branch the same for
+at least `snowball` and `snowball-data`, push to the latter repo first, and the
+CI should use your new vocabulary list when running the testsuite.
+
+Some points to note about algorithm implementations:
+
+* Avoid literal non-ASCII characters in snowball string literals - they will
+  work OK for languages that use UTF-8, but not wide-character Unicode or other
+  encodings.  Instead use ``stringdef`` like the existing stemmers do, and
+  please use the newer `U+` notation rather than the older ``hex`` or
+  ``decimal`` as this allows us to support different encodings without having
+  to modify the source files - for example::
+
+    stringdef o" {U+00F6}
+    define foo 'o{o"}'
+
+  not::
+
+    stringdef o" hex F6
+    define foo 'o{o"}'
+
+  and definitely not::
+
+    define foo 'oö'
+
+  It's helpful to consistently use the same ``stringdef`` codes across the
+  different stemmers - the website has `guidance on what to use
+  <https://snowballstem.org/codesets/guide.html>`_ and a `list of stringdef
+  lines for common characters to cut and paste from
+  <https://snowballstem.org/codesets/latin-stringdef-list.txt>`_.
+
+snowball repo
+-------------
+
+Add `.sbl` source to algorithms subdirectory.
+
+Add entry to `libstemmer/modules.txt`, maintaining the current sorted order by
+the first column.  The columns are:
+
+* Algorithm name (needs to match the `.sbl` source without extension)
+* Encodings to support.  Wide-character Unicode is always supported
+  and doesn't need to be listed here.  You should always include `UTF_8`, and
+  also any of `ISO_8859_1`, `ISO_8859_2` and `KOI8_R` which the language can
+  usefully be written using only characters from (in particular they need to
+  contain all the characters the stemmer explicitly uses).  Support for other
+  single-byte character sets is easy to add if they're useful.
+* Names and ISO-639 codes for the language.  Wikipedia has a handy list of `all
+  the ISO-639 codes <https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes>`_ -
+  find the row for your new language and include the codes from the "639-1",
+  "639-2/T" and (if different) "639-2/B" columns.  For example, for the `Afar`
+  language you'd put `afar,aa,aar` here.
+
+snowball-data repo
+------------------
+
+Add subdirectory named after new stemmer containing:
+
+* voc.txt - word list
+* output.txt - stemmed equivalents
+* COPYING - licensing details (word lists need to be under an OSI-approved
+  licence)
+
+If you don't have access to a suitably licensed word list of a suitable size,
+you may be able to use the `wikipedia-most-common-words` script to generate
+one by extracting the most frequent words from a Wikipedia dump in the
+language the stemmer is for.  You need to specify the Unicode "script" (that's
+"script" in the sense of alphabet) to use - you can find the appropriate one
+by looking in the Unicode `Scripts.txt`::
+
+https://www.unicode.org/Public/13.0.0/ucd/Scripts.txt
+
+The script name is the second column, between `;` and `#`.  The first entries
+are all "Common" which isn't what you want - scroll down to get to the entries
+that are useful here.
+
+You also need to specify the minimum frequency to select.  Picking this value
+will probably need some experimentation as the appropriate threshold depends on
+how much data there is in the wikipedia dump for a particular language, as well
+as the size of the vocabulary for the language, and how inflected the language
+is.  Try counting the number of unique words extracted (`wc -l voc.txt` on
+Unix) and also looking through the list - some proper nouns, words from other
+languages, typos, etc are OK (since the stemmer will encounter all these in
+practice too), but at some point "more" stops being "better".
+
+snowball-website repo
+---------------------
+
+Create subdirectory of `algorithms/` named after the language.
+
+Create `stemmer.tt` which describes the stemming algorithm.  This is a
+"template toolkit" template which is essentially a mix of HTML and some
+macros for adding the navigation, sample vocabulary, etc.  See the
+existing `stemmer.tt` files for other algorithms for inspiration.
+
+If it is based on an academic paper, cite the paper and describe any difference
+between your implementation and that described in the paper (for example,
+sometimes papers have ambiguities that need resolving to re-implement the
+algorithm described).
+
+If you have a stopword list, add that as `stop.txt` in your new subdirectory.
+The `generate` script checks if such a file exists and if it does a link to
+it is automatically added.
+
+Link to your new `stemmer.tt` from `algorithms/index.tt`.
+
+Add a news entry to `index.tt`.
+
+Add the new stemmer to the online demo.  Assuming you have checkouts of the
+`snowball`, `snowball-data` and `snowball-website` repos in sibling
+directories:
+
+* run `make check_js` in the `snowball` repo
+* run `./update-js`
+* add the new stemmer to git with: `git add js/*-stemmer.js`
+* if the new language is written right-to-left (RTL) then add it to the check
+  in `demo.tt` (search for `rtl` to find the place to change.)
+* `git commit`.
+
+Adding a new programming language generator
+===========================================
+
+This is a short guide to adding support for generating code for another
+programming language.
+
+Is a new generator the right solution?
+--------------------------------------
+
+Adding a new code generator is probably not your only option if you want
+to use Snowball from another language - most languages have support for
+writing bindings to a C library, so this is probably another option.
+
+Generating code can have advantages.  For example, it can be simpler to
+deploy without C bindings which need to be built for a specific platform.
+
+However, it's likely to be significantly more work to implement a new generator
+than to write bindings to the generated C code, especially as the libstemmer
+C API is a very small and simple one.  Generated code can also be slower -
+currently the Snowball compiler often generates code that assumes an optimising
+compiler will clean up redundant constructs, which is not a problem for C, and
+probably not for most compiled languages, but for a language like Python C
+bindings are much faster than the generated Python code (using pypy helps a
+lot, but is still slower).  See doc/libstemmer_python_README for some timings.
+
+That said, the unoptimised generated code has improved over time, and is likely
+to improve further in the future.
+
+Key problems to solve
+---------------------
+
+* A key problem to solve is how to map the required flow of control in response
+  to Snowball signals.
+
+  In the generated C code this is mostly done using `goto`.  If your language
+  doesn't provide an equivalent to `goto` then you'll need an alternative
+  solution.
+
+  In Java and JavaScript we use labelled `break` from blocks and loops
+  instead.  If your language has an equivalent to this feature, that will
+  probably work.
+
+  For Python, we currently generate a `try:` ... `raise lab123` ...
+  `except lab123: pass` construct.  This works, but doesn't seem ideal.
+
+  If one of the mechanisms above sounds suitable then take a look at the
+  generator for the respective generated output and generator code.  If
+  not, come and talk to us on the snowball-discuss mailing list.
+
+* Snowball's division is specified as integer division with semantics
+  matching C - i.e. the result should be truncated (rounded towards zero).
+  Some languages lack a built-in integer division operation, or have one
+  which instead implements rounding towards negative infinity.  Existing
+  backends with special handling handling here which may be useful to look at
+  include Javascript, Pascal and Python.
+
+Don't hardcode algorithm names
+------------------------------
+
+We want to avoid hard-coded lists of algorithms in the language-specific code
+that have to be manually updated each time a new algorithm is added, because
+that adds some extra tedious work for adding a new algorithm, and mechanical
+updates done by hand tend to miss places that need updating, or code gets
+copied and pasted from an existing case but not fully updated.
+
+All the existing language backends generate any such code at build time, and
+adding a new algorithm just requires updating `libstemmer/modules.txt`.
+
+You can probably copy the approach used for Pascal (script `pascal/generate.pl`
+works from template `stemwords-template.dpr` which has marked blocks of code
+that get expanded for each stemming algorithm with a placeholder replaced by
+the algorithm name.  For an alternative approach, see Rust where this is done
+by `rust/build.rs`.
+
+Mechanics of adding a new generator
+-----------------------------------
+
+Copy an existing `compiler/generator_*.c` for your new language and modify
+away (`generator.c` has the generator for C, but also some common functions
+so if you start from this one you'll need to remove those common functions).
+Please resist reformatting existing C code - there's currently a lot of code
+repeated in each generator which ought to be pulled out as common code, and
+if you reformat that just makes that job harder.
+
+Add your new source to `COMPILER_SOURCES` in `GNUmakefile`.
+
+Add prototypes for the new functions to `compiler/header.h`.
+
+Add support to `compiler/driver.c`.
+
+Add targets to `GNUmakefile` to run tests for the new language.
+
+Hook up automated testing via CI in `.travis.yml`.
+
+Add to the list of languages in `README.rst`.