mittens 0.3.1 → 0.3.2

data/vendor/snowball/algorithms/persian.sbl

@@ -0,0 +1,387 @@
+/*
+ * -----------------------------------------------------------------------------
+ * Persian Stemmer (HPS-like)
+ * -----------------------------------------------------------------------------
+ * Based on the paper:
+ *   "HPS: A Hierarchical Persian Stemming Method"
+ *   Ayshe Rashidi, Mina Zolfy Lighvan (2014)
+ * -----------------------------------------------------------------------------
+ * Differences from original HPS:
+ * 1) No POS-tagger stage:
+ *      - HPS uses a POS tagger to route tokens to Noun, Adjective, or Verb
+ *        suffix rules directly.
+ *      - Here we fall back to the fixed order: Noun or Adjective, then Verb
+ *        when POS is unknown.
+ *      Reason: Implementation is tagger-agnostic and lightweight.
+ * 2) Lexical "-AN" guard:
+ *      - Added `Protect_Lexical_AN` to avoid stripping true lexical endings
+ *        like "...stan", "...ran", and known stems (Iran, Tehran, Ensan).
+ *      Reason: reduces over-stemming where HPS relied on explicit hash lists.
+ * -----------------------------------------------------------------------------
+ * Implemented by: https://saeiddrv.com
+ * -----------------------------------------------------------------------------
+ */
+
+stringescapes { }
+
+// ============================================================================
+//  Alphabet & special symbols
+// ============================================================================
+stringdef alef       '{U+0627}'
+stringdef aa         '{U+0622}'
+stringdef be         '{U+0628}'
+stringdef pe         '{U+067E}'
+stringdef te         '{U+062A}'
+stringdef se         '{U+062B}'
+stringdef jim        '{U+062C}'
+stringdef che        '{U+0686}'
+stringdef heh_jimi   '{U+062D}'
+stringdef khe        '{U+062E}'
+stringdef dal        '{U+062F}'
+stringdef zal        '{U+0630}'
+stringdef re         '{U+0631}'
+stringdef ze         '{U+0632}'
+stringdef zhe        '{U+0698}'
+stringdef sin        '{U+0633}'
+stringdef shin       '{U+0634}'
+stringdef sad        '{U+0635}'
+stringdef zad        '{U+0636}'
+stringdef ta         '{U+0637}'
+stringdef za         '{U+0638}'
+stringdef ain        '{U+0639}'
+stringdef ghain      '{U+063A}'
+stringdef fe         '{U+0641}'
+stringdef ghaf       '{U+0642}'
+stringdef kaf        '{U+06A9}'
+stringdef gaf        '{U+06AF}'
+stringdef lam        '{U+0644}'
+stringdef mim        '{U+0645}'
+stringdef nun        '{U+0646}'
+stringdef vav        '{U+0648}'
+stringdef heh        '{U+0647}'
+stringdef ye         '{U+06CC}'
+
+stringdef space             '{U+0020}'
+stringdef zero_width_joiner '{U+200D}'
+stringdef half_space        '{U+200C}' // zero-width non-joiner (ZWNJ)
+// Note: U+200C (ZERO WIDTH NON-JOINER) is commonly referred to as "nim-faseleh" ("half-space") in Persian.
+// Although this name is not standard in English Unicode terminology, it reflects its widespread use
+// in Persian as a morpheme separator within words. This character is here to normalize
+
+// Arabic variants to normalize to Persian forms
+stringdef ar_kaf                   '{U+0643}'
+stringdef ar_ye                    '{U+064A}'
+stringdef ar_ye_with_hamza_above   '{U+0626}'
+stringdef ar_heh                   '{U+06C1}'
+stringdef ar_teh_marbuta           '{U+0629}'
+stringdef ar_alef_with_hamza_above '{U+0623}'
+stringdef ar_alef_with_hamza_below '{U+0625}'
+stringdef ar_vav_with_hamza_above  '{U+0624}'
+
+// ============================================================================
+//  Declarations (routines, flags, and marks)
+// ============================================================================
+routines (
+  Normalize_Characters  // Forward: Unicode/script normalization
+  Prefixes              // Forward: handles prefixes
+  Delete_ZWNJ           // Forward: delete remaining ZWNJs after prefix handling
+  Protect_Lexical_AN    // Probe: flag lexical -AN endings (per pass)
+
+  R1 // Test if cursor is in R1
+
+  // Noun tier (HPS categories)
+  AN_Exception
+  Irregular_Noun
+  Stem_Noun_or_Adjective
+  Stem_Verb
+)
+
+externals ( stem )
+
+// Guard flags:
+//  - 'saw_present_prefix':
+//    true when a present-tense prefix (mi-/nemi-) is stripped; used as the
+//    default for `remove_verb_person_endings` for all suffix removal passes.
+//  - 'remove_verb_person_endings':
+//    true if a verb cue has been seen; used to enable verb person-ending rules
+//    in this pass.
+booleans ( saw_present_prefix remove_verb_person_endings )
+
+integers ( p1 )
+
+// ============================================================================
+//  PHASE 1 — NORMALIZATION (forward)
+//  - Unify Arabic forms to Persian letters for stable downstream matching.
+//  - Delete ZWJ (U+200D) and ASCII spaces so the token is solid.
+//  - ZWNJ (U+200C) is preserved here; used as prefix-stem separator in Phase 2,
+//    then deleted by Delete_ZWNJ after prefix stripping is complete.
+// ============================================================================
+define Normalize_Characters as (
+  repeat (
+    [substring] among (
+      '{ar_kaf}'                                                 (<- '{kaf}')
+      '{ar_ye}' '{ar_ye_with_hamza_above}'                       (<- '{ye}')
+      '{ar_teh_marbuta}' '{ar_heh}'                              (<- '{heh}')
+      '{ar_alef_with_hamza_above}' '{ar_alef_with_hamza_below}'  (<- '{alef}')
+      '{ar_vav_with_hamza_above}'                                (<- '{vav}')
+      '{zero_width_joiner}'                                      ( delete )
+      '{space}'                                                  ( delete )
+      ''                                                         ( next )
+    )
+  )
+)
+
+// ============================================================================
+//  PHASE 2 — PREFIX HANDLING (forward, anchored)
+//  - HPS: handle known prefixes before suffix processing.
+//  - Requires ZWNJ (U+200C) between prefix and stem to avoid mishandling
+//    words that begin with the same syllables but are not prefixed forms.
+//  - At least two characters are required after the prefix.
+//  - Sets `saw_present_prefix` when matching {mi}/{nemi} to enable person endings later.
+//  - Note: negating derivational prefixes na- and bi- are intentionally not
+//    stripped as they create words with opposite meanings (e.g. na+dorost =
+//    incorrect, bi+khatr = safe) and conflating them with their base forms
+//    would harm search precision.
+//  - Similarly, nemi- negates the meaning so we don't remove it but do still
+//    set `saw_present_prefix` when it is present.
+// ============================================================================
+define Prefixes as (
+  [substring] among (
+    '{nun}{mim}{ye}{half_space}'  // NEMI‌ + ZWNJ: detect but keep
+        (hop 2  set saw_present_prefix)
+    '{mim}{ye}{half_space}'       // MI‌ + ZWNJ
+        (hop 2  delete  set saw_present_prefix)
+  )
+)
+
+// ============================================================================
+//  PHASE 2b — DELETE REMAINING ZWNJs (forward)
+//  - After prefix detection, any remaining ZWNJs (e.g. in compound words)
+//    are no longer needed and are removed here.
+// ============================================================================
+define Delete_ZWNJ as (
+  repeat (
+    goto (['{half_space}'] delete)
+  )
+)
+
+backwardmode (
+
+  define R1 as $p1 <= cursor
+
+// ============================================================================
+//  PROBE — PROTECT LEXICAL "-AN" (not plural)
+//  - HPS strips -AN as a plural; however, many stems end in orthographic "AN":
+//      ...stan / ...san / ...ran (place names/lexical stems), and frequent
+//      stems such as Iran/Tehran/Ensan/Ostan.
+//  - We signal f if a trailing pattern indicates lexical AN.
+//  - This is a probe only: it NEVER edits the buffer; it only signals.
+// ============================================================================
+  define Protect_Lexical_AN as (
+    not AN_Exception
+    not among (
+      '{sin}{te}{alef}{nun}'  // ...stan
+      '{sin}{alef}{nun}'      // ...san
+      '{re}{alef}{nun}'       // ...ran
+      '{vav}{alef}{nun}'      // ...van: onvaan, divaan, javaan, ravaan
+    )
+  )
+
+// ============================================================================
+//  PHASE 3 — SUFFIX STRIPPING (backward)
+//  - All routines in backward mode start matching from the end of the token.
+//  - IMPORTANT: no successful "no-op" arms; every success must modify the text.
+//    This ensures 'repeat' loops terminate.
+//  - Most suffixes are only removed/replaced if they are entirely in R1 to
+//    avoid over-stemming short words.
+// ============================================================================
+
+  // --- Lexical "-AN" suffix list.
+  //     Small curated set of stems ending in orthographic -AN that are not plural.
+  //     Used by `Protect_Lexical_AN` to prevent stripping on place names / lexical nouns.
+  define AN_Exception as (
+    among (
+      '{aa}{lam}{mim}{alef}{nun}'                     // alman
+      '{aa}{sin}{mim}{alef}{nun}'                     // asman
+      '{alef}{ye}{mim}{alef}{nun}'                    // eiman
+      '{alef}{ye}{shin}{alef}{nun}'                   // ishan
+      '{alef}{mim}{kaf}{alef}{nun}'                   // emkan
+      '{alef}{sad}{fe}{heh}{alef}{nun}'               // esfahan
+      '{aa}{zal}{re}{be}{alef}{ye}{jim}{alef}{nun}'   // azerbaijan
+      '{be}{ye}{alef}{nun}'                           // bayan
+      '{pe}{alef}{ye}{alef}{nun}'                     // payan
+      '{pe}{ye}{mim}{alef}{nun}'                      // payman
+      '{jim}{re}{ye}{alef}{nun}'                      // jaryan
+      '{dal}{re}{mim}{alef}{nun}'                     // darman
+      '{re}{mim}{alef}{nun}'                          // roman
+      '{ze}{nun}{dal}{alef}{nun}'                     // zendan
+      '{sin}{alef}{ze}{mim}{alef}{nun}'               // sazman
+      '{sin}{lam}{ta}{alef}{nun}'                     // soltan
+      '{gaf}{ye}{lam}{alef}{nun}'                     // Gilan
+      '{ghaf}{heh}{re}{mim}{alef}{nun}'               // ghahramaan
+      '{kaf}{re}{mim}{alef}{nun}'                     // Kerman
+      '{khe}{alef}{nun}{dal}{alef}{nun}'              // khandan
+      '{lam}{be}{nun}{alef}{nun}'                     // lobnan
+      '{mim}{ye}{ze}{alef}{nun}'                      // mizan
+      '{mim}{sin}{lam}{mim}{alef}{nun}'               // mosalman
+      '{nun}{shin}{alef}{nun}'                        // neshan
+      '{heh}{mim}{dal}{alef}{nun}'                    // hamedan
+      '{ye}{vav}{nun}{alef}{nun}'                     // yunan
+      '{kaf}{heh}{kaf}{shin}{alef}{nun}'              // kahkeshan (galaxy)
+      '{aa}{te}{shin}{fe}{shin}{alef}{nun}'           // atashfeshan (volcano)
+      '{pe}{re}{ye}{shin}{alef}{nun}'                 // perishan (confused)
+      '{dal}{re}{khe}{shin}{alef}{nun}'               // darakhshan (shining)
+      '{heh}{mim}{ze}{mim}{alef}{nun}'                // hamzaman (simultaneous)
+      '{sin}{alef}{khe}{te}{mim}{alef}{nun}'          // sakhteman (building)
+      '{sin}{lam}{ye}{mim}{alef}{nun}'                // soleyman (Solomon)
+          ( atlimit )
+    )
+  )
+
+  // --- Noun irregular rewrites.
+  //     Only include entries that actually change the buffer.
+  define Irregular_Noun as (
+    [substring] among (
+      '{alef}{khe}{be}{alef}{re}'      (<- '{khe}{be}{re}')
+      '{alef}{sin}{alef}{te}{ye}{dal}' (<- '{alef}{sin}{te}{alef}{dal}')
+    )
+  )
+
+  // --- Noun/Adjective step.
+  define Stem_Noun_or_Adjective as (
+    Irregular_Noun or
+    setlimit tomark p1 for (
+      [substring] among (
+        // Noun possessive suffixes
+        '{alef}{mim}'       (delete) // -AM
+        '{alef}{shin}'      (delete) // -ASH
+
+        // -MAN/-TAN/-SHAN are intentionally disabled: these 3-char suffixes
+        // are ambiguous with plurals of nouns whose root ends in the same letter:
+        //   ketab+maan (our book)   -> strip -maan -> ketab  (correct)
+        //   mardom+an  (people pl.) -> strip -an   -> mardom (correct, NOT mardom->mard)
+        //   naqqash+an (painters)   -> strip -an   -> naqqash (correct, NOT ->naqqaa)
+        //   derakht+an (trees)      -> strip -an   -> derakht (correct, NOT ->derakh)
+        // Without a lexicon we cannot distinguish the two cases. The -AN
+        // rule below handles both at the cost of missing possessive stripping.
+        //        '{mim}{alef}{nun}'  (delete) // -MAN
+        //        '{te}{alef}{nun}'   (delete) // -TAN
+        //        '{shin}{alef}{nun}' (delete) // -SHAN
+
+        // Noun plurals.
+        // -YAN and -GAN are productive.
+        '{ye}{alef}{nun}'   (delete) // -YAN
+        '{gaf}{alef}{nun}'  (delete) // -GAN
+        '{heh}{alef}{ye}'   (delete) // -HAY
+        '{alef}{nun}{ye}'   (delete) // -ANI
+        '{heh}{alef}'       (delete) // -HA
+        '{alef}{te}'        (delete) // -AT (Arabic sound plural; can also be possessive "your")
+        '{alef}{nun}'       (delete) // -AN (only if not lexical)
+        '{ye}{nun}'         (delete) // -IN (also adjective derivation ending)
+
+        // Other derivational noun endings (conservative set).
+        '{gaf}{alef}{heh}'  (delete) // -GAH
+        '{be}{alef}{nun}'   (delete) // -BAN
+        '{gaf}{ye}'         (delete) // -GI (abstract noun)
+        '{ye}{te}'          (delete) // -YAT
+        '{ye}{ye}'          (delete) // -YY (double Y; also adj. relative -Y)
+
+        // Adjective: comparative/superlative (HPS).
+        '{te}{re}{ye}{nun}' (delete) // -TARIN
+        '{te}{re}'          (not atlimit delete) // -TAR
+
+        // Adjective derivational endings (HPS list + common variants).
+        '{alef}{nun}{heh}'  (delete) // -ANE
+        '{mim}{nun}{dal}'   (delete) // -MAND
+        '{vav}{alef}{re}'   (delete) // -VAR
+        '{nun}{alef}{kaf}'  (delete) // -NAK
+        '{gaf}{alef}{re}'   (delete) // -GAR
+      )
+    )
+  )
+
+  // --- Verb step.
+  define Stem_Verb as (
+    (
+      // Participle + person/aux clitic tails
+      [substring] among (
+        '{alef}{ye}{dal}'
+        '{alef}{ye}{mim}'
+        '{alef}{nun}{dal}'
+        '{alef}{sin}{te}'
+        '{alef}{sin}'
+        '{alef}{ye}'
+        '{ye}{dal}'
+        '{ye}{mim}'
+            (R1 delete)
+      )
+    ) or (
+      [substring] among (
+        // Generic person and singular/plural endings — only when a verb cue is active
+        '{alef}{nun}{dal}'  // -AND
+        '{ye}{dal}'         // -ID
+        '{ye}{mim}'         // -IM
+        '{alef}{mim}'       // -AM
+        '{dal}'             // -D
+        '{mim}'             // -M
+            ( remove_verb_person_endings R1 delete )
+
+        // Specific past-3sg stem fixes
+        '{re}{fe}{te}{mim}'
+        '{re}{fe}{te}{ye}'
+        '{re}{fe}{te}{ye}{mim}'
+        '{re}{fe}{te}{ye}{dal}'
+        '{re}{fe}{te}{alef}{nun}{dal}'
+            (<- '{re}{fe}{te}')
+
+        // Mood/tense markers (HPS).
+        '{nun}{dal}{heh}'   // agent noun -NDEH → present root
+        '{alef}{nun}'       // infinitive -N / -AN
+            (R1 delete set remove_verb_person_endings)
+        '{dal}{heh}'        // past part. -deh
+            (not atlimit <-'{dal}' set remove_verb_person_endings)
+        '{te}{heh}'         // past part. -teh
+            (not atlimit <-'{te}'  set remove_verb_person_endings)
+      )
+    )
+  )
+)
+
+// ============================================================================
+//  MAIN (HPS pipeline)
+// ============================================================================
+define stem as (
+  unset saw_present_prefix
+
+  // 1) Normalize script/spacing (ZWNJ preserved for prefix detection)
+  do Normalize_Characters
+
+  // 2) Handle leading prefixes (requires ZWNJ separator)
+  do Prefixes
+
+  // 2b) Delete remaining ZWNJs now that prefix detection is done
+  do Delete_ZWNJ
+
+  // Set p1 to be 3 characters into the string.
+  $p1 = limit
+  do ( hop 3 setmark p1 )
+
+  // 3) Remove suffixes.  Each pass removes one suffix, trying noun and
+  // adjective endings first, then verb endings.  So if we remove a noun
+  // or adjective ending, we next check for another noun or adjective
+  // ending.
+  //
+  // Note: The loop exits if no suffix is removed and the buffer gets
+  // shorter when a suffix is removed so we can't loop forever.
+  backwards repeat test (
+    // Set remove_verb_person_endings = saw_present_prefix.
+    unset remove_verb_person_endings
+    try ( saw_present_prefix set remove_verb_person_endings )
+
+    Protect_Lexical_AN
+    // HPS as described in the paper uses a POS tagger.  We don't have that
+    // information so we use a heuristic and try noun/adjective endings first,
+    // then verb endings.
+    Stem_Noun_or_Adjective or Stem_Verb
+  )
+)

data/vendor/snowball/algorithms/polish.sbl

@@ -0,0 +1,177 @@
+/* Polish stemmer. Author: Dmitry Shachnev */
+
+stringescapes {}
+
+stringdef ak '{U+0105}' // ą a + ogonek
+stringdef ek '{U+0119}' // ę e + ogonek
+stringdef l/ '{U+0142}' // ł l + stroke
+stringdef c' '{U+0107}' // ć c + acute (kreska)
+stringdef n' '{U+0144}' // ń n + acute (kreska)
+stringdef o' '{U+00f3}' // ó o + acute (kreska)
+stringdef s' '{U+015b}' // ś s + acute (kreska)
+stringdef z' '{U+017a}' // ź z + acute (kreska)
+
+
+externals (stem)
+
+
+routines (
+  mark_regions
+  remove_endings
+  normalize_consonant
+  R1
+)
+
+integers ( p1 )
+
+groupings ( v )
+
+define v 'a{ak}e{ek}io{o'}uy'
+
+
+define mark_regions as (
+  $p1 = limit
+  gopast v
+  gopast non-v  setmark p1
+)
+
+backwardmode (
+  define R1 as ($p1 <= cursor)
+
+  define remove_endings as (
+    // Verbs.
+    do (
+      setlimit tomark p1 for ([substring]) among (
+        // conditionals:
+        'bym'        // 1st person singular (czytał(a)bym)
+        'by{s'}'     // 2nd person singular (czytał(a)byś)
+        'by{s'}my'   // 1st person plural (czytalibyśmy)
+        'by{s'}cie'  // 2nd person plural (czytalibyście)
+        'by'         // 3rd person singular/plural (czytał(a)by, czytaliby)
+          (delete)
+      )
+    )
+    [substring] among (
+      'asz'  'esz'  'isz'  // present 2nd person singular (czytasz, piszesz, nosisz)
+      'amy'  'emy'  'imy'  // present 1st person plural (czytamy, piszemy, nosimy)
+      'acie' 'ecie' 'icie' // present 2nd person plural (czytacie, piszecie, nosicie)
+      'aj{ak}'             // present 3rd person plural (czytają)
+      'e{s'}{c'}'          // infinitive (przynieść)
+      'a{s'}{c'}'          // infinitive (popaść)
+      'a{c'}'              // infinitive (czytać)
+      'ie{c'}'             // infinitive (lecieć)
+      'i{c'}'              // infinitive (wozić)
+      '{ak}{c'}'           // infinitive (marznąć)
+      'aj{ak}c' '{ak}c'    // contemporary adverbial participle (transgressive) (czytając, lecąc)
+      'a{l/}em'       'ia{l/}em'       'i{l/}em'       // past 1st person singular masculine (czytałem, leciałem, chodziłem)
+      'a{l/}am'       'ia{l/}am'       'i{l/}am'  'am' // past 1st person singular feminine (czytałam, leciałam, chodziłam, marzłam)
+      'a{l/}e{s'}'    'ia{l/}e{s'}'    'i{l/}e{s'}'    // past 2nd person singular masculine (czytałeś, leciałeś, chodziłeś)
+      'a{l/}a{s'}'    'ia{l/}a{s'}'    'i{l/}a{s'}'    // past 2nd person singular feminine (czytałaś, leciałaś, chodziłaś)
+      'a{l/}'         'ia{l/}'         'i{l/}'         // past 3rd person singular masculine (czytał, leciał, chodził)
+      'a{l/}a'        'ia{l/}a'        'i{l/}a'        // past 3rd person singular feminine (czytała, leciała, chodziła)
+      'a{l/}o'        'ia{l/}o'        'i{l/}o'        // past 3rd person singular neuter (czytało, leciało, chodziło)
+      'ali{s'}my'     'ieli{s'}my'     'ili{s'}my'     // past 1st person plural virile (czytaliśmy, lecieliśmy, chodziliśmy)
+      'a{l/}y{s'}my'  'ia{l/}y{s'}my'  'i{l/}y{s'}my'  // past 1st person plural nonvirile (czytałyśmy, leciałyśmy, chodziłyśmy)
+      'ali{s'}cie'    'ieli{s'}cie'    'ili{s'}cie'    // past 2nd person plural virile (czytaliście, lecieliście, chodziliście)
+      'a{l/}y{s'}cie' 'ia{l/}y{s'}cie' 'i{l/}y{s'}cie' // past 2nd person plural nonvirile (czytałyście, leciałyście, chodziłyście)
+      'ali'           'ieli'           'ili'           // past 3rd person plural virile (czytali, lecieli, chodzili)
+      'a{l/}y'        'ia{l/}y'        'i{l/}y'        // past 3rd person plural nonvirile (czytały, leciały, chodziły)
+      'aj'     // imperative 2nd person singular (czytaj)
+      'ajcie'  // imperative 2nd person plural (czytajcie)
+      'cie'    // imperative 2nd person plural (chodźcie)
+      '{ek}'   // present 1st person singular (lecę)
+        (delete)
+      'sz{ek}' // present 1st person singular (noszę)
+        (<- 's')
+      'sz{ak}' // present 3rd person plural (noszą)
+        // Also an adjectival form (singular feminine accusative), e.g. lepszą.
+        // This heuristic does the right thing in common cases.
+        (R1 and delete or <-'s')
+
+      // There are short verbs whose root consists of only one consonant, e.g. być, żyć.
+      // Stemming them to one letter would merge them with these letters used in other
+      // contexts, which is undesirable. But let's at least merge all past tense forms
+      // together, e.g. byłem, byłam, byłyśmy, etc. to był.
+      '{l/}e{s'}'
+      '{l/}a{s'}'
+      'li{s'}my'
+      '{l/}y{s'}my'
+      'li{s'}cie'
+      '{l/}y{s'}cie'
+        (<- '{l/}')
+
+      // Adjectives (including comparative/superlative forms)
+      // as well as participles.
+      'y'          // singular masculine nominative (nowy)
+      'ego' 'iego' // singular masculine genitive (nowego, polskiego)
+      'emu' 'iemu' // singular masculine dative (nowemu, polskiemu)
+      'ym'  'im'   // singular masculine instrumental (nowym, polskim)
+      'ej'  'iej'  // singular feminine genitive (nowej, polskiej)
+      'ych' 'ich'  // plural genitive (nowych, polskich)
+      'ymi' 'imi'  // plural instrumental (nowymi, polskimi)
+        (
+          delete
+          try (
+            [substring] among (
+              'aj{ak}c'  // participle suffix (czytający)
+              '{ak}c'    // participle suffix (lecący)
+              'iejsz'    // comparative suffix (piękniejszy)
+              'sz'       // comparative suffix (lepszy)
+                (delete)
+              'sz{ak}c'  // participle suffix (noszący)
+                (<- 's')
+            )
+          )
+        )
+      // We cannot remove endings like -ą and -e unconditionally, because these
+      // letters appear in too many contexts. But we can safely remove them if we
+      // know that our word is a participle or a comparative/superlative form.
+      'aj{ak}ca'    '{ak}ca'    'iejsza'    'sza'  // singular feminine nominative (czytająca, lecąca, piękniejsza, lepsza)
+      'aj{ak}c{ak}' '{ak}c{ak}' 'iejsz{ak}'        // singular feminine accusative (czytającą, lecącą, piękniejszą); -szą is handled separately
+      'aj{ak}ce'    '{ak}ce'    'iejsze'    'sze'  // singular neuter nominative (czytające, lecące, piękniejsze, lepsze)
+        (delete)
+      // Handle participles like nosząca, prosząca.
+      'sz{ak}ca'
+      'sz{ak}c{ak}'
+      'sz{ak}ce'
+        (<- 's')
+
+      // Noun forms (excluding endings that were already handled above).
+      'a' R1  'o' R1                            // singular nominative (książka, lato)
+      'i' R1  'u' R1  'ia' R1                   // singular genitive (książki, stołu, słonia)
+      'owi' R1  'iowi' R1                       // singular dative (stołowi, słoniowi)
+      '{ak}' R1  'i{ak}' R1  'em' R1  'iem' R1  // singular instrumental (książką, możliwością, stołem, słoniem)
+      'e' R1  'iu' R1                           // singular locative (stole, słoniu)
+      'ie' R1                                   // plural nominative (słonie)
+      '{o'}w' R1                                // plural genitive (stołów)
+      'om' R1   'iom' R1                        // plural dative (książkom, słoniom)
+      'ami' R1  'iami' R1                       // plural instrumental (książkami, słoniami)
+      'ach' R1  'iach' R1                       // plural locative (książkach, słoniach)
+        (delete)
+    )
+    try (['{'}'] delete)
+  )
+
+  define normalize_consonant as (
+    // Remove kreska mark, because most of oblique cases do not have it.
+    // Don't mutate single character inputs.
+    [substring] not atlimit among (
+      '{c'}' (<- 'c') // e.g. miłość → miłośc
+      '{n'}' (<- 'n') // e.g. słoń → słon
+      '{s'}' (<- 's') // e.g. gęś → gęs
+      '{z'}' (<- 'z') // e.g. miedź → miedz
+    )
+  )
+)
+
+define stem as (
+  do mark_regions
+  // Make sure we don't produce too short outputs.  The "backwards" will
+  // set the backwards limit to the current cursor position.
+  (
+    hop 2
+    backwards remove_endings
+  ) or (
+    backwards normalize_consonant
+  )
+)

data/vendor/snowball/algorithms/sesotho.sbl

@@ -0,0 +1,115 @@
+/* stringescapes UTF-8 */
+
+/*
+  Sesotho stemmer for the Snowball project
+  ----------------------------------------
+  Author: Kamohelo Lebjane
+  Purpose: To reduce Sesotho words to their morphological stems.
+
+  Language notes:
+  Sesotho (Southern Sotho) is an agglutinative Bantu language.
+  Words often contain prefixes for noun classes and suffixes
+  for tense, aspect, or derivation.
+
+  An agglutinative language is a type of language that primarily forms words
+  by stringing together morphemes (word parts)—each typically representing a single
+  grammatical meaning—without significant modification to their forms (agglutinations).
+  In such languages, affixes (prefixes, suffixes, infixes, or circumfixes)
+  are added to a root word in a linear and systematic way, creating complex
+  words that encode detailed grammatical information.
+
+  Examples:
+    baruti      -> rut (root)
+    moruti      -> rut (root)
+    rutile      -> rut (root)
+
+  The rules below remove common noun class prefixes
+  and common verb suffixes, keeping the main root form.
+*/
+
+/* --- Routine declarations --- */
+routines (
+  mark_regions
+  remove_noun_prefixes
+  remove_verb_suffixes
+  remove_nominal_suffixes
+)
+
+/* --- External declarations --- */
+externals ( stem )
+
+/* --- Groupings --- */
+groupings ( v )
+
+/* --- Character sets --- */
+define v 'aeiou'
+
+/* --- Integer for tracking position --- */
+integers ( pV )
+
+/* --- Mark vowel region --- */
+define mark_regions as (
+  // Set pV after the first vowel and at least 2 characters into the string.
+  // Signals f if the string doesn't contain a vowel or is shorter than 2
+  // characters.
+  test (gopast v  setmark pV)
+  test (hop 2 do ($(cursor > pV) setmark pV))
+)
+
+/* --- Remove noun class prefixes --- */
+define remove_noun_prefixes as (
+  [substring] among(
+    'mo'   'ba'   'me'   'le'
+    'ma'   'se'   'boi'   'li'
+  )
+  /* Require at least two characters remain */
+  test (next not atlimit)
+  /* Only delete if there's a vowel after the cursor position */
+  gopast v delete
+)
+
+backwardmode (
+
+  /* --- Remove verb suffixes (from end of word) --- */
+  define remove_verb_suffixes as (
+    setlimit tomark pV for (
+      [substring] among(
+        'ile'   /* perfect tense */
+        'isa'   /* causative */
+        'etse'  /* applicative */
+        'ela'   /* applicative */
+        'ang'   /* plural imperative */
+        'ong'    /* continuous/derived form */
+        'eng'
+        'wa'    /* passive */
+        'a'     /* infinitive marker */
+        (delete)
+      )
+    )
+  )
+
+  /* --- Remove nominal suffixes --- */
+  define remove_nominal_suffixes as (
+    setlimit tomark pV for (
+      [substring] among(
+        'nyana'   /* diminutive form */
+        'ana'     /* diminutive form */
+        'ano'
+        'oa'
+        'i'
+        (delete)
+      )
+    )
+  )
+
+)
+
+/* --- MAIN STEMMER --- */
+define stem as (
+  mark_regions // Signals f if the string is too short to stem.
+  backwards (
+    do remove_nominal_suffixes
+    do remove_verb_suffixes
+  )
+  do remove_noun_prefixes
+)