mittens 0.1.0

data/vendor/snowball/algorithms/yiddish.sbl

@@ -0,0 +1,460 @@
+/* *******************************************
+ * Stemmer for Yiddish language in YIVO script
+ *
+ * Author: Assaf Urieli
+ * Emails: assaf.urieli at gmail.com
+ * Version: 0.1 (15.05.2020)
+ *
+ ********************************************* */
+
+routines (
+       prelude
+       mark_regions
+       R1
+       R1plus3
+       standard_suffix
+)
+
+externals ( stem )
+
+integers ( p1 x )
+
+groupings ( vowel niked alefBeys consonant )
+
+stringescapes {}
+
+// AlefBeys
+stringdef Alef         '{U+05D0}'
+stringdef Beys         '{U+05D1}'
+stringdef Giml         '{U+05D2}'
+stringdef Dalet        '{U+05D3}'
+stringdef Hey          '{U+05D4}'
+stringdef Vov          '{U+05D5}'
+stringdef Zayen        '{U+05D6}'
+stringdef Khes         '{U+05D7}'
+stringdef Tes          '{U+05D8}'
+stringdef Yud          '{U+05D9}'
+stringdef LangerKhof   '{U+05DA}'
+stringdef Khof         '{U+05DB}'
+stringdef Lamed        '{U+05DC}'
+stringdef ShlosMem     '{U+05DD}'
+stringdef Mem          '{U+05DE}'
+stringdef LangerNun    '{U+05DF}'
+stringdef Nun          '{U+05E0}'
+stringdef Samekh       '{U+05E1}'
+stringdef Ayen         '{U+05E2}'
+stringdef LangerFey    '{U+05E3}'
+stringdef Fey          '{U+05E4}'
+stringdef LangerTsadek '{U+05E5}'
+stringdef Tsadek       '{U+05E6}'
+stringdef Kuf          '{U+05E7}'
+stringdef Reysh        '{U+05E8}'
+stringdef Shin         '{U+05E9}'
+stringdef Sof          '{U+05EA}'
+stringdef TsveyVovn    '{U+05F0}'
+stringdef VovYud       '{U+05F1}'
+stringdef TsveyYudn    '{U+05F2}'
+
+// Niked
+stringdef Shvo          '{U+05B0}'
+stringdef Khirik        '{U+05B4}'
+stringdef Tseyre        '{U+05B5}'
+stringdef Segl          '{U+05B6}'
+stringdef ReducedSegl   '{U+05B1}'
+stringdef Pasekh        '{U+05B7}'
+stringdef ReducedPasekh '{U+05B2}'
+stringdef Komets        '{U+05B8}'
+stringdef ReducedKomets '{U+05B3}'
+stringdef Rafe          '{U+05BF}'
+stringdef SinDot        '{U+05C2}'
+stringdef ShinDot       '{U+05C1}'
+stringdef Khoylm        '{U+05B9}'
+stringdef Melupm        '{U+05BC}'
+stringdef Kubuts        '{U+05BB}'
+
+// Groupings
+define niked       '{Shvo}{Khirik}{Tseyre}{Segl}{ReducedSegl}{Pasekh}{ReducedPasekh}{Komets}{ReducedKomets}{SinDot}{ShinDot}{Khoylm}{Melupm}{Kubuts}{Rafe}'
+define alefBeys    '{Alef}{Beys}{Giml}{Dalet}{Hey}{Vov}{Zayen}{Khes}{Tes}{Yud}{LangerKhof}{Khof}{Lamed}{ShlosMem}{Mem}{LangerNun}{Nun}{Samekh}{Ayen}{LangerFey}{Fey}{LangerTsadek}{Tsadek}{Kuf}{Reysh}{Shin}{Sof}{TsveyVovn}{VovYud}{TsveyYudn}'
+define vowel           '{Alef}{Vov}{Yud}{Ayen}{VovYud}{TsveyYudn}'
+define consonant   alefBeys - vowel
+
+define prelude as (
+  do (
+    repeat goto (
+      [substring] among (
+        '{Vov}{Vov}' ( not '{Melupm}' <- '{TsveyVovn}' )
+        '{Vov}{Yud}' ( not '{Khirik}' <- '{VovYud}' )
+        '{Yud}{Yud}' ( not '{Khirik}' <- '{TsveyYudn}' )
+        '{LangerKhof}' ( <- '{Khof}')
+        '{ShlosMem}' ( <- '{Mem}' )
+        '{LangerNun}' ( <- '{Nun}' )
+        '{LangerFey}' ( <- '{Fey}' )
+        '{LangerTsadek}' ( <- '{Tsadek}' )
+      )
+    )
+  )
+
+  do (repeat goto ( [niked] delete ))
+)
+
+define mark_regions as (
+  $p1 = limit
+
+  (
+    try (
+      // Replace past participle ge- at start of word
+      // Unless word starts with gelt- or gebn-
+      ['{Giml}{Ayen}']
+      not ('{Lamed}{Tes}' or '{Beys}{Nun}') <- 'GE'
+    )
+
+    try (
+      // skip verbal prefix
+      among(
+        // Free stressed: Adurkh-, Durkh-, Ahin-, Aher-, Avek-, Mit-, Antkegn-, Akegn-, Anider-, Arop-, Aroys-, Aroyf-, Arum-, Arayn-, Arunter-, Ariber-, Nokh-, Farbay-, Aheym-, Afir-, Faroys-, Funander-, Tsuzamen-, Tsunoyf-, Tsurik-
+        '{Alef}{Dalet}{Vov}{Reysh}{Khof}' '{Dalet}{Vov}{Reysh}{Khof}' '{Alef}{Hey}{Yud}{Nun}' '{Alef}{Hey}{Ayen}{Reysh}' '{Alef}{TsveyVovn}{Ayen}{Kuf}' '{Mem}{Yud}{Tes}' '{Alef}{Nun}{Tes}{Kuf}{Ayen}{Giml}{Nun}' '{Alef}{Kuf}{Ayen}{Giml}{Nun}' '{Alef}{Nun}{Yud}{Dalet}{Ayen}{Reysh}' '{Alef}{Reysh}{Alef}{Fey}' '{Alef}{Reysh}{VovYud}{Samekh}' '{Alef}{Reysh}{VovYud}{Fey}' '{Alef}{Reysh}{Vov}{Mem}' '{Alef}{Reysh}{TsveyYudn}{Nun}' '{Alef}{Reysh}{Vov}{Nun}{Tes}{Ayen}{Reysh}' '{Alef}{Reysh}{Yud}{Beys}{Ayen}{Reysh}' '{Nun}{Alef}{Khof}' '{Fey}{Alef}{Reysh}{Beys}{TsveyYudn}' '{Alef}{Hey}{TsveyYudn}{Mem}' '{Alef}{Fey}{Yud}{Reysh}' '{Fey}{Alef}{Reysh}{VovYud}{Samekh}' '{Fey}{Vov}{Nun}{Alef}{Nun}{Dalet}{Ayen}{Reysh}' '{Tsadek}{Vov}{Zayen}{Alef}{Mem}{Ayen}{Nun}' '{Tsadek}{Vov}{Nun}{VovYud}{Fey}' '{Tsadek}{Vov}{Reysh}{Yud}{Kuf}'
+
+        // Stressed: Oys-, Oyf-, Um-, Unter-, Iber-, Ayn-, On-, Op-, Bay-, For-, Tsu-.
+        '{Alef}{VovYud}{Samekh}' '{Alef}{VovYud}{Fey}' '{Alef}{Vov}{Mem}' '{Alef}{Vov}{Nun}{Tes}{Ayen}{Reysh}' '{Alef}{Yud}{Beys}{Ayen}{Reysh}' '{Alef}{TsveyYudn}{Nun}' '{Alef}{Nun}' '{Alef}{Fey}' '{Beys}{TsveyYudn}' '{Fey}{Alef}{Reysh}' '{Tsadek}{Vov}'
+
+        // Unstressed:  Ant-, Ba-, Der-, Tse-. Far- already covered by For-. Ge- comes later.
+        '{Alef}{Nun}{Tes}' '{Beys}{Alef}' '{Dalet}{Ayen}{Reysh}' '{Tsadek}{Ayen}'
+
+        // If verbal prefix followed by Tsu- or Ge-, replace it
+        (
+          // Don't mark the TSU- prefix inside verbs like "oys-tsugn"
+          test (('{Tsadek}{Vov}{Giml}{Nun}' or '{Tsadek}{Vov}{Kuf}{Tes}' or '{Tsadek}{Vov}{Kuf}{Nun}') atlimit)
+          or
+          // Don't mark the GE- prefix inside verbs like "avek-gebn"
+          test ('{Giml}{Ayen}{Beys}{Nun}')
+          or
+          ( ['{Giml}{Ayen}'] <- 'GE')
+          or
+          (['{Tsadek}{Vov}'] <- 'TSU')
+        )
+      )
+    )
+
+    test(hop 3 setmark x)
+
+    // We want to allow three-consonant Hebrew roots.
+    // To this end, we skip three-consonant combinations that exist in non-Hebraic Yiddish.
+    try (
+      among(
+        '{Shin}{Fey}{Reysh}' '{Shin}{Tes}{Reysh}' '{Shin}{Tes}{Shin}' '{Dalet}{Zayen}{Shin}'
+          ( true )
+      )
+    )
+
+    // Either 3 consonants or the first non-vowel after a vowel
+    (
+      not (consonant consonant consonant setmark p1)
+      goto vowel repeat vowel setmark p1
+    )
+    try($p1 < x  $p1 = x)  // at least 3 past the prefix
+  )
+
+)
+
+backwardmode (
+  define R1 as $p1 <= cursor
+  // Like R1, but also allows the cursor to be outside R1 by the width of Giml Yud Samekh
+  define R1plus3 as $p1 <= cursor + sizeof '{Giml}{Yud}{Samekh}'
+
+  define standard_suffix as (
+    do (
+      [substring] among(
+        // Plural/adjective endings: -er, -ers, -e, -n, -s, -en, -ns, -eners, -ens, -es
+         '{Ayen}{Reysh}{Samekh}' '{Ayen}{Nun}' '{Nun}{Samekh}' '{Ayen}{Nun}{Ayen}{Reysh}{Samekh}' '{Ayen}{Samekh}' '{Ayen}' '{Nun}' '{Samekh}' '{Ayen}{Mem}' '{Ayen}{Reysh}'
+          ( R1 delete )
+
+        // Exception: don't delete noun endings -ie, like "agitatsie"
+        '{Yud}{Ayen}'
+          ( true )
+
+        // -ies => ie
+        '{Yud}{Ayen}{Samekh}'
+          ( R1  <- '{Yud}{Ayen}'  )
+
+        // Plural/adjective endings: -enem, -ener, -ene, -ens
+        '{Ayen}{Nun}{Ayen}' '{Ayen}{Nun}{Ayen}{Mem}' '{Ayen}{Nun}{Ayen}{Reysh}' '{Ayen}{Nun}{Samekh}'
+          (R1 delete 
+            [substring] among (
+              // -gegangen => -gey
+              '{Giml}{Alef}{Nun}{Giml}' (<- '{Giml}{TsveyYudn}')
+              // -genumen => -nem
+              '{Nun}{Vov}{Mem}' (<- '{Nun}{Ayen}{Mem}')
+              // -gemiten => -mayd
+              '{Mem}{Yud}{Tes}' (<- '{Mem}{TsveyYudn}{Dalet}')
+              // -gebiten => -bayt
+              '{Beys}{Yud}{Tes}' (<- '{Beys}{TsveyYudn}{Tes}')
+              // -gebisen => -bays
+              '{Beys}{Yud}{Samekh}' (<- '{Beys}{TsveyYudn}{Samekh}')
+              // -gevizen => -vayz
+              '{TsveyVovn}{Yud}{Zayen}' (<- '{TsveyVovn}{TsveyYudn}{Zayen}')
+              // -getriben => -trayb
+              '{Tes}{Reysh}{Yud}{Beys}' (<- '{Tes}{Reysh}{TsveyYudn}{Beys}')
+              // -geliten => -layt
+              '{Lamed}{Yud}{Tes}' (<- '{Lamed}{TsveyYudn}{Tes}')
+              // -gekliben => -klayb
+              '{Kuf}{Lamed}{Yud}{Beys}' (<- '{Kuf}{Lamed}{TsveyYudn}{Beys}')
+              // -geriben => -rayb
+              '{Reysh}{Yud}{Beys}' (<- '{Reysh}{TsveyYudn}{Beys}')
+              // -gerisen => -rays
+              '{Reysh}{Yud}{Samekh}' (<- '{Reysh}{TsveyYudn}{Samekh}')
+              // -geshvigen => -shvayg
+              '{Shin}{TsveyVovn}{Yud}{Giml}' (<- '{Shin}{TsveyVovn}{TsveyYudn}{Giml}')
+              // -geshmisen => -shmays
+              '{Shin}{Mem}{Yud}{Samekh}' (<- '{Shin}{Mem}{TsveyYudn}{Samekh}')
+              // -geshniten => -shnayd
+              '{Shin}{Nun}{Yud}{Tes}' (<- '{Shin}{Nun}{TsveyYudn}{Dalet}')
+              // -geshriben => -shrayb
+              '{Shin}{Reysh}{Yud}{Beys}' (<- '{Shin}{Reysh}{TsveyYudn}{Beys}')
+              // -gebunden => -bind
+              '{Beys}{Vov}{Nun}{Dalet}' (<- '{Beys}{Yud}{Nun}{Dalet}')
+              // -gevuntshn => -vintsh
+              '{TsveyVovn}{Vov}{Tes}{Shin}' (<- '{TsveyVovn}{Yud}{Tes}{Shin}')
+              // -gezungen => -zing
+              '{Zayen}{Vov}{Nun}{Giml}' (<- '{Zayen}{Yud}{Nun}{Giml}')
+              // -getrunken => -trink
+              '{Tes}{Reysh}{Vov}{Nun}{Kuf}' (<- '{Tes}{Reysh}{Yud}{Nun}{Kuf}')
+              // -getsvungen => -tsving
+              '{Tsadek}{TsveyVovn}{Vov}{Nun}{Giml}' (<- '{Tsadek}{TsveyVovn}{Yud}{Nun}{Giml}')
+              // -geshlungen => -shling
+              '{Shin}{Lamed}{Vov}{Nun}{Giml}' (<- '{Shin}{Lamed}{Yud}{Nun}{Giml}')
+              // -geboygen => -beyg
+              '{Beys}{VovYud}{Giml}' (<- '{Beys}{TsveyYudn}{Giml}')
+              // -gehoyben => -heyb
+              '{Hey}{VovYud}{Beys}' (<- '{Hey}{TsveyYudn}{Beys}')
+              // -farloyren => -farlir
+              '{Fey}{Alef}{Reysh}{Lamed}{VovYud}{Reysh}' (<- '{Fey}{Alef}{Reysh}{Lamed}{Yud}{Reysh}')
+              // -shtanen => -shtey
+              '{Shin}{Tes}{Alef}{Nun}' (<- '{Shin}{Tes}{TsveyYudn}')
+              // -geshvoyrn => -shver
+              '{Shin}{TsveyVovn}{VovYud}{Reysh}' (<- '{Shin}{TsveyVovn}{Ayen}{Reysh}')
+            )
+         )
+
+        // Verb/past participle ending: -t
+        '{Tes}'
+          ( R1 delete ) 
+
+        // As well as noun/adjectives ending in -tn, -te, -ter, -ts so that the "-t" doesn't differentiate
+        // Similarly for past participles: -tns, -tene, -tenem, -tener
+        // If the Tes was before R1, we try to perform the same action while leaving the Tes in place
+        '{Tes}{Nun}' '{Tes}{Ayen}' '{Tes}{Ayen}{Reysh}' '{Tes}{Samekh}'
+        '{Tes}{Nun}{Samekh}' '{Tes}{Ayen}{Nun}{Ayen}' '{Tes}{Ayen}{Nun}{Ayen}{Mem}' '{Tes}{Ayen}{Nun}{Ayen}{Reysh}'
+          ( ((R1 delete) or ( <- '{Tes}'))
+            // -(ge)brakht => -breng
+            ['{Beys}{Reysh}{Alef}{Khof}' try '{Giml}{Ayen}'] <- '{Beys}{Reysh}{Ayen}{Nun}{Giml}'
+          )
+
+        // Past participles: -et, -etn, -ets, -ete, -eter
+        '{Ayen}{Tes}' '{Ayen}{Tes}{Nun}' '{Ayen}{Tes}{Samekh}' '{Ayen}{Tes}{Ayen}' '{Ayen}{Tes}{Ayen}{Reysh}'
+          ( R1 delete )
+
+        // -geyn shorted to -gey
+        '{Giml}{TsveyYudn}{Nun}'
+                    ( <- '{Giml}{TsveyYudn}')
+
+        // ##################### Long list of irregular past participles
+        // -(ge)gangen (shortened to -gangen after prefixes) => -gey
+        '{Giml}{Alef}{Nun}{Giml}{Ayen}{Nun}'
+                    ( <- '{Giml}{TsveyYudn}' )
+
+        // -(ge)numen (shortened to -numen after prefixes) => -nem
+        '{Nun}{Vov}{Mem}{Ayen}{Nun}'
+          (<- '{Nun}{Ayen}{Mem}' )
+
+        // -(ge)shribn (shortened to -shribn after prefixes) => -shrayb
+        '{Shin}{Reysh}{Yud}{Beys}{Nun}'
+          (<- '{Shin}{Reysh}{TsveyYudn}{Beys}' )
+
+        // -gemiten => -mayd
+        'GE{Mem}{Yud}{Tes}{Nun}' 
+          (<- '{Mem}{TsveyYudn}{Dalet}')
+
+        // -gebiten => -bayt
+        'GE{Beys}{Yud}{Tes}{Nun}' 
+          (<- '{Beys}{TsveyYudn}{Tes}')
+
+        // -gebisen => -bays
+        'GE{Beys}{Yud}{Samekh}{Nun}' 
+          ( <- '{Beys}{TsveyYudn}{Samekh}')
+
+        // -gevizen => -vayz
+        '{TsveyVovn}{Yud}{Zayen}{Nun}' 
+           ( <- '{TsveyVovn}{TsveyYudn}{Zayen}')
+
+        // -getriben => -trayb
+        '{Tes}{Reysh}{Yud}{Beys}{Nun}' 
+           ( <- '{Tes}{Reysh}{TsveyYudn}{Beys}')
+
+        // -geliten => -layt
+        'GE{Lamed}{Yud}{Tes}{Nun}' 
+           ( <- '{Lamed}{TsveyYudn}{Tes}')
+
+        // -gekliben => -klayb
+        '{Kuf}{Lamed}{Yud}{Beys}{Nun}' 
+           ( <- '{Kuf}{Lamed}{TsveyYudn}{Beys}')
+
+        // -geriben => -rayb
+        '{Reysh}{Yud}{Beys}{Nun}' 
+           ( <- '{Reysh}{TsveyYudn}{Beys}')
+
+        // -gerisen => -rays
+        'GE{Reysh}{Yud}{Samekh}{Nun}' 
+           ( <- '{Reysh}{TsveyYudn}{Samekh}')
+
+        // -geshvigen => -shvayg
+        '{Shin}{TsveyVovn}{Yud}{Giml}{Nun}' 
+           ( <- '{Shin}{TsveyVovn}{TsveyYudn}{Giml}')
+
+        // -geshmisen => -shmays
+        '{Shin}{Mem}{Yud}{Samekh}{Nun}' 
+           ( <- '{Shin}{Mem}{TsveyYudn}{Samekh}')
+
+        // -geshniten => -shnayd
+        '{Shin}{Nun}{Yud}{Tes}{Nun}' 
+           ( <- '{Shin}{Nun}{TsveyYudn}{Dalet}')
+
+        // -gebunden => -bind
+        '{Beys}{Vov}{Nun}{Dalet}{Nun}' 
+         ( <- '{Beys}{Yud}{Nun}{Dalet}')
+
+        // -gevuntshn => -vintsh
+        '{TsveyVovn}{Vov}{Tes}{Shin}{Nun}' 
+         ( <- '{TsveyVovn}{Yud}{Tes}{Shin}')
+
+        // -gezungen => -zing
+        '{Zayen}{Vov}{Nun}{Giml}{Nun}' 
+         ( <- '{Zayen}{Yud}{Nun}{Giml}')
+
+        // -getrunken => -trink
+        '{Tes}{Reysh}{Vov}{Nun}{Kuf}{Nun}' 
+         ( <- '{Tes}{Reysh}{Yud}{Nun}{Kuf}')
+
+        // -getsvungen => -tsving
+        '{Tsadek}{TsveyVovn}{Vov}{Nun}{Giml}{Nun}' 
+         ( <- '{Tsadek}{TsveyVovn}{Yud}{Nun}{Giml}')
+
+        // -geshlungen => -shling
+        '{Shin}{Lamed}{Vov}{Nun}{Giml}{Nun}' 
+         ( <- '{Shin}{Lamed}{Yud}{Nun}{Giml}')
+
+        // -geboygen => -beyg
+        '{Beys}{VovYud}{Giml}{Nun}' 
+         ( <- '{Beys}{TsveyYudn}{Giml}')
+
+        // -gehoyben => -heyb
+        '{Hey}{VovYud}{Beys}{Nun}' 
+         ( <- '{Hey}{TsveyYudn}{Beys}')
+
+        // -farloyren => -farlir
+        '{Fey}{Alef}{Reysh}{Lamed}{VovYud}{Reysh}{Nun}' 
+         ( <- '{Fey}{Alef}{Reysh}{Lamed}{Yud}{Reysh}')
+
+        // -shtanen => -shtey
+        '{Shin}{Tes}{Alef}{Nun}{Ayen}{Nun}' 
+         ( <- '{Shin}{Tes}{TsveyYudn}')
+
+         // -geshvoyrn => -shver
+        '{Shin}{TsveyVovn}{VovYud}{Reysh}{Nun}' 
+         ( <- '{Shin}{TsveyVovn}{Ayen}{Reysh}')
+
+        // -(ge)brakht (shortened to -brakht after prefixes) => -breng
+        '{Beys}{Reysh}{Alef}{Khof}{Tes}'
+          (<- '{Beys}{Reysh}{Ayen}{Nun}{Giml}' )
+
+        // ###### End of irregular past participles
+
+        // Noun endings: -ung, -hayt, -kayt, -ikayt, -shaft
+        '{Vov}{Nun}{Giml}' '{Hey}{TsveyYudn}{Tes}' '{Kuf}{TsveyYudn}{Tes}' '{Yud}{Kuf}{TsveyYudn}{Tes}' '{Shin}{Alef}{Fey}{Tes}'
+          ( R1 delete )
+
+        // Noun endings: -izm, izmen
+        '{Yud}{Zayen}{Mem}' '{Yud}{Zayen}{Mem}{Ayen}{Nun}'
+          ( R1 delete )
+
+        // Plural ending: -im
+        '{Yud}{Mem}'
+          ( R1 delete )
+          
+        // Plural ending: -os (Hebraic), replace with -h
+        '{Vov}{Sof}'
+          ( R1  <- '{Hey}' )
+
+        // Diminutive endings: -elekh, -ele, -lekh, -eles, -elen
+        '{Ayen}{Lamed}{Ayen}{Khof}' '{Ayen}{Lamed}{Ayen}' '{Lamed}{Ayen}{Khof}' '{Ayen}{Lamed}{Ayen}{Samekh}' '{Ayen}{Lamed}{Ayen}{Nun}'
+           ( R1 delete )
+        
+        // Noun ending: -ist
+        '{Yud}{Samekh}{Tes}'
+          (
+            // Exceptions: -gist, -shist
+            ( ('{Giml}' or '{Shin}') try (R1plus3 <- '{Yud}{Samekh}') )
+            or
+            ( R1 delete )
+          )
+
+        // Noun ending: -istn
+        '{Yud}{Samekh}{Tes}{Nun}'
+          ( R1 delete )
+        
+        // Verb ending: -stu
+        '{Samekh}{Tes}{Vov}'
+          ( R1 delete )
+
+        // Superlative ending: -ster, -ste, -stn 
+        '{Samekh}{Tes}{Ayen}{Reysh}' '{Samekh}{Tes}{Ayen}' '{Samekh}{Tes}{Nun}' 
+          ( R1 delete )
+          
+        // Ambiguous verb ending: -st
+        '{Samekh}{Tes}'
+          ( R1 delete ) 
+      )
+    )
+
+    do (
+      [substring] among(
+        // Noun endings: -ung, -hayt, -kayt, -ikayt, -shaft
+        '{Vov}{Nun}{Giml}' '{Hey}{TsveyYudn}{Tes}' '{Kuf}{TsveyYudn}{Tes}' '{Yud}{Kuf}{TsveyYudn}{Tes}' '{Shin}{Alef}{Fey}{Tes}'
+          ( R1 delete )
+
+        // Diminutive endings: -l
+        '{Lamed}'
+          ( R1 consonant delete )
+      )
+    )
+
+    do (
+      [substring] among(
+        // Adjective endings: -ig, -ik, -ish, -nik, -dik
+        '{Yud}{Giml}' '{Yud}{Kuf}' '{Yud}{Shin}' '{Nun}{Yud}{Kuf}' '{Dalet}{Yud}{Kuf}'
+          ( R1 delete )
+
+        // Exceptions to above: -blik, -glik
+        '{Beys}{Lamed}{Yud}{Kuf}' '{Giml}{Lamed}{Yud}{Kuf}'
+          ( true )
+          
+        // Present participle endings: -ndik
+        '{Nun}{Dalet}{Yud}{Kuf}'
+           ( R1 delete )
+
+        // Present participle ending -endik: delete if after a -ng, -nk, -n, -m, consonant+l, or vowel.
+        // Otherwise, delete just the -ndik part.
+        '{Ayen}{Nun}{Dalet}{Yud}{Kuf}'
+           ( R1 delete )
+      )
+    )
+
+    do (repeat goto ( ['GE' or 'TSU'] delete ))
+  )
+)
+
+define stem as (
+  do prelude
+  do mark_regions
+  backwards
+    do standard_suffix
+)

data/vendor/snowball/charsets/ISO-8859-2.sbl

@@ -0,0 +1,98 @@
+// ISO-8859-2 character mappings.
+
+stringdef U+00A0  hex 'A0'
+stringdef U+0104  hex 'A1'
+stringdef U+02D8  hex 'A2'
+stringdef U+0141  hex 'A3'
+stringdef U+00A4  hex 'A4'
+stringdef U+013D  hex 'A5'
+stringdef U+015A  hex 'A6'
+stringdef U+00A7  hex 'A7'
+stringdef U+00A8  hex 'A8'
+stringdef U+0160  hex 'A9'
+stringdef U+015E  hex 'AA'
+stringdef U+0164  hex 'AB'
+stringdef U+0179  hex 'AC'
+stringdef U+00AD  hex 'AD'
+stringdef U+017D  hex 'AE'
+stringdef U+017B  hex 'AF'
+stringdef U+00B0  hex 'B0'
+stringdef U+0105  hex 'B1'
+stringdef U+02DB  hex 'B2'
+stringdef U+0142  hex 'B3'
+stringdef U+00B4  hex 'B4'
+stringdef U+013E  hex 'B5'
+stringdef U+015B  hex 'B6'
+stringdef U+02C7  hex 'B7'
+stringdef U+00B8  hex 'B8'
+stringdef U+0161  hex 'B9'
+stringdef U+015F  hex 'BA'
+stringdef U+0165  hex 'BB'
+stringdef U+017A  hex 'BC'
+stringdef U+02DD  hex 'BD'
+stringdef U+017E  hex 'BE'
+stringdef U+017C  hex 'BF'
+stringdef U+0154  hex 'C0'
+stringdef U+00C1  hex 'C1'
+stringdef U+00C2  hex 'C2'
+stringdef U+0102  hex 'C3'
+stringdef U+00C4  hex 'C4'
+stringdef U+0139  hex 'C5'
+stringdef U+0106  hex 'C6'
+stringdef U+00C7  hex 'C7'
+stringdef U+010C  hex 'C8'
+stringdef U+00C9  hex 'C9'
+stringdef U+0118  hex 'CA'
+stringdef U+00CB  hex 'CB'
+stringdef U+011A  hex 'CC'
+stringdef U+00CD  hex 'CD'
+stringdef U+00CE  hex 'CE'
+stringdef U+010E  hex 'CF'
+stringdef U+0110  hex 'D0'
+stringdef U+0143  hex 'D1'
+stringdef U+0147  hex 'D2'
+stringdef U+00D3  hex 'D3'
+stringdef U+00D4  hex 'D4'
+stringdef U+0150  hex 'D5'
+stringdef U+00D6  hex 'D6'
+stringdef U+00D7  hex 'D7'
+stringdef U+0158  hex 'D8'
+stringdef U+016E  hex 'D9'
+stringdef U+00DA  hex 'DA'
+stringdef U+0170  hex 'DB'
+stringdef U+00DC  hex 'DC'
+stringdef U+00DD  hex 'DD'
+stringdef U+0162  hex 'DE'
+stringdef U+00DF  hex 'DF'
+stringdef U+0155  hex 'E0'
+stringdef U+00E1  hex 'E1'
+stringdef U+00E2  hex 'E2'
+stringdef U+0103  hex 'E3'
+stringdef U+00E4  hex 'E4'
+stringdef U+013A  hex 'E5'
+stringdef U+0107  hex 'E6'
+stringdef U+00E7  hex 'E7'
+stringdef U+010D  hex 'E8'
+stringdef U+00E9  hex 'E9'
+stringdef U+0119  hex 'EA'
+stringdef U+00EB  hex 'EB'
+stringdef U+011B  hex 'EC'
+stringdef U+00ED  hex 'ED'
+stringdef U+00EE  hex 'EE'
+stringdef U+010F  hex 'EF'
+stringdef U+0111  hex 'F0'
+stringdef U+0144  hex 'F1'
+stringdef U+0148  hex 'F2'
+stringdef U+00F3  hex 'F3'
+stringdef U+00F4  hex 'F4'
+stringdef U+0151  hex 'F5'
+stringdef U+00F6  hex 'F6'
+stringdef U+00F7  hex 'F7'
+stringdef U+0159  hex 'F8'
+stringdef U+016F  hex 'F9'
+stringdef U+00FA  hex 'FA'
+stringdef U+0171  hex 'FB'
+stringdef U+00FC  hex 'FC'
+stringdef U+00FD  hex 'FD'
+stringdef U+0163  hex 'FE'
+stringdef U+02D9  hex 'FF'

data/vendor/snowball/charsets/KOI8-R.sbl

@@ -0,0 +1,74 @@
+// KOI8-R character mappings.
+
+stringdef U+00A0  hex '9A'
+stringdef U+00A9  hex 'BF'
+stringdef U+00B0  hex '9C'
+stringdef U+00B2  hex '9D'
+stringdef U+00B7  hex '9E'
+stringdef U+00F7  hex '9F'
+stringdef U+0401  hex 'B3'
+stringdef U+0410  hex 'E1'
+stringdef U+0411  hex 'E2'
+stringdef U+0412  hex 'F7'
+stringdef U+0413  hex 'E7'
+stringdef U+0414  hex 'E4'
+stringdef U+0415  hex 'E5'
+stringdef U+0416  hex 'F6'
+stringdef U+0417  hex 'FA'
+stringdef U+0418  hex 'E9'
+stringdef U+0419  hex 'EA'
+stringdef U+041A  hex 'EB'
+stringdef U+041B  hex 'EC'
+stringdef U+041C  hex 'ED'
+stringdef U+041D  hex 'EE'
+stringdef U+041E  hex 'EF'
+stringdef U+041F  hex 'F0'
+stringdef U+0420  hex 'F2'
+stringdef U+0421  hex 'F3'
+stringdef U+0422  hex 'F4'
+stringdef U+0423  hex 'F5'
+stringdef U+0424  hex 'E6'
+stringdef U+0425  hex 'E8'
+stringdef U+0426  hex 'E3'
+stringdef U+0427  hex 'FE'
+stringdef U+0428  hex 'FB'
+stringdef U+0429  hex 'FD'
+stringdef U+042A  hex 'FF'
+stringdef U+042B  hex 'F9'
+stringdef U+042C  hex 'F8'
+stringdef U+042D  hex 'FC'
+stringdef U+042E  hex 'E0'
+stringdef U+042F  hex 'F1'
+stringdef U+0430  hex 'C1'
+stringdef U+0431  hex 'C2'
+stringdef U+0432  hex 'D7'
+stringdef U+0433  hex 'C7'
+stringdef U+0434  hex 'C4'
+stringdef U+0435  hex 'C5'
+stringdef U+0436  hex 'D6'
+stringdef U+0437  hex 'DA'
+stringdef U+0438  hex 'C9'
+stringdef U+0439  hex 'CA'
+stringdef U+043A  hex 'CB'
+stringdef U+043B  hex 'CC'
+stringdef U+043C  hex 'CD'
+stringdef U+043D  hex 'CE'
+stringdef U+043E  hex 'CF'
+stringdef U+043F  hex 'D0'
+stringdef U+0440  hex 'D2'
+stringdef U+0441  hex 'D3'
+stringdef U+0442  hex 'D4'
+stringdef U+0443  hex 'D5'
+stringdef U+0444  hex 'C6'
+stringdef U+0445  hex 'C8'
+stringdef U+0446  hex 'C3'
+stringdef U+0447  hex 'DE'
+stringdef U+0448  hex 'DB'
+stringdef U+0449  hex 'DD'
+stringdef U+044A  hex 'DF'
+stringdef U+044B  hex 'D9'
+stringdef U+044C  hex 'D8'
+stringdef U+044D  hex 'DC'
+stringdef U+044E  hex 'C0'
+stringdef U+044F  hex 'D1'
+stringdef U+0451  hex 'A3'