mittens 0.3.1 → 0.3.2

data/vendor/snowball/algorithms/esperanto.sbl

@@ -7,7 +7,6 @@ routines (
     initial_apostrophe
     long_word
     merged_numeral
-    not_after_letter
     pronoun
     standard_suffix
     ujn_suffix
@@ -124,19 +123,19 @@ backwardmode (
         loop 2 gopast vowel or (gopast '-' next) or gopast digit
     )
 
-    define not_after_letter as ('-' or digit)
-
     define standard_suffix as (
-        [substring try '-']
+        [
         among(
             'a' 'aj' 'ajn' 'an'
             'e' 'en'
             'i' 'as' 'is' 'os' 'u' 'us'
             'o' 'oj' 'ojn' 'on'
-            'j' not_after_letter
-            'jn' not_after_letter
-            'n' not_after_letter
+                ()
+            'j' 'jn' 'n'
+                (test ('-' or digit))
         )
+        try '-'
+        ]
         delete
     )
 )

data/vendor/snowball/algorithms/estonian.sbl

@@ -34,7 +34,7 @@ integers ( p1 )
 groupings ( V1 RV KI GI)
 
 define V1 'aeiou{o~}{a"}{o"}{u"}'
-define RV 'aeiuo'
+define RV 'aeiuo{'}'
 define KI 'kptgbdshf{sv}z{zv}'
 define GI 'cjlmnqrvwxaeiou{o~}{a"}{o"}{u"}'
 
@@ -42,7 +42,12 @@ define mark_regions as (
 
     $p1 = limit
 
-    gopast V1  gopast non-V1  setmark p1
+    (
+      hop 2  gopast '{'}'
+    ) or (
+      gopast V1  gopast non-V1
+    )
+    setmark p1
 )
 
 
@@ -131,7 +136,7 @@ backwardmode (
             'ikkude' (<-'iku') //plural genitive: õnnelikkude -> õnneliku
             'ikke' (<-'iku') //plural partitive: rahulikke -> rahuliku
             'ike' (<-'iku') //plural genitive: ohtlike -> ohtliku
-            'sid' (not LONGV delete) //plural partitive and sg2pst and pl3pst: auto-sid and laul-sid (exludes plural nominative with words like gaasid, roosid)
+            'sid' (not LONGV delete) //plural partitive and sg2pst and pl3pst: auto-sid and laul-sid (excludes plural nominative with words like gaasid, roosid)
             // plural genitive and pl2: ministri-te, oluliste -> olulise and saada-te, laula-te;
             // also torte -> tort (if not in compound word) and kokkuvõtte -> kokkuvõte and roheliste -> rohelise, tegemiste -> tegemise, teadlaste -> teadlase
             'te' (
@@ -264,6 +269,6 @@ define stem as (
         do emphasis
         do ( verb or substantive )
         do undouble_kpt
-
+        ['{'}'] delete
     )
 )

data/vendor/snowball/algorithms/finnish.sbl

@@ -11,7 +11,7 @@ routines (
            mark_regions
            R2
            particle_etc possessive
-           LONG VI
+           LV VI A E I O U A_ O_
            case_ending
            i_plural
            t_plural
@@ -24,7 +24,7 @@ externals ( stem )
 integers ( p1 p2 )
 strings ( x )
 booleans ( ending_removed )
-groupings ( AEI C V1 V2 particle_end )
+groupings ( AEI C v particle_end )
 
 stringescapes {}
 
@@ -32,20 +32,20 @@ stringescapes {}
 
 stringdef a"   '{U+00E4}'
 stringdef o"   '{U+00F6}'
+stringdef o/   '{U+00F8}'
 
 define AEI 'a{a"}ei'
 define C 'bcdfghjklmnpqrstvwxz'
-define V1 'aeiouy{a"}{o"}'
-define V2 'aeiou{a"}{o"}'
-define particle_end V1 + 'nt'
+define v 'aeiouy{a"}{o"}'
+define particle_end v + 'nt'
 
 define mark_regions as (
 
     $p1 = limit
     $p2 = limit
 
-    gopast V1  gopast non-V1  setmark p1
-    gopast V1  gopast non-V1  setmark p2
+    gopast v  gopast non-v  setmark p1
+    gopast v  gopast non-v  setmark p2
 )
 
 backwardmode (
@@ -88,28 +88,39 @@ backwardmode (
         )
     )
 
-    define LONG as
+    define LV as
         among('aa' 'ee' 'ii' 'oo' 'uu' '{a"}{a"}' '{o"}{o"}')
 
-    define VI as ('i' V2)
+    define VI as
+        among('ai' 'ei' 'ii' 'oi' 'ui' '{a"}i' '{o"}i' '{'}')
+
+    define A as ('a' or '{'}')
+    define E as ('e' or '{'}')
+    define I as ('i' or '{'}')
+    define O as ('o' or '{'}')
+    define U as ('u' or '{'}')
+    define A_ as ('{a"}' or '{'}')
+    // -ø-hön seen with Norwegian place names, e.g. Bodøhön
+    define O_ as ('{o"}' or '{o/}' or '{'}')
 
     define case_ending as (
         setlimit tomark p1 for ([substring])
         among(
-            'han'    ('a')          //-.
-            'hen'    ('e')          // |
-            'hin'    ('i')          // |
-            'hon'    ('o')          // |
-            'h{a"}n' ('{a"}')       // Illative   [43]
-            'h{o"}n' ('{o"}')       // |
+            'h{a"}n' A_             //-.
+            'h{o"}n' O_             // |
+            'han'    A              // |
+            'hen'    E              // |
+            'hin'    I              // Illative   [43]
+            'hon'    O              // |
+            'hun'    U              // |
             'siin'   VI             // |
-            'seen'   LONG           //-'
+            'seen'   LV             //-'
 
             'den'    VI
             'tten'   VI             // Genitive plurals [34]
                      ()
             'n'                     // Genitive or Illative
-                ( try ( LONG // Illative
+                ( try ( LV // Illative
                         or 'ie' // Genitive
                           and next ]
                       )
@@ -117,7 +128,7 @@ backwardmode (
                 )
 
             'a' '{a"}'              //-.
-                     (V1 C)         // |
+                     (v C)          // |
             'tta' 'tt{a"}'          // Partitive  [32]
                      ('e')          // |
             'ta' 't{a"}'            //-'
@@ -160,7 +171,7 @@ backwardmode (
     )
     define t_plural as (            // [26]
         setlimit tomark p1 for (
-            ['t'] test V1
+            ['t'] test v
             delete
         )
         setlimit tomark p2 for ([substring])
@@ -172,12 +183,13 @@ backwardmode (
     )
     define tidy as (
         setlimit tomark p1 for (
-            do ( LONG and ([next] delete ) ) // undouble vowel
+            do ( LV and ([next] delete ) ) // undouble vowel
             do ( [AEI] C delete ) // remove trailing a, a", e, i
             do ( ['j'] 'o' or 'u' delete )
             do ( ['o'] 'j' delete )
         )
-        goto non-V1 [C] -> x  x delete // undouble consonant
+        do (goto non-v [C] -> x  x delete) // undouble consonant
+        ['{'}'] delete
     )
 )
 

data/vendor/snowball/algorithms/german.sbl

@@ -148,6 +148,11 @@ backwardmode (
                 )
             )
         )
+        do (
+            [substring] among(
+                '{'}s' '{'}sch' '{'}' (next  not atlimit  delete)
+            )
+        )
     )
 )
 

data/vendor/snowball/algorithms/indonesian.sbl

@@ -34,11 +34,6 @@ routines (
     remove_first_order_prefix
     remove_second_order_prefix
     remove_suffix
-    KER
-    SUFFIX_KAN_OK
-    SUFFIX_AN_OK
-    SUFFIX_I_OK
-    VOWEL
 )
 
 externals ( stem )
@@ -59,101 +54,112 @@ backwardmode (
         )
     )
 
-    // prefix not in {ke, peng, per}
-    define SUFFIX_KAN_OK as (
-        // On page 29, the example "kompas Q.31" says "Both Nazief and Porter
-        // stemmer converted the word peledakan (blast, explotion [sic]) to
-        // ledak (to blast, to explode)".  However, the algorithm as described
-        // doesn't behave in this way - grammatically the prefix pe- occurs as a
-        // variation of both the first-order derivational prefix peng- and the
-        // second-order derivational prefix per-, but table 2.5 doesn't include
-        // "pe", only table 2.6 does, so "peledakan" is handled (incorrectly)
-        // as having prefix "per" not "peng", and so we remove derivational
-        // suffix "kan" rather than "an" to give stem leda.  (Porter-style
-        // stemmers remove the longest suffix they can amongst those available,
-        // which this paper notes in the last paragraph on page 15).
-        //
-        // We resolve this by amending the condition on suffix "kan" to
-        // "prefix ∉ {ke, peng, per}", which seems to make the stemmer's
-        // behaviour match all the examples in the paper except for one:
-        // "perbaikan" is shown in table 3.4 as stemming to "bai", but with
-        // this change it now stems to "baik".  The table notes that "baik" is
-        // the actual root so this deviation is an improvement.  In a sample
-        // vocabulary derived from the most common words in id.wikipedia.org,
-        // this change only affects 0.12% of words (76 out of 64,587, including
-        // "peledakan" and "perbaikan").
-        $prefix != 3 and $prefix != 2
-    )
-
-    // prefix not in {di, meng, ter}
-    define SUFFIX_AN_OK as ( $prefix != 1 )
-
-    define SUFFIX_I_OK as (
-        // prefix not in {ke, peng, ber}
-        $prefix <= 2
-
-        // The rest of the condition from the paper is:
-        //   V|K...c₁c₁, c₁ ≠ s, c₂ ≠ i
-        //
-        // The meaning of this is unclear in several ways, and none of the
-        // examples given of the stemmer's behaviour in the paper help to
-        // resolve these issues.
-        //
-        // Notice that c₂ isn't actually used - the most obvious explanation
-        // seems to be that "c₁c₁" should read "c₁c₂", or maybe "c₂c₁".
-        //
-        // Elsewhere the paper defines V... as meaning "the stem starts with
-        // a vowel" and K... as meaning "the stem starts with a consonant".
-        // The meaning of | isn't actually defined, but clearly means
-        // alternation.
-        //
-        // However nowhere is the precedence of | vs ... defined, and there
-        // isn't a standard precedence we could reasonably assume.  In other
-        // places where the paper says X|Y... it seems the | binds more
-        // tightly, so it's (V|K)...cᵢcⱼ not V|(K...cᵢcⱼ).  That seems a bit
-        // odd as the first letter must be either a vowel or a consonant, so
-        // that really just means "ends cᵢcⱼ" (and has at least one letter
-        // before cᵢ but we only call SUFFIX_I_OK if $measure > 2 which
-        // ensures that part).  However, nowhere in the paper uses or defines
-        // a notation such as ...X, which may explain this seemingly redundant
-        // way of specifying this.
-        //
-        // The conditions elsewhere on prefix removal (e.g. V...) are clearly
-        // on the stem left after the prefix is removed.  None of the other
-        // rules for suffix removal have conditions on the stem, but for
-        // consistency with the prefix rules we might expect that the cᵢcⱼ
-        // test is on what's left *after* removing the "i" suffix.
-        //
-        // Studying Indonesian wordlists and discussion with a native
-        // speaker leads us to conclude that the purpose of this check is to
-        // protect words of foreign origin (e.g. "televisi", "organisasi",
-        // "komunikasi") from stemming, and the common feature of these is
-        // that the word ends "-si", so we conclude that the condition here
-        // should be read as "word does not end -si", and this is what we
-        // have implemented.
-        not 's'
-    )
-
     define remove_suffix as (
         [substring] among (
-            'kan' SUFFIX_KAN_OK 'an' SUFFIX_AN_OK 'i' SUFFIX_I_OK
-                (delete $measure-=1)
+            'an' (
+                (
+                    // prefix not in {ke, peng, per}. [See SUFFIX_KAN_NOTE]
+                    ($prefix != 3 and $prefix != 2)
+                    // Remove suffix 'kan'
+                    'k']
+                ) or (
+                    // prefix not in {di, meng, ter}
+                    $prefix != 1
+                )
+            )
+            'i' (
+                // prefix not in {ke, peng, ber}
+                $prefix <= 2
+                // word does not end '-si'.  [See SUFFIX_I_NOTE]
+                not 's'
+            )
         )
+        delete $measure-=1
     )
+
+    // SUFFIX_KAN_NOTE:
+    // On page 29, the example "kompas Q.31" says "Both Nazief and Porter
+    // stemmer converted the word peledakan (blast, explotion [sic]) to
+    // ledak (to blast, to explode)".  However, the algorithm as described
+    // doesn't behave in this way - grammatically the prefix pe- occurs as a
+    // variation of both the first-order derivational prefix peng- and the
+    // second-order derivational prefix per-, but table 2.5 doesn't include
+    // "pe", only table 2.6 does, so "peledakan" is handled (incorrectly)
+    // as having prefix "per" not "peng", and so we remove derivational
+    // suffix "kan" rather than "an" to give stem leda.  (Porter-style
+    // stemmers remove the longest suffix they can amongst those available,
+    // which this paper notes in the last paragraph on page 15).
+    //
+    // We resolve this by amending the condition on suffix "kan" to
+    // "prefix ∉ {ke, peng, per}", which seems to make the stemmer's
+    // behaviour match all the examples in the paper except for one:
+    // "perbaikan" is shown in table 3.4 as stemming to "bai", but with
+    // this change it now stems to "baik".  The table notes that "baik" is
+    // the actual root so this deviation is an improvement.  In a sample
+    // vocabulary derived from the most common words in id.wikipedia.org,
+    // this change only affects 0.12% of words (76 out of 64,587, including
+    // "peledakan" and "perbaikan").
+
+    // SUFFIX_I_NOTE:
+    // The rest of the condition from the paper is:
+    //   V|K...c₁c₁, c₁ ≠ s, c₂ ≠ i
+    //
+    // The meaning of this is unclear in several ways, and none of the
+    // examples given of the stemmer's behaviour in the paper help to
+    // resolve these issues.
+    //
+    // Notice that c₂ isn't actually used - the most obvious explanation
+    // seems to be that "c₁c₁" should read "c₁c₂", or maybe "c₂c₁".
+    //
+    // Elsewhere the paper defines V... as meaning "the stem starts with
+    // a vowel" and K... as meaning "the stem starts with a consonant".
+    // The meaning of | isn't actually defined, but clearly means
+    // alternation.
+    //
+    // However nowhere is the precedence of | vs ... defined, and there
+    // isn't a standard precedence we could reasonably assume.  In other
+    // places where the paper says X|Y... it seems the | binds more
+    // tightly, so it's (V|K)...cᵢcⱼ not V|(K...cᵢcⱼ).  That seems a bit
+    // odd as the first letter must be either a vowel or a consonant, so
+    // that really just means "ends cᵢcⱼ" (and has at least one letter
+    // before cᵢ but we only call remove_suffix if $measure > 2 which
+    // ensures that part).  However, nowhere in the paper uses or defines
+    // a notation such as ...X, which may explain this seemingly redundant
+    // way of specifying this.
+    //
+    // The conditions elsewhere on prefix removal (e.g. V...) are clearly
+    // on the stem left after the prefix is removed.  None of the other
+    // rules for suffix removal have conditions on the stem, but for
+    // consistency with the prefix rules we might expect that the cᵢcⱼ
+    // test is on what's left *after* removing the "i" suffix.
+    //
+    // Studying Indonesian wordlists and discussion with a native
+    // speaker leads us to conclude that the purpose of this check is to
+    // protect words of foreign origin (e.g. "televisi", "organisasi",
+    // "komunikasi") from stemming, and the common feature of these is
+    // that the word ends "-si", so we conclude that the condition here
+    // should be read as "word does not end -si", and this is what we
+    // have implemented.
 )
 
 define vowel 'aeiou'
 
-define VOWEL as ( vowel )
-
-define KER as ( non-vowel 'er' )
-
 define remove_first_order_prefix as (
     [substring] among (
-        'di' 'meng' 'men' 'me' 'ter' (delete $prefix=1 $measure-=1)
-        'ke' 'peng' 'pen' (delete $prefix=3 $measure-=1)
-        'meny' VOWEL ($prefix=1 <-'s' $measure-=1)
-        'peny' VOWEL ($prefix=3 <-'s' $measure-=1)
+        'di' 'meng' 'me' 'ter'
+            (delete $prefix=1 $measure-=1)
+        'men' (
+            ('y' test vowel ] <-'s' $prefix=1 $measure-=1)
+            or
+            (delete $prefix=1 $measure-=1)
+        )
+        'ke' 'peng'
+            (delete $prefix=3 $measure-=1)
+        'pen' (
+            ('y' test vowel ] <-'s' $prefix=3 $measure-=1)
+            or
+            (delete $prefix=3 $measure-=1)
+        )
         'mem' ($prefix=1 $measure-=1 vowel and <-'p' or delete)
         'pem' ($prefix=3 $measure-=1 vowel and <-'p' or delete)
     )
@@ -165,13 +171,26 @@ define remove_second_order_prefix as (
     // is intended so that e.g. "pelajaran" stems to "ajar" not "lajar".
     // This change only affects a very small number of words (11 out of
     // 64,587) and only for the better.
-    [substring] among (
-        'per' 'pe' (delete $prefix=2 $measure-=1)
-        'pelajar' (<-'ajar' $measure-=1)
-        'ber' (delete $prefix=4 $measure-=1)
-        'belajar' (<-'ajar' $prefix=4 $measure-=1)
-        'be' KER (delete $prefix=4 $measure-=1)
+    [among (
+        'pe' (
+            ( 'r'] $prefix=2 )
+            or
+            ( 'l'] 'ajar' )
+            or
+            ( ] $prefix=2 )
+        )
+        'be' (
+            ( 'r'] )
+            or
+            ( 'l'] 'ajar')
+            or
+            ( ] non-vowel 'er' )
+            $prefix=4
+        )
     )
+    // All prefixes we remove here contain exactly one vowel.
+    $measure-=1
+    delete
 )
 
 define stem as (

data/vendor/snowball/algorithms/italian.sbl

@@ -1,5 +1,6 @@
 
 routines (
+           elisions
            prelude postlude mark_regions
            RV R1 R2
            attached_pronoun
@@ -31,6 +32,29 @@ stringdef u`   '{U+00F9}'
 
 define v 'aeiou{a`}{e`}{i`}{o`}{u`}'
 
+define elisions as (
+    [ substring ] not atlimit among (
+      // 'c{'}' doesn't seem useful to remove here.
+      'd{'}'      // e.g. d'Italia ("of Italy")
+      'l{'}'      // e.g. l'anno ("the year")
+      'm{'}'      // e.g. m'ama ("he loves me")
+      's{'}'      // e.g. s'innamora ("he falls in love")
+      't{'}'      // e.g. t'amo ("I love you")
+      'v{'}'      // e.g. v'adoro ("I adore you")
+      'all{'}'    // e.g. all'università ("at university")
+      'dall{'}'   // e.g. dall'album ("from the album")
+      'dell{'}'   // e.g. dell'anno ("of the year")
+      'gl{'}'     // e.g. gl'inglesi ("the English")
+      'nell{'}'   // e.g. nell'estate ("in the summer")
+      'quell{'}'  // e.g. quell'anno ("that year")
+      'quest{'}'  // e.g. quest'anno ("this year")
+      'sull{'}'   // e.g. sull'isola ("on the island")
+      'tutt{'}'   // e.g. tutt'Europa ("all of Europe")
+      'un{'}'     // e.g. un'eccentricità ("an eccentricity")
+    )
+    delete
+)
+
 define prelude as (
     test repeat (
         [substring] among(
@@ -96,7 +120,7 @@ backwardmode (
             'cela' 'cele' 'celi' 'celo' 'cene'
             'vela' 'vele' 'veli' 'velo' 'vene'
         )
-        among( (RV)
+        substring RV among(
             'ando' 'endo'   (delete)
             'ar' 'er' 'ir'  (<- 'e')
         )
@@ -181,6 +205,7 @@ backwardmode (
 )
 
 define stem as (
+      do elisions
       do prelude
       do mark_regions
       backwards (

data/vendor/snowball/algorithms/lithuanian.sbl

@@ -4,7 +4,7 @@ externals ( stem )
 stringescapes { }
 
 /* Special characters in Unicode Latin Extended-A */
-// ' nosine
+// k nosine
 stringdef ak   '{U+0105}'  // ą a + ogonek
 stringdef ek   '{U+0119}'  // ę e + ogonek
 stringdef ik   '{U+012F}'  // į i + ogonek
@@ -111,21 +111,20 @@ backwardmode (
 
 
       // V linksniuote (declension V)
-      'ies' 'ens' 'enio' 'ers'    // avies, vandens, sesers
-      'eniui' 'eriai'             // vandeniui, eriai
-      'en{ik}' 'er{ik}'           // vandenį, seserį
-      'imi' 'eniu' 'erimi' 'eria' // avimi, vandeniu, seserimi, seseria
-      'enyje' 'eryje'             // vandenyje, seseryje
-      'ie' 'enie' 'erie'          // avie, vandenie, seserie
-
-      'enys' 'erys'               // vandenys, seserys
+      'ies' 'ens' 'enio'          // avies, vandens, sesers
+      'eniui'                     // vandeniui
+      'en{ik}'                    // vandenį
+      'imi' 'eniu'                // avimi, vandeniu
+      'enyje'                     // vandenyje
+      'ie' 'enie'                 // avie, vandenide
+
+      'enys'                      // vandenys
       // 'en{uk}' konfliktas su 'žandenų' 'antenų'
-      'er{uk}'                    // seserų
-      'ims' 'enims' 'erims'       // avims, vandemins, seserims
+      'ims' 'enims'               // avims, vandemins
       'enis'                      // vandenis
       'imis'                      // žebenkštimis
       'enimis'                    // vandenimis
-      'yse' 'enyse' 'eryse'       // avyse, vandenyse, seseryse
+      'yse' 'enyse'               // avyse, vandenyse
 
 
       // Būdvardžiai (Adjectives)
@@ -355,7 +354,7 @@ define stem as (
 
   do (
     // priešdėlis 'a' ilgeniuose nei 6 raidės žodžiuose, pvz. 'a-liejus'.
-    try (test 'a' $(len > 6) hop 1)
+    try ('a' $(len > 6))
 
     gopast v  gopast non-v  setmark p1
   )
@@ -367,6 +366,7 @@ define stem as (
     do step2
     do fix_chdz
     do fix_gd
+    ['{'}'] delete
   )
 
 )

data/vendor/snowball/algorithms/norwegian.sbl

@@ -7,7 +7,7 @@ routines (
 
 externals ( stem )
 
-integers ( p1 x )
+integers ( p1 )
 
 groupings ( v s_ending )
 
@@ -28,12 +28,21 @@ define v 'ae{e^}io{o`}{o'}{o^}uy{ae}{ao}{o/}'
 define s_ending  'bcdfghjlmnoptvyz'
 
 define mark_regions as (
-
     $p1 = limit
 
-    test ( hop 3 setmark x )
-    gopast v  gopast non-v  setmark p1
-    try ( $p1 < x  $p1 = x )
+    do (
+      (
+        // If there's an apostrophe, start R1 after it to handle
+        // acronym loanwords such as "pc'en" and "ep'en".
+        gopast '{'}'
+      ) or (
+        gopast v  gopast non-v
+      )
+      setmark p1
+    )
+
+    // Ensure at least 3 characters before R1.
+    test (hop 3  do ($p1 < cursor  $p1 = cursor))
 )
 
 backwardmode (
@@ -85,10 +94,12 @@ backwardmode (
 
 define stem as (
 
-    do mark_regions
+    mark_regions
     backwards (
         do main_suffix
         do consonant_pair
         do other_suffix
+        // Remove trailing apostrophe.
+        ['{'}'] delete
     )
 )