mittens 0.1.0

data/vendor/snowball/algorithms/portuguese.sbl

@@ -0,0 +1,218 @@
+routines (
+           prelude postlude mark_regions
+           RV R1 R2
+           standard_suffix
+           verb_suffix
+           residual_suffix
+           residual_form
+)
+
+externals ( stem )
+
+integers ( pV p1 p2 )
+
+groupings ( v )
+
+stringescapes {}
+
+/* special characters */
+
+stringdef a'   '{U+00E1}'  // a-acute
+stringdef a^   '{U+00E2}'  // a-circumflex e.g. 'bota^nico
+stringdef e'   '{U+00E9}'  // e-acute
+stringdef e^   '{U+00EA}'  // e-circumflex
+stringdef i'   '{U+00ED}'  // i-acute
+stringdef o^   '{U+00F4}'  // o-circumflex
+stringdef o'   '{U+00F3}'  // o-acute
+stringdef u'   '{U+00FA}'  // u-acute
+stringdef c,   '{U+00E7}'  // c-cedilla
+
+stringdef a~   '{U+00E3}'  // a-tilde
+stringdef o~   '{U+00F5}'  // o-tilde
+
+
+define v 'aeiou{a'}{e'}{i'}{o'}{u'}{a^}{e^}{o^}'
+
+define prelude as repeat (
+    [substring] among(
+        '{a~}' (<- 'a~')
+        '{o~}' (<- 'o~')
+        ''     (next)
+    ) //or next
+)
+
+define mark_regions as (
+
+    $pV = limit
+    $p1 = limit
+    $p2 = limit  // defaults
+
+    do (
+        ( v (non-v gopast v) or (v gopast non-v) )
+        or
+        ( non-v (non-v gopast v) or (v next) )
+        setmark pV
+    )
+    do (
+        gopast v gopast non-v setmark p1
+        gopast v gopast non-v setmark p2
+    )
+)
+
+define postlude as repeat (
+    [substring] among(
+        'a~' (<- '{a~}')
+        'o~' (<- '{o~}')
+        ''   (next)
+    ) //or next
+)
+
+backwardmode (
+
+    define RV as $pV <= cursor
+    define R1 as $p1 <= cursor
+    define R2 as $p2 <= cursor
+
+    define standard_suffix as (
+        [substring] among(
+
+            'eza' 'ezas'
+            'ico' 'ica' 'icos' 'icas'
+            'ismo' 'ismos'
+            '{a'}vel'
+            '{i'}vel'
+            'ista' 'istas'
+            'oso' 'osa' 'osos' 'osas'
+            'amento' 'amentos'
+            'imento' 'imentos'
+
+           'adora' 'ador' 'a{c,}a~o'
+           'adoras' 'adores' 'a{c,}o~es'  // no -ic test
+           'ante' 'antes' '{a^}ncia' // Note 1
+            (
+                R2 delete
+            )
+            'logia'
+            'logias'
+            (
+                R2 <- 'log'
+            )
+            'u{c,}a~o' 'u{c,}o~es'
+            (
+                R2 <- 'u'
+            )
+            '{e^}ncia' '{e^}ncias'
+            (
+                R2 <- 'ente'
+            )
+            'amente'
+            (
+                R1 delete
+                try (
+                    [substring] R2 delete among(
+                        'iv' (['at'] R2 delete)
+                        'os'
+                        'ic'
+                        'ad'
+                    )
+                )
+            )
+            'mente'
+            (
+                R2 delete
+                try (
+                    [substring] among(
+                        'ante' // Note 1
+                        'avel'
+                        '{i'}vel' (R2 delete)
+                    )
+                )
+            )
+            'idade'
+            'idades'
+            (
+                R2 delete
+                try (
+                    [substring] among(
+                        'abil'
+                        'ic'
+                        'iv'   (R2 delete)
+                    )
+                )
+            )
+            'iva' 'ivo'
+            'ivas' 'ivos'
+            (
+                R2 delete
+                try (
+                    ['at'] R2 delete // but not a further   ['ic'] R2 delete
+                )
+            )
+            'ira' 'iras'
+            (
+                RV 'e'  // -eira -eiras usually non-verbal
+                <- 'ir'
+            )
+        )
+    )
+
+    define verb_suffix as setlimit tomark pV for (
+        [substring] among(
+            'ada' 'ida' 'ia' 'aria' 'eria' 'iria' 'ar{a'}' 'ara' 'er{a'}'
+            'era' 'ir{a'}' 'ava' 'asse' 'esse' 'isse' 'aste' 'este' 'iste'
+            'ei' 'arei' 'erei' 'irei' 'am' 'iam' 'ariam' 'eriam' 'iriam'
+            'aram' 'eram' 'iram' 'avam' 'em' 'arem' 'erem' 'irem' 'assem'
+            'essem' 'issem' 'ado' 'ido' 'ando' 'endo' 'indo' 'ara~o'
+            'era~o' 'ira~o' 'ar' 'er' 'ir' 'as' 'adas' 'idas' 'ias'
+            'arias' 'erias' 'irias' 'ar{a'}s' 'aras' 'er{a'}s' 'eras'
+            'ir{a'}s' 'avas' 'es' 'ardes' 'erdes' 'irdes' 'ares' 'eres'
+            'ires' 'asses' 'esses' 'isses' 'astes' 'estes' 'istes' 'is'
+            'ais' 'eis' '{i'}eis' 'ar{i'}eis' 'er{i'}eis' 'ir{i'}eis'
+            '{a'}reis' 'areis' '{e'}reis' 'ereis' '{i'}reis' 'ireis'
+            '{a'}sseis' '{e'}sseis' '{i'}sseis' '{a'}veis' 'ados' 'idos'
+            '{a'}mos' 'amos' '{i'}amos' 'ar{i'}amos' 'er{i'}amos'
+            'ir{i'}amos' '{a'}ramos' '{e'}ramos' '{i'}ramos' '{a'}vamos'
+            'emos' 'aremos' 'eremos' 'iremos' '{a'}ssemos' '{e^}ssemos'
+            '{i'}ssemos' 'imos' 'armos' 'ermos' 'irmos' 'eu' 'iu' 'ou'
+
+            'ira' 'iras'
+                (delete)
+        )
+    )
+
+    define residual_suffix as (
+        [substring] among(
+            'os'
+            'a' 'i' 'o' '{a'}' '{i'}' '{o'}'
+                ( RV delete )
+        )
+    )
+
+    define residual_form as (
+        [substring] among(
+            'e' '{e'}' '{e^}'
+                ( RV delete [('u'] test 'g') or
+                             ('i'] test 'c') RV delete )
+            '{c,}' (<-'c')
+        )
+    )
+)
+
+define stem as (
+    do prelude
+    do mark_regions
+    backwards (
+        do (
+            ( ( standard_suffix or verb_suffix )
+              and do ( ['i'] test 'c' RV delete )
+            )
+            or residual_suffix
+        )
+        do residual_form
+    )
+    do postlude
+)
+
+/*
+    Note 1: additions of 15 Jun 2005
+*/

data/vendor/snowball/algorithms/romanian.sbl

@@ -0,0 +1,236 @@
+
+routines (
+           prelude postlude mark_regions
+           RV R1 R2
+           step_0
+           standard_suffix combo_suffix
+           verb_suffix
+           vowel_suffix
+)
+
+externals ( stem )
+
+integers ( pV p1 p2 )
+
+groupings ( v )
+
+booleans  ( standard_suffix_removed )
+
+stringescapes {}
+
+/* special characters */
+
+stringdef a^   '{U+00E2}'  // a circumflex
+stringdef i^   '{U+00EE}'  // i circumflex
+stringdef a+   '{U+0103}'  // a breve
+stringdef s,   '{U+015F}'  // s cedilla
+stringdef t,   '{U+0163}'  // t cedilla
+
+define v 'aeiou{a^}{i^}{a+}'
+
+define prelude as (
+    repeat goto (
+        v [ ('u' ] v <- 'U') or
+            ('i' ] v <- 'I')
+    )
+)
+
+define mark_regions as (
+
+    $pV = limit
+    $p1 = limit
+    $p2 = limit // defaults
+
+    do (
+        ( v (non-v gopast v) or (v gopast non-v) )
+        or
+        ( non-v (non-v gopast v) or (v next) )
+        setmark pV
+    )
+    do (
+        gopast v gopast non-v setmark p1
+        gopast v gopast non-v setmark p2
+    )
+)
+
+define postlude as repeat (
+
+    [substring] among(
+        'I'  (<- 'i')
+        'U'  (<- 'u')
+        ''   (next)
+    )
+
+)
+
+backwardmode (
+
+    define RV as $pV <= cursor
+    define R1 as $p1 <= cursor
+    define R2 as $p2 <= cursor
+
+    define step_0 as (
+        [substring] R1 among(
+            'ul' 'ului'
+                ( delete )
+            'aua'
+                ( <-'a' )
+            'ea' 'ele' 'elor'
+                ( <-'e' )
+            'ii' 'iua' 'iei' 'iile' 'iilor' 'ilor'
+                ( <-'i')
+            'ile'
+                ( not 'ab' <- 'i' )
+            'atei'
+                ( <- 'at' )
+            'a{t,}ie' 'a{t,}ia'
+                ( <- 'a{t,}i' )
+        )
+    )
+
+    define combo_suffix as test (
+        [substring] R1 (
+            among(
+            /* 'IST'. alternative: include the following
+                'alism' 'alisme'
+                'alist' 'alista' 'aliste' 'alisti' 'alist{a+}' 'ali{s,}ti' (
+                    <- 'al'
+                )
+            */
+                'abilitate' 'abilitati' 'abilit{a+}i' 'abilit{a+}{t,}i' (
+                    <- 'abil'
+                )
+                'ibilitate' (
+                    <- 'ibil'
+                )
+                'ivitate' 'ivitati' 'ivit{a+}i' 'ivit{a+}{t,}i' (
+                    <- 'iv'
+                )
+                'icitate' 'icitati' 'icit{a+}i' 'icit{a+}{t,}i'
+                'icator' 'icatori'
+                'iciv' 'iciva' 'icive' 'icivi' 'iciv{a+}'
+                'ical' 'icala' 'icale' 'icali' 'ical{a+}' (
+                    <- 'ic'
+                )
+                'ativ' 'ativa' 'ative' 'ativi' 'ativ{a+}' 'a{t,}iune'
+                'atoare' 'ator' 'atori'
+                '{a+}toare' '{a+}tor' '{a+}tori' (
+                    <- 'at'
+                )
+                'itiv' 'itiva' 'itive' 'itivi' 'itiv{a+}' 'i{t,}iune'
+                'itoare' 'itor' 'itori' (
+                    <- 'it'
+                )
+            )
+            set standard_suffix_removed
+        )
+    )
+
+    define standard_suffix as (
+        unset standard_suffix_removed
+        repeat combo_suffix
+        [substring] R2 (
+            among(
+
+                // past participle is treated here, rather than
+                // as a verb ending:
+                'at' 'ata' 'at{a+}' 'ati' 'ate'
+                'ut' 'uta' 'ut{a+}' 'uti' 'ute'
+                'it' 'ita' 'it{a+}' 'iti' 'ite'
+
+                'ic' 'ica' 'ice' 'ici' 'ic{a+}'
+                'abil' 'abila' 'abile' 'abili' 'abil{a+}'
+                'ibil' 'ibila' 'ibile' 'ibili' 'ibil{a+}'
+                'oasa' 'oas{a+}' 'oase' 'os' 'osi' 'o{s,}i'
+                'ant' 'anta' 'ante' 'anti' 'ant{a+}'
+                'ator' 'atori'
+                'itate' 'itati' 'it{a+}i' 'it{a+}{t,}i'
+                'iv' 'iva' 'ive' 'ivi' 'iv{a+}' (
+                    delete
+                )
+                'iune' 'iuni' (
+                    '{t,}'] <- 't'
+                )
+                'ism' 'isme'
+                'ist' 'ista' 'iste' 'isti' 'ist{a+}' 'i{s,}ti' (
+                    <- 'ist'
+                    /* 'IST'. alternative: remove with <- '' */
+                )
+            )
+            set standard_suffix_removed
+        )
+    )
+
+    define verb_suffix as setlimit tomark pV for (
+        [substring] among(
+            // 'long' infinitive:
+            'are' 'ere' 'ire' '{a^}re'
+
+            // gerund:
+            'ind' '{a^}nd'
+            'indu' '{a^}ndu'
+
+            'eze'
+            'easc{a+}'
+            // present:
+            'ez' 'ezi' 'eaz{a+}' 'esc' 'e{s,}ti'
+            'e{s,}te'
+            '{a+}sc' '{a+}{s,}ti'
+            '{a+}{s,}te'
+
+            // imperfect:
+            'am' 'ai' 'au'
+            'eam' 'eai' 'ea' 'ea{t,}i' 'eau'
+            'iam' 'iai' 'ia' 'ia{t,}i' 'iau'
+
+            // past: // (not 'ii')
+            'ui'
+            'a{s,}i' 'ar{a+}m' 'ar{a+}{t,}i' 'ar{a+}'
+            'u{s,}i' 'ur{a+}m' 'ur{a+}{t,}i' 'ur{a+}'
+            'i{s,}i' 'ir{a+}m' 'ir{a+}{t,}i' 'ir{a+}'
+            '{a^}i' '{a^}{s,}i' '{a^}r{a+}m' '{a^}r{a+}{t,}i' '{a^}r{a+}'
+
+            // pluferfect:
+            'asem' 'ase{s,}i' 'ase' 'aser{a+}m' 'aser{a+}{t,}i' 'aser{a+}'
+            'isem' 'ise{s,}i' 'ise' 'iser{a+}m' 'iser{a+}{t,}i' 'iser{a+}'
+            '{a^}sem' '{a^}se{s,}i' '{a^}se' '{a^}ser{a+}m' '{a^}ser{a+}{t,}i'
+            '{a^}ser{a+}'
+            'usem' 'use{s,}i' 'use' 'user{a+}m' 'user{a+}{t,}i' 'user{a+}'
+
+                ( non-v or 'u'  delete )
+
+            // present:
+            '{a+}m' 'a{t,}i'
+            'em' 'e{t,}i'
+            'im' 'i{t,}i'
+            '{a^}m' '{a^}{t,}i'
+
+            // past:
+            'se{s,}i' 'ser{a+}m' 'ser{a+}{t,}i' 'ser{a+}'
+            'sei' 'se'
+
+            // pluperfect:
+            'sesem' 'sese{s,}i' 'sese' 'seser{a+}m' 'seser{a+}{t,}i' 'seser{a+}'
+                (delete)
+        )
+    )
+
+    define vowel_suffix as (
+        [substring] RV among (
+            'a' 'e' 'i' 'ie' '{a+}' ( delete )
+        )
+    )
+)
+
+define stem as (
+    do prelude
+    do mark_regions
+    backwards (
+        do step_0
+        do standard_suffix
+        do ( standard_suffix_removed or verb_suffix )
+        do vowel_suffix
+    )
+    do postlude
+)
+

data/vendor/snowball/algorithms/russian.sbl

@@ -0,0 +1,221 @@
+stringescapes {}
+
+/* the 33 Cyrillic letters represented in ASCII characters following the
+ * conventions of the standard Library of Congress transliteration: */
+
+stringdef a    '{U+0430}'
+stringdef b    '{U+0431}'
+stringdef v    '{U+0432}'
+stringdef g    '{U+0433}'
+stringdef d    '{U+0434}'
+stringdef e    '{U+0435}'
+stringdef e"   '{U+0451}'
+stringdef zh   '{U+0436}'
+stringdef z    '{U+0437}'
+stringdef i    '{U+0438}'
+stringdef i`   '{U+0439}'
+stringdef k    '{U+043A}'
+stringdef l    '{U+043B}'
+stringdef m    '{U+043C}'
+stringdef n    '{U+043D}'
+stringdef o    '{U+043E}'
+stringdef p    '{U+043F}'
+stringdef r    '{U+0440}'
+stringdef s    '{U+0441}'
+stringdef t    '{U+0442}'
+stringdef u    '{U+0443}'
+stringdef f    '{U+0444}'
+stringdef kh   '{U+0445}'
+stringdef ts   '{U+0446}'
+stringdef ch   '{U+0447}'
+stringdef sh   '{U+0448}'
+stringdef shch '{U+0449}'
+stringdef "    '{U+044A}'
+stringdef y    '{U+044B}'
+stringdef '    '{U+044C}'
+stringdef e`   '{U+044D}'
+stringdef iu   '{U+044E}'
+stringdef ia   '{U+044F}'
+
+routines ( mark_regions R2
+           perfective_gerund
+           adjective
+           adjectival
+           reflexive
+           verb
+           noun
+           derivational
+           tidy_up
+)
+
+externals ( stem )
+
+integers ( pV p2 )
+
+groupings ( v )
+
+define v '{a}{e}{i}{o}{u}{y}{e`}{iu}{ia}'
+
+define mark_regions as (
+
+    $pV = limit
+    $p2 = limit
+    do (
+        gopast v  setmark pV  gopast non-v
+        gopast v  gopast non-v  setmark p2
+       )
+)
+
+backwardmode (
+
+    define R2 as $p2 <= cursor
+
+    define perfective_gerund as (
+        [substring] among (
+            '{v}'
+            '{v}{sh}{i}'
+            '{v}{sh}{i}{s}{'}'
+                ('{a}' or '{ia}' delete)
+            '{i}{v}'
+            '{i}{v}{sh}{i}'
+            '{i}{v}{sh}{i}{s}{'}'
+            '{y}{v}'
+            '{y}{v}{sh}{i}'
+            '{y}{v}{sh}{i}{s}{'}'
+                (delete)
+        )
+    )
+
+    define adjective as (
+        [substring] among (
+            '{e}{e}' '{i}{e}' '{y}{e}' '{o}{e}' '{i}{m}{i}' '{y}{m}{i}'
+            '{e}{i`}' '{i}{i`}' '{y}{i`}' '{o}{i`}' '{e}{m}' '{i}{m}'
+            '{y}{m}' '{o}{m}' '{e}{g}{o}' '{o}{g}{o}' '{e}{m}{u}'
+            '{o}{m}{u}' '{i}{kh}' '{y}{kh}' '{u}{iu}' '{iu}{iu}' '{a}{ia}'
+            '{ia}{ia}'
+                        // and -
+            '{o}{iu}'   // - which is somewhat archaic
+            '{e}{iu}'   // - soft form of {o}{iu}
+                (delete)
+        )
+    )
+
+    define adjectival as (
+        adjective
+
+        /* of the participle forms, em, vsh, ivsh, yvsh are readily removable.
+           nn, {iu}shch, shch, u{iu}shch can be removed, with a small proportion of
+           errors. Removing im, uem, enn creates too many errors.
+        */
+
+        try (
+            [substring] among (
+                '{e}{m}'                  // present passive participle
+                '{n}{n}'                  // adjective from past passive participle
+                '{v}{sh}'                 // past active participle
+                '{iu}{shch}' '{shch}'     // present active participle
+                    ('{a}' or '{ia}' delete)
+
+     //but not  '{i}{m}' '{u}{e}{m}'      // present passive participle
+     //or       '{e}{n}{n}'               // adjective from past passive participle
+
+                '{i}{v}{sh}' '{y}{v}{sh}'// past active participle
+                '{u}{iu}{shch}'          // present active participle
+                    (delete)
+            )
+        )
+
+    )
+
+    define reflexive as (
+        [substring] among (
+            '{s}{ia}'
+            '{s}{'}'
+                (delete)
+        )
+    )
+
+    define verb as (
+        [substring] among (
+            '{l}{a}' '{n}{a}' '{e}{t}{e}' '{i`}{t}{e}' '{l}{i}' '{i`}'
+            '{l}' '{e}{m}' '{n}' '{l}{o}' '{n}{o}' '{e}{t}' '{iu}{t}'
+            '{n}{y}' '{t}{'}' '{e}{sh}{'}'
+
+            '{n}{n}{o}'
+                ('{a}' or '{ia}' delete)
+
+            '{i}{l}{a}' '{y}{l}{a}' '{e}{n}{a}' '{e}{i`}{t}{e}'
+            '{u}{i`}{t}{e}' '{i}{t}{e}' '{i}{l}{i}' '{y}{l}{i}' '{e}{i`}'
+            '{u}{i`}' '{i}{l}' '{y}{l}' '{i}{m}' '{y}{m}' '{e}{n}'
+            '{i}{l}{o}' '{y}{l}{o}' '{e}{n}{o}' '{ia}{t}' '{u}{e}{t}'
+            '{u}{iu}{t}' '{i}{t}' '{y}{t}' '{e}{n}{y}' '{i}{t}{'}'
+            '{y}{t}{'}' '{i}{sh}{'}' '{u}{iu}' '{iu}'
+                (delete)
+            /* note the short passive participle tests:
+               '{n}{a}' '{n}' '{n}{o}' '{n}{y}'
+               '{e}{n}{a}' '{e}{n}' '{e}{n}{o}' '{e}{n}{y}'
+            */
+        )
+    )
+
+    define noun as (
+        [substring] among (
+            '{a}' '{e}{v}' '{o}{v}' '{i}{e}' '{'}{e}' '{e}'
+            '{i}{ia}{m}{i}' '{ia}{m}{i}' '{a}{m}{i}' '{e}{i}' '{i}{i}'
+            '{i}' '{i}{e}{i`}' '{e}{i`}' '{o}{i`}' '{i}{i`}' '{i`}'
+            '{i}{ia}{m}' '{ia}{m}' '{i}{e}{m}' '{e}{m}' '{a}{m}' '{o}{m}'
+            '{o}' '{u}' '{a}{kh}' '{i}{ia}{kh}' '{ia}{kh}' '{y}' '{'}'
+            '{i}{iu}' '{'}{iu}' '{iu}' '{i}{ia}' '{'}{ia}' '{ia}'
+                (delete)
+            /* the small class of neuter forms '{e}{n}{i}' '{e}{n}{e}{m}'
+               '{e}{n}{a}' '{e}{n}' '{e}{n}{a}{m}' '{e}{n}{a}{m}{i}' '{e}{n}{a}{x}'
+               omitted - they only occur on 12 words.
+            */
+        )
+    )
+
+    define derivational as (
+        [substring] R2 among (
+            '{o}{s}{t}'
+            '{o}{s}{t}{'}'
+                (delete)
+        )
+    )
+
+    define tidy_up as (
+        [substring] among (
+
+            '{e}{i`}{sh}'
+            '{e}{i`}{sh}{e}'  // superlative forms
+               (delete
+                ['{n}'] '{n}' delete
+               )
+            '{n}'
+               ('{n}' delete) // e.g. -nno endings
+            '{'}'
+               (delete)  // with some slight false conflations
+        )
+    )
+)
+
+define stem as (
+
+    // Normalise {e"} to {e}.  The documentation has long suggested the user
+    // should do this before calling the stemmer - we now do it for them.
+    do repeat ( goto (['{e"}']) <- '{e}' )
+
+    do mark_regions
+    backwards setlimit tomark pV for (
+        do (
+             perfective_gerund or
+             ( try reflexive
+               adjectival or verb or noun
+             )
+        )
+        try([ '{i}' ] delete)
+        // because noun ending -i{iu} is being treated as verb ending -{iu}
+
+        do derivational
+        do tidy_up
+    )
+)