mittens 0.1.0

data/vendor/snowball/algorithms/hindi.sbl

@@ -0,0 +1,323 @@
+// An implementation of "A Lightweight Stemmer for Hindi":
+// http://www.kbcs.in/downloads/papers/StmmerHindi.pdf
+
+externals ( stem )
+
+stringescapes {}
+
+// The transliteration scheme used for our stringdefs matches that used in the
+// paper, as documented in the appendix.  It appears to match the WX notation
+// (https://en.wikipedia.org/wiki/WX_notation) except that WX apparently
+// uses 'z' for Anunasika whereas the paper uses Mh.
+//
+// We discriminate dependent vowels by adding a leading "_" to their stringdef
+// names (mnemonic: the _ signifies removing the implicit a from the preceding
+// character).
+
+// Vowels and sonorants:
+stringdef a  '{U+0905}'
+stringdef A  '{U+0906}'
+stringdef i  '{U+0907}'
+stringdef I  '{U+0908}'
+stringdef u  '{U+0909}'
+stringdef U  '{U+090A}'
+stringdef q  '{U+090B}'
+stringdef e  '{U+090F}'
+stringdef E  '{U+0910}'
+stringdef o  '{U+0913}'
+stringdef O  '{U+0914}'
+
+// Vowel signs:
+stringdef _A '{U+093E}'
+stringdef _i '{U+093F}'
+stringdef _I '{U+0940}'
+stringdef _u '{U+0941}'
+stringdef _U '{U+0942}'
+stringdef _q '{U+0943}'
+stringdef _e '{U+0947}'
+stringdef _E '{U+0948}'
+stringdef _o '{U+094B}'
+stringdef _O '{U+094C}'
+
+// Diacritics:
+stringdef M  '{U+0902}'
+stringdef H  '{U+0903}'
+stringdef Mh '{U+0901}'
+stringdef Z  '{U+093C}' // Nukta
+stringdef virama '{U+094D}'
+
+// Velar consonants:
+stringdef k  '{U+0915}'
+stringdef K  '{U+0916}'
+stringdef g  '{U+0917}'
+stringdef G  '{U+0918}'
+stringdef f  '{U+0919}'
+
+// Palatal consonants:
+stringdef c  '{U+091A}'
+stringdef C  '{U+091B}'
+stringdef j  '{U+091C}'
+stringdef J  '{U+091D}'
+stringdef F  '{U+091E}'
+
+// Retroflex consonants:
+stringdef t  '{U+091F}'
+stringdef T  '{U+0920}'
+stringdef d  '{U+0921}'
+stringdef D  '{U+0922}'
+stringdef N  '{U+0923}'
+
+// Dental consonants:
+stringdef w  '{U+0924}'
+stringdef W  '{U+0925}'
+stringdef x  '{U+0926}'
+stringdef X  '{U+0927}'
+stringdef n  '{U+0928}'
+
+// Labial consonants:
+stringdef p  '{U+092A}'
+stringdef P  '{U+092B}'
+stringdef b  '{U+092C}'
+stringdef B  '{U+092D}'
+stringdef m  '{U+092E}'
+
+// Semi-vowels:
+stringdef y  '{U+092F}'
+stringdef r  '{U+0930}'
+stringdef l  '{U+0932}'
+stringdef v  '{U+0935}'
+
+// Fricatives:
+stringdef S  '{U+0936}'
+stringdef R  '{U+0937}'
+stringdef s  '{U+0938}'
+stringdef h  '{U+0939}'
+
+stringdef lY '{U+0933}'
+
+// Precomposed characters - letters + nukta:
+stringdef nZ '{U+0929}' // ≡ {n}{Z}
+stringdef rZ '{U+0931}' // ≡ {r}{Z}
+stringdef lYZ '{U+0934}' // ≡ {lY}{Z}
+stringdef kZ '{U+0958}' // ≡ {k}{Z}
+stringdef KZ '{U+0959}' // ≡ {K}{Z}
+stringdef gZ '{U+095A}' // ≡ {g}{Z}
+stringdef jZ '{U+095B}' // ≡ {j}{Z}
+stringdef dZ '{U+095C}' // ≡ {d}{Z}
+stringdef DZ '{U+095D}' // ≡ {D}{Z}
+stringdef PZ '{U+095E}' // ≡ {P}{Z}
+stringdef yZ '{U+095F}' // ≡ {y}{Z}
+
+groupings ( consonant )
+
+routines ( CONSONANT )
+
+define consonant '{k}{K}{g}{G}{f}' +
+                 '{c}{C}{j}{J}{F}' +
+                 '{t}{T}{d}{D}{N}' +
+                 '{w}{W}{x}{X}{n}' +
+                 '{p}{P}{b}{B}{m}' +
+                 '{y}{r}{l}{v}' +
+                 '{S}{R}{s}{h}' +
+                 '{lY}' +
+                 '{Z}' + // Nukta
+                 // Precomposed characters - letter and nukta:
+                 '{nZ}{rZ}{lYZ}{kZ}{KZ}{gZ}{jZ}{dZ}{DZ}{PZ}{yZ}'
+
+backwardmode ( define CONSONANT as ( consonant ) )
+
+define stem as (
+    // We assume in this implementation that the whole word doesn't count
+    // as a valid suffix to remove, so we remove the longest suffix from
+    // the list which leaves at least one character.  This change affects
+    // 47 words out of the 65,140 in the sample vocabulary from Hindi
+    // wikipedia.
+    //
+    // The trick here is we use `next` in forward mode to advance the cursor
+    // to the second character, then `backwards` swaps the cursor and limit.
+    next
+    backwards (
+        [substring] among (
+            // The list below is derived from figure 3 in the paper.
+            //
+            // We perform the stemming on the Devanagari characters rather than
+            // transliterating to Latin, so we have adapted the list below to
+            // reflect this by converting suffixes back to Devanagari as
+            // follows:
+            //
+            // * within the suffixes, "a" after a consonant is dropped since
+            //   consonants have an implicit "a".
+            //
+            // * within the suffixes, a vowel other than "a" after a consonant
+            //   is a dependent vowel (vowel sign); a vowel (including "a")
+            //   after a non-consonant is an independent vowel.
+            //
+            // * to allow the vowel at the start of each suffix being dependent
+            //   or independent, we include each suffix twice.  For the
+            //   dependent version, a leading "a" is dropped and we check that
+            //   the suffix is preceded by a consonant (which will have an
+            //   implicit "a").
+            //
+            // * we add '{a}', which is needed for the example given right at
+            //   the end of section 5 to work (conflating BarawIya and
+            //   BarawIyawA), and which 3.1 a.v strongly suggests should be in
+            //   the list:
+            //
+            //     Thus, the following suffix deletions (longest possible
+            //     match) are required to reduce inflected forms of masculine
+            //     nouns to a common stem:
+            //     a A i [...]
+            //
+            //   Adding '{a}' only affect 2 words out of the 65,140 in the
+            //   sample vocabulary.
+            //
+            // * The transliterations of our stems would end with "a" when our
+            //   stems end in a consonant, so we also include {virama} in the
+            //   list of suffixes to remove (this affects 222 words from the
+            //   sample vocabulary).
+            //
+            // We've also assumed that Mh in the suffix list always means {Mh}
+            // and never {M}{h}{virama}.  Only one of the 65,140 words in the
+            // sample vocabulary stems differently due to this (and that word
+            // seems to be a typo).
+
+            '{virama}'
+
+            '{a}'
+            '{A}'
+            '{i}'
+            '{I}'
+            '{u}'
+            '{U}'
+            '{e}'
+            '{o}'
+            '{e}{M}'
+            '{o}{M}'
+            '{A}{M}'
+            '{u}{A}{M}'
+            '{u}{e}{M}'
+            '{u}{o}{M}'
+            '{A}{e}{M}'
+            '{A}{o}{M}'
+            '{i}{y}{_A}{M}'
+            '{i}{y}{_o}{M}'
+            '{A}{i}{y}{_A}{M}'
+            '{A}{i}{y}{_o}{M}'
+            '{A}{Mh}'
+            '{i}{y}{_A}{Mh}'
+            '{A}{i}{y}{_A}{Mh}'
+            '{a}{w}{_A}{e}{M}'
+            '{a}{w}{_A}{o}{M}'
+            '{a}{n}{_A}{e}{M}'
+            '{a}{n}{_A}{o}{M}'
+            '{a}{w}{_A}'
+            '{a}{w}{_I}'
+            '{I}{M}'
+            '{a}{w}{_I}{M}'
+            '{a}{w}{_e}'
+            '{A}{w}{_A}'
+            '{A}{w}{_I}'
+            '{A}{w}{_I}{M}'
+            '{A}{w}{_e}'
+            '{a}{n}{_A}'
+            '{a}{n}{_I}'
+            '{a}{n}{_e}'
+            '{A}{n}{_A}'
+            '{A}{n}{_e}'
+            '{U}{M}{g}{_A}'
+            '{U}{M}{g}{_I}'
+            '{A}{U}{M}{g}{_A}'
+            '{A}{U}{M}{g}{_I}'
+            '{e}{M}{g}{_e}'
+            '{e}{M}{g}{_I}'
+            '{A}{e}{M}{g}{_e}'
+            '{A}{e}{M}{g}{_I}'
+            '{o}{g}{_e}'
+            '{o}{g}{_I}'
+            '{A}{o}{g}{_e}'
+            '{A}{o}{g}{_I}'
+            '{e}{g}{_A}'
+            '{e}{g}{_I}'
+            '{A}{e}{g}{_A}'
+            '{A}{e}{g}{_I}'
+            '{A}{y}{_A}'
+            '{A}{e}'
+            '{A}{I}'
+            '{A}{I}{M}'
+            '{i}{e}'
+            '{A}{o}'
+            '{A}{i}{e}'
+            '{a}{k}{r}'
+            '{A}{k}{r}'
+
+            '{_A}'
+            '{_i}'
+            '{_I}'
+            '{_u}'
+            '{_U}'
+            '{_e}'
+            '{_o}'
+            '{_e}{M}'
+            '{_o}{M}'
+            '{_A}{M}'
+            '{_u}{A}{M}'
+            '{_u}{e}{M}'
+            '{_u}{o}{M}'
+            '{_A}{e}{M}'
+            '{_A}{o}{M}'
+            '{_i}{y}{_A}{M}'
+            '{_i}{y}{_o}{M}'
+            '{_A}{i}{y}{_A}{M}'
+            '{_A}{i}{y}{_o}{M}'
+            '{_A}{Mh}'
+            '{_i}{y}{_A}{Mh}'
+            '{_A}{i}{y}{_A}{Mh}'
+            '{_I}{M}'
+            '{_A}{w}{_A}'
+            '{_A}{w}{_I}'
+            '{_A}{w}{_I}{M}'
+            '{_A}{w}{_e}'
+            '{_A}{n}{_A}'
+            '{_A}{n}{_e}'
+            '{_U}{M}{g}{_A}'
+            '{_U}{M}{g}{_I}'
+            '{_A}{U}{M}{g}{_A}'
+            '{_A}{U}{M}{g}{_I}'
+            '{_e}{M}{g}{_e}'
+            '{_e}{M}{g}{_I}'
+            '{_A}{e}{M}{g}{_e}'
+            '{_A}{e}{M}{g}{_I}'
+            '{_o}{g}{_e}'
+            '{_o}{g}{_I}'
+            '{_A}{o}{g}{_e}'
+            '{_A}{o}{g}{_I}'
+            '{_e}{g}{_A}'
+            '{_e}{g}{_I}'
+            '{_A}{e}{g}{_A}'
+            '{_A}{e}{g}{_I}'
+            '{_A}{y}{_A}'
+            '{_A}{e}'
+            '{_A}{I}'
+            '{_A}{I}{M}'
+            '{_i}{e}'
+            '{_A}{o}'
+            '{_A}{i}{e}'
+            '{_A}{k}{r}'
+
+            /* Suffixes with a leading implicit a: */
+            '{w}{_A}{e}{M}' CONSONANT
+            '{w}{_A}{o}{M}' CONSONANT
+            '{n}{_A}{e}{M}' CONSONANT
+            '{n}{_A}{o}{M}' CONSONANT
+            '{w}{_A}' CONSONANT
+            '{w}{_I}' CONSONANT
+            '{w}{_I}{M}' CONSONANT
+            '{w}{_e}' CONSONANT
+            '{n}{_A}' CONSONANT
+            '{n}{_I}' CONSONANT
+            '{n}{_e}' CONSONANT
+            '{k}{r}' CONSONANT
+        )
+        delete
+    )
+)

data/vendor/snowball/algorithms/hungarian.sbl

@@ -0,0 +1,241 @@
+/*
+Hungarian Stemmer
+Removes noun inflections
+*/
+
+routines (
+    mark_regions
+    R1
+    v_ending
+    case
+    case_special
+    case_other
+    plural
+    owned
+    sing_owner
+    plur_owner
+    instrum
+    factive
+    undouble
+    double
+)
+
+externals ( stem )
+
+integers ( p1 )
+groupings ( v )
+
+stringescapes {}
+
+/* special characters */
+
+stringdef a'  '{U+00E1}'  //a-acute
+stringdef e'  '{U+00E9}'  //e-acute
+stringdef i'  '{U+00ED}'  //i-acute
+stringdef o'  '{U+00F3}'  //o-acute
+stringdef o"  '{U+00F6}'  //o-umlaut
+stringdef oq  '{U+0151}' //o-double acute
+stringdef u'  '{U+00FA}'  //u-acute
+stringdef u"  '{U+00FC}'  //u-umlaut
+stringdef uq  '{U+0171}' //u-double acute
+
+define v 'aeiou{a'}{e'}{i'}{o'}{o"}{oq}{u'}{u"}{uq}'
+
+define mark_regions as (
+
+    $p1 = limit
+
+    (v goto non-v
+     among('cs' 'gy' 'ly' 'ny' 'sz' 'ty' 'zs' 'dzs') or next
+     setmark p1)
+    or
+
+    (non-v gopast v setmark p1)
+)
+
+backwardmode (
+
+    define R1 as $p1 <= cursor
+
+    define v_ending as (
+        [substring] R1 among(
+            '{a'}' (<- 'a')
+            '{e'}' (<- 'e')
+        )
+    )
+
+    define double as (
+        test among('bb' 'cc' 'ccs' 'dd' 'ff' 'gg' 'ggy' 'jj' 'kk' 'll' 'lly' 'mm'
+        'nn' 'nny' 'pp' 'rr' 'ss' 'ssz' 'tt' 'tty' 'vv' 'zz' 'zzs')
+    )
+
+    define undouble as (
+        next [hop 1] delete
+    )
+
+    define instrum as(
+        [substring] R1 among(
+            'al' (double)
+            'el' (double)
+        )
+        delete
+        undouble
+    )
+
+
+    define case as (
+        [substring] R1 among(
+            'ban' 'ben'
+            'ba' 'be'
+            'ra' 're'
+            'nak' 'nek'
+            'val' 'vel'
+            't{o'}l' 't{oq}l'
+            'r{o'}l' 'r{oq}l'
+            'b{o'}l' 'b{oq}l'
+            'hoz' 'hez' 'h{o"}z'
+            'n{a'}l' 'n{e'}l'
+            'ig'
+            'at' 'et' 'ot' '{o"}t'
+            '{e'}rt'
+            'k{e'}pp' 'k{e'}ppen'
+            'kor'
+            'ul' '{u"}l'
+            'v{a'}' 'v{e'}'
+            'onk{e'}nt' 'enk{e'}nt' 'ank{e'}nt'
+            'k{e'}nt'
+            'en' 'on' 'an' '{o"}n'
+            'n'
+            't'
+        )
+        delete
+        v_ending
+    )
+
+    define case_special as(
+        [substring] R1 among(
+            '{e'}n' (<- 'e')
+            '{a'}n' (<- 'a')
+            '{a'}nk{e'}nt' (<- 'a')
+        )
+    )
+
+    define case_other as(
+        [substring] R1 among(
+            'astul' 'est{u"}l' (delete)
+            'stul' 'st{u"}l' (delete)
+            '{a'}stul' (<- 'a')
+            '{e'}st{u"}l' (<- 'e')
+        )
+    )
+
+    define factive as(
+        [substring] R1 among(
+            '{a'}' (double)
+            '{e'}' (double)
+        )
+        delete
+        undouble
+    )
+
+    define plural as (
+        [substring] R1 among(
+            '{a'}k' (<- 'a')
+            '{e'}k' (<- 'e')
+            '{o"}k' (delete)
+            'ak' (delete)
+            'ok' (delete)
+            'ek' (delete)
+            'k' (delete)
+        )
+    )
+
+    define owned as (
+        [substring] R1 among (
+            'ok{e'}' '{o"}k{e'}' 'ak{e'}' 'ek{e'}' (delete)
+            '{e'}k{e'}' (<- 'e')
+            '{a'}k{e'}' (<- 'a')
+            'k{e'}' (delete)
+            '{e'}{e'}i' (<- 'e')
+            '{a'}{e'}i' (<- 'a')
+            '{e'}i'  (delete)
+            '{e'}{e'}' (<- 'e')
+            '{e'}' (delete)
+        )
+    )
+
+    define sing_owner as (
+        [substring] R1 among(
+            '{u"}nk' 'unk' (delete)
+            '{a'}nk' (<- 'a')
+            '{e'}nk' (<- 'e')
+            'nk' (delete)
+            '{a'}juk' (<- 'a')
+            '{e'}j{u"}k' (<- 'e')
+            'juk' 'j{u"}k' (delete)
+            'uk' '{u"}k' (delete)
+            'em' 'om' 'am' (delete)
+            '{a'}m' (<- 'a')
+            '{e'}m' (<- 'e')
+            'm' (delete)
+            'od' 'ed' 'ad' '{o"}d' (delete)
+            '{a'}d' (<- 'a')
+            '{e'}d' (<- 'e')
+            'd' (delete)
+            'ja' 'je' (delete)
+            'a' 'e' 'o' (delete)
+            '{a'}' (<- 'a')
+            '{e'}' (<- 'e')
+        )
+    )
+
+    define plur_owner as (
+        [substring] R1 among(
+            'jaim' 'jeim' (delete)
+            '{a'}im' (<- 'a')
+            '{e'}im' (<- 'e')
+            'aim' 'eim' (delete)
+            'im' (delete)
+            'jaid' 'jeid' (delete)
+            '{a'}id' (<- 'a')
+            '{e'}id' (<- 'e')
+            'aid' 'eid' (delete)
+            'id' (delete)
+            'jai' 'jei' (delete)
+            '{a'}i' (<- 'a')
+            '{e'}i' (<- 'e')
+            'ai' 'ei' (delete)
+            'i' (delete)
+            'jaink' 'jeink' (delete)
+            'eink' 'aink' (delete)
+            '{a'}ink' (<- 'a')
+            '{e'}ink' (<- 'e')
+            'ink'
+            'jaitok' 'jeitek' (delete)
+            'aitok' 'eitek' (delete)
+            '{a'}itok' (<- 'a')
+            '{e'}itek' (<- 'e')
+            'itek' (delete)
+            'jeik' 'jaik' (delete)
+            'aik' 'eik' (delete)
+            '{a'}ik' (<- 'a')
+            '{e'}ik' (<- 'e')
+            'ik' (delete)
+        )
+    )
+)
+
+define stem as (
+    do mark_regions
+    backwards (
+      do instrum
+        do case
+        do case_special
+        do case_other
+        do factive
+        do owned
+        do sing_owner
+        do plur_owner
+        do plural
+    )
+)