lingo 1.8.1 → 1.8.2

data/porter/stem.cfg

@@ -1,311 +0,0 @@
-# Stem.cfg
-#
-# Rules for Porter-Stemmer
-#
-#
-# based on:
-#               An algorithm for suffix stripping
-#
-#                            M.F.Porter
-#                               1980
-#
-# Originally published in Program, 14 no. 3, pp 130-137, July 1980. (A
-# few typos have been corrected.)
-#
-# http://tartarus.org/~martin/PorterStemmer/def.txt
-#
-#         --------------------------------------------------
-#
-#
-#
-#
-# 2. THE ALGORITHM
-# 
-# To present the suffix stripping algorithm in its entirety we will need a few
-# difinitions.
-# 
-# A \consonant\ in a word is a letter other than A, E, I, O or U, and other
-# than Y preceded by a consonant. (The fact that the term `consonant' is
-# defined to some extent in terms of itself does not make it ambiguous.) So in
-# TOY the consonants are T and Y, and in SYZYGY they are S, Z and G. If a
-# letter is not a consonant it is a \vowel\.
-# 
-# A consonant will be denoted by c, a vowel by v. A list ccc... of length
-# greater than 0 will be denoted by C, and a list vvv... of length greater
-# than 0 will be denoted by V. Any word, or part of a word, therefore has one
-# of the four forms:
-# 
-#     CVCV ... C
-#     CVCV ... V
-#     VCVC ... C
-#     VCVC ... V
-# 
-# These may all be represented by the single form
-# 
-#     [C]VCVC ... [V]
-# 
-# where the square brackets denote arbitrary presence of their contents.
-# Using (VC){m} to denote VC repeated m times, this may again be written as
-# 
-#     [C](VC){m}[V].
-# 
-# m will be called the \measure\ of any word or word part when represented in
-# this form. The case m = 0 covers the null word. Here are some examples:
-# 
-#     m=0    TR,  EE,  TREE,  Y,  BY.
-#     m=1    TROUBLE,  OATS,  TREES,  IVY.
-#     m=2    TROUBLES,  PRIVATE,  OATEN,  ORRERY.
-# 
-# The \rules\ for removing a suffix will be given in the form
-# 
-#     (condition) S1 -> S2
-# 
-# This means that if a word ends with the suffix S1, and the stem before S1
-# satisfies the given condition, S1 is replaced by S2. The condition is
-# usually given in terms of m, e.g.
-# 
-#     (m > 1) EMENT ->
-# 
-# Here S1 is `EMENT' and S2 is null. This would map REPLACEMENT to REPLAC,
-# since REPLAC is a word part for which m = 2.
-# 
-# The `condition' part may also contain the following:
-# 
-# *S  - the stem ends with S (and similarly for the other letters).
-# 
-# *v* - the stem contains a vowel.
-# 
-# *d  - the stem ends with a double consonant (e.g. -TT, -SS).
-# 
-# *o  - the stem ends cvc, where the second c is not W, X or Y (e.g.
-#        -WIL, -HOP).
-# 
-# And the condition part may also contain expressions with \and\, \or\ and
-# \not\, so that
-# 
-#     (m>1 and (*S or *T))
-# 
-# tests for a stem with m>1 ending in S or T, while
-# 
-#     (*d and not (*L or *S or *Z))
-# 
-# tests for a stem ending witha double consonant other than L, S or Z.
-# Elaborate conditions like this are required only rarely.
-# 
-# In a set of rules written beneath each other, only one is obeyed, and this
-# will be the one with the longest matching S1 for the given word. For
-# example, with
-# 
-#     SSES -> SS
-#     IES  -> I
-#     SS   -> SS
-#     S    ->
-# 
-# (here the conditions are all null) CARESSES maps to CARESS since SSES is
-# the longest match for S1. Equally CARESS maps to CARESS (S1=`SS') and CARES
-# to CARE (S1=`S').
-# 
-#
----
-stemmer:
-  # In the rules below, examples of their application, successful or otherwise,
-  # are given on the right in lower case. The algorithm now follows:
-  #
-  # Step 1a
-  #     SSES -> SS                         caresses  ->  caress
-  #     IES  -> I                          ponies    ->  poni
-  #                                        ties      ->  ti
-  #     SS   -> SS                         caress    ->  caress
-  #     S    ->                            cats      ->  cat
-  S100:
-  - SSES -> SS
-  - IES  -> I
-  - SS   -> SS
-  - S    ->   
-  #
-  # Step 1b
-  #
-  #     (m>0) EED -> EE                    feed      ->  feed
-  #                                        agreed    ->  agree
-  #     (*v*) ED  ->                       plastered ->  plaster
-  #                                        bled      ->  bled
-  #     (*v*) ING ->                       motoring  ->  motor
-  #                                        sing      ->  sing
-  S110:
-  - (m>0) EED -> EE goto(S120)
-  - (*v*) ED  ->    goto(S111)
-  - (*v*) ING ->    goto(S111)
-  - goto(S120)
-  #
-  # If the second or third of the rules in Step 1b is successful, the following
-  # is done:
-  #
-  #     AT -> ATE                       conflat(ed)  ->  conflate
-  #     BL -> BLE                       troubl(ed)   ->  trouble
-  #     IZ -> IZE                       siz(ed)      ->  size
-  #     (*d and not (*L or *S or *Z))
-  #        -> single letter
-  #                                     hopp(ing)    ->  hop
-  #                                     tann(ed)     ->  tan
-  #                                     fall(ing)    ->  fall
-  #                                     hiss(ing)    ->  hiss
-  #                                     fizz(ed)     ->  fizz
-  #     (m=1 and *o) -> E               fail(ing)    ->  fail
-  #                                     fil(ing)     ->  file
-  S111:
-  - AT -> ATE
-  - BL -> BLE
-  - IZ -> IZE
-  - (*d and not (*L or *S or *Z)) -> -1
-  - (m=1 and *o) -> E
-  #
-  # The rule to map to a single letter causes the removal of one of the double
-  # letter pair. The -E is put back on -AT, -BL and -IZ, so that the suffixes
-  # -ATE, -BLE and -IZE can be recognised later. This E may be removed in step
-  # 4.
-  #
-  # Step 1c
-  #
-  #     (*v*) Y -> I                    happy        ->  happi
-  #                                     sky          ->  sky
-  S120:
-  - (*v*) Y -> I
-  #
-  # Step 1 deals with plurals and past participles. The subsequent steps are
-  # much more straightforward.
-  #
-  # Step 2
-  #
-  #     (m>0) ATIONAL ->  ATE           relational     ->  relate
-  #     (m>0) TIONAL  ->  TION          conditional    ->  condition
-  #                                     rational       ->  rational
-  #     (m>0) ENCI    ->  ENCE          valenci        ->  valence
-  #     (m>0) ANCI    ->  ANCE          hesitanci      ->  hesitance
-  #     (m>0) IZER    ->  IZE           digitizer      ->  digitize
-  #     (m>0) ABLI    ->  ABLE          conformabli    ->  conformable
-  #     (m>0) ALLI    ->  AL            radicalli      ->  radical
-  #     (m>0) ENTLI   ->  ENT           differentli    ->  different
-  #     (m>0) ELI     ->  E             vileli        - >  vile
-  #     (m>0) OUSLI   ->  OUS           analogousli    ->  analogous
-  #     (m>0) IZATION ->  IZE           vietnamization ->  vietnamize
-  #     (m>0) ATION   ->  ATE           predication    ->  predicate
-  #     (m>0) ATOR    ->  ATE           operator       ->  operate
-  #     (m>0) ALISM   ->  AL            feudalism      ->  feudal
-  #     (m>0) IVENESS ->  IVE           decisiveness   ->  decisive
-  #     (m>0) FULNESS ->  FUL           hopefulness    ->  hopeful
-  #     (m>0) OUSNESS ->  OUS           callousness    ->  callous
-  #     (m>0) ALITI   ->  AL            formaliti      ->  formal
-  #     (m>0) IVITI   ->  IVE           sensitiviti    ->  sensitive
-  #     (m>0) BILITI  ->  BLE           sensibiliti    ->  sensible
-  S200:
-  - (m>0) ATIONAL ->  ATE 
-  - (m>0) TIONAL  ->  TION  
-  - (m>0) ENCI    ->  ENCE
-  - (m>0) ANCI    ->  ANCE
-  - (m>0) IZER    ->  IZE 
-  - (m>0) ABLI    ->  ABLE
-  - (m>0) ALLI    ->  AL  
-  - (m>0) ENTLI   ->  ENT 
-  - (m>0) ELI     ->  E   
-  - (m>0) OUSLI   ->  OUS 
-  - (m>0) IZATION ->  IZE 
-  - (m>0) ATION   ->  ATE 
-  - (m>0) ATOR    ->  ATE 
-  - (m>0) ALISM   ->  AL  
-  - (m>0) IVENESS ->  IVE 
-  - (m>0) FULNESS ->  FUL 
-  - (m>0) OUSNESS ->  OUS 
-  - (m>0) ALITI   ->  AL  
-  - (m>0) IVITI   ->  IVE 
-  - (m>0) BILITI  ->  BLE 
-  #
-  # The test for the string S1 can be made fast by doing a program switch on
-  # the penultimate letter of the word being tested. This gives a fairly even
-  # breakdown of the possible values of the string S1. It will be seen in fact
-  # that the S1-strings in step 2 are presented here in the alphabetical order
-  # of their penultimate letter. Similar techniques may be applied in the other
-  # steps.
-  #
-  # Step 3
-  #
-  #     (m>0) ICATE ->  IC              triplicate     ->  triplic
-  #     (m>0) ATIVE ->                  formative      ->  form
-  #     (m>0) ALIZE ->  AL              formalize      ->  formal
-  #     (m>0) ICITI ->  IC              electriciti    ->  electric
-  #     (m>0) ICAL  ->  IC              electrical     ->  electric
-  #     (m>0) FUL   ->                  hopeful        ->  hope
-  #     (m>0) NESS  ->                  goodness       ->  good
-  S300:
-  - (m>0) ICATE ->  IC
-  - (m>0) ATIVE ->    
-  - (m>0) ALIZE ->  AL
-  - (m>0) ICITI ->  IC
-  - (m>0) ICAL  ->  IC
-  - (m>0) FUL   ->    
-  - (m>0) NESS  ->    
-  #
-  # Step 4
-  #
-  #     (m>1) AL    ->                  revival        ->  reviv
-  #     (m>1) ANCE  ->                  allowance      ->  allow
-  #     (m>1) ENCE  ->                  inference      ->  infer
-  #     (m>1) ER    ->                  airliner       ->  airlin
-  #     (m>1) IC    ->                  gyroscopic     ->  gyroscop
-  #     (m>1) ABLE  ->                  adjustable     ->  adjust
-  #     (m>1) IBLE  ->                  defensible     ->  defens
-  #     (m>1) ANT   ->                  irritant       ->  irrit
-  #     (m>1) EMENT ->                  replacement    ->  replac
-  #     (m>1) MENT  ->                  adjustment     ->  adjust
-  #     (m>1) ENT   ->                  dependent      ->  depend
-  #     (m>1 and (*S or *T)) ION ->     adoption       ->  adopt
-  #     (m>1) OU    ->                  homologou      ->  homolog
-  #     (m>1) ISM   ->                  communism      ->  commun
-  #     (m>1) ATE   ->                  activate       ->  activ
-  #     (m>1) ITI   ->                  angulariti     ->  angular
-  #     (m>1) OUS   ->                  homologous     ->  homolog
-  #     (m>1) IVE   ->                  effective      ->  effect
-  #     (m>1) IZE   ->                  bowdlerize     ->  bowdler
-  S400:
-  - (m>1) AL    ->  
-  - (m>1) ANCE  ->  
-  - (m>1) ENCE  ->  
-  - (m>1) ER    ->  
-  - (m>1) IC    ->  
-  - (m>1) ABLE  ->  
-  - (m>1) IBLE  ->  
-  - (m>1) ANT   ->  
-  - (m>1) EMENT ->  
-  - (m>1) MENT  ->  
-  - (m>1) ENT   ->  
-  - (m>1 and (*S or *T)) ION ->   
-  - (m>1) OU    ->    
-  - (m>1) ISM   ->    
-  - (m>1) ATE   ->    
-  - (m>1) ITI   ->    
-  - (m>1) OUS   ->    
-  - (m>1) IVE   ->    
-  - (m>1) IZE   ->    
-  #
-  # The suffixes are now removed. All that remains is a little tidying up.
-  #
-  # Step 5a
-  #
-  #     (m>1) E     ->                  probate        ->  probat
-  #                                     rate           ->  rate
-  #     (m=1 and not *o) E ->           cease          ->  ceas
-  S500:
-  - (m>1) E     ->  
-  - (m=1 and not *o) E ->   
-  #
-  # Step 5b
-  #
-  #     (m > 1 and *d and *L) -> single letter
-  #                                     controll       ->  control
-  #                                     roll           ->  roll
-  S510:
-  - (m > 1 and *d and *L) -> -1
-  #
-  # The algorithm is careful not to remove a suffix when the stem is too short,
-  # the length of the stem being given by its measure, m. There is no linguistic
-  # basis for this approach. It was merely observed that m could be used quite
-  # effectively to help decide whether or not it was wise to take off a suffix.

data/porter/stem.rb

@@ -1,150 +0,0 @@
-# encoding: utf-8
-
-require "yaml"
-class String
-  def to_shadow
-    shadow = self.gsub(/[^aeiouy]/, 'c')
-    shadow.gsub!(/[aeiou]/, 'v')
-    shadow.gsub!(/cy/, 'cv')
-    shadow.gsub!(/y/, 'c')
-    shadow
-  end
-end
-
-
-
-# => condition nil oder eine evaluierbare regel
-# => matchExp eine Regexp
-# => replacement ist downcase
-# => return new stem or nil, if rule didn't match
-def checkSingleRule(word, condition, matchExp, replacement)
-
-  # => check for matching rule
-  return nil unless matchExp.match(word)
-
-  # => remember stem
-  stem = $1
-
-  # => check condition for rule
-  unless condition.nil?
-    evalCondition = condition.dup
-
-    stemShadow = stem.to_shadow
-    
-    unless condition.index("m").nil?
-      m = stemShadow.squeeze.scan(/vc/).size
-      evalCondition.gsub!(/m/, m.to_s)
-    end
-    
-    unless condition.index("*v*").nil?
-      evalCondition.gsub!(/\*v\*/, stemShadow.index("v").nil? ? "false" : "true")
-    end
-    
-    unless condition.index("*d").nil?
-      evalCondition.gsub!(/\*d/, (stemShadow[-1..-1]=="c" && stem[-1]==stem[-2]) ? "true" : "false")
-    end
-    
-    unless condition.index("*o").nil?
-      bool = /cvc$/.match(stemShadow) && "wxy".index(stemShadow[-1..-1]).nil?
-      evalCondition.gsub!(/\*o/, bool ? "true" : "false")
-    end
-
-    while /\*(\w)/.match(evalCondition)
-      char = $1
-      if char.downcase == char
-        abort "unbekannter Buchstabe %s in Regel: %" % [char, condition]
-      end
-      
-      bool = (stem[-1..-1].upcase == char)
-      evalCondition.gsub!(Regexp.new(Regexp.escape("*#{char}")), bool ? "true" : "false")
-    end
-
-    evalCondition.gsub!(/and/, '&&')
-    evalCondition.gsub!(/or/, '||')
-    evalCondition.gsub!(/not/, '!')
-    evalCondition.gsub!(/=/, '==')
-p evalCondition
-    return unless eval(evalCondition)
-  end
-
-  # => stem with replacement
-  if /^(-\d+)$/.match(replacement)
-    # => delete last characters from stem, if replacement looks like '-1' oder '-2'
-    stem[0...($1.to_i)]
-  else
-    # => append replacement to stem
-    stem + replacement
-  end
-    
-end
-
-def checkAllRules(word, rules)
-  sequence = rules.keys.sort.reverse
-
-  actualRuleSet = sequence.pop.to_s
-
-  begin
-    label = nil
-
-    rules[actualRuleSet].each do |rule|
-      unless /^(\(.+\)){0,1}\s*(\S*)\s*->\s*(\S*?)\s*(?:goto\((\S+)\))*\s*$/.match(rule)
-        unless /^\s*goto\s*\(\s*(\S+)\s*\)$/.match(rule)
-          abort "ungültige Regel: %s" % rule
-        else
-          label = $1
-          break
-        end
-      end
-      
-      condition, ending, replacement, label = $1, $2.downcase, $3.downcase, $4     
-      p   [rule, word, condition, ending, replacement, label ]
-      result = checkSingleRule(word, condition, Regexp.new("(.+)#{ending}$"), replacement)
-
-      unless result.nil?
-        p [word, actualRuleSet, rule]
-        word = result
-        break
-      end
-    end
-    
-    if label.nil?
-      actualRuleSet = sequence.pop.to_s
-    else
-      while label != actualRuleSet && !actualRuleSet.nil?
-        actualRuleSet = sequence.pop.to_s
-      end
-    end
-  end until actualRuleSet.empty?
-
-  word
-end
-
-stemmerConfig = YAML::load_file("stem.cfg")
-
-$rules = stemmerConfig["stemmer"]
-
-word = $*[0]
-p checkAllRules(word, $rules)
-
-def test(word, stem)
-  result = checkAllRules(word, $rules)
-  if stem != result
-    warn "Falsches Wort %s, Stem %s, Result %s" % [word, stem, result]
-  else
-    warn "Korrekt: Wort %s, Stem %s" % [word, stem]
-  end
-end
-
-
-#test("caresses", "caress")
-#test("ponies", "poni")
-#test("ties", "ti")
-#test("caress", "caress")
-#test("cats", "cat")
-
-#test("feed", "feed")
-#?test("agreed", "agree")
-#test("plastered", "plaster")
-#test("bled", "bled")
-#test("motoring", "motor")
-#test("sing", "sing")