estem 0.2.0

data/Rakefile

@@ -0,0 +1,18 @@
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.libs << 'test'
+end
+
+desc "Run tests"
+task :default => :test
+
+# Generate documentation.
+require 'rdoc/task'
+RDoc::Task.new do |rdoc|
+  rdoc.rdoc_files.include('README.rdoc',
+                          'lib/**/*',
+                          'bin/**/*')
+  rdoc.options = ['--main', 'README.rdoc']
+  rdoc.rdoc_dir = 'rdoc'
+end

data/bin/es_stem.rb

@@ -0,0 +1,178 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+
+# Copyright (c) 2012 Manuel A. Güílamo
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+require 'estem.rb'
+require 'getoptlong'
+require 'iconv'
+
+$version = "0.1.9"
+
+def usage(error=false)
+	out = error ? $stderr : $stdout
+	out.puts DATA.read()
+end
+
+opts = GetoptLong.new(
+	['--help', '-h', GetoptLong::NO_ARGUMENT],
+	['--version', '-v', GetoptLong::NO_ARGUMENT],
+	['--file', '-f', GetoptLong::REQUIRED_ARGUMENT],
+	['--in-enc', '-i', GetoptLong::REQUIRED_ARGUMENT],
+	['--out-enc', '-o', GetoptLong::REQUIRED_ARGUMENT])
+
+opts.quiet = true
+
+filename = nil
+ienc = nil
+oenc = nil
+
+begin
+	opts.each do |op, arg|
+		case op
+		when '--help'
+			usage()
+			exit
+		when '--version'
+			puts "EStem\nSpanish stemmer // lexemador\nVer: #{$version}"
+ 			exit
+		when '--file'
+			filename = arg
+		when '--in-enc'
+			ienc =  arg
+		when '--out-enc'
+			oenc = arg
+		end
+	end
+rescue GetoptLong::MissingArgument
+	$stderr.puts 'Option requires an argument // La opción requiere un argumento'
+	exit
+rescue GetoptLong::InvalidOption
+	$stderr.puts 'Unknown option // Opción desconocida.'
+	usage(true)
+	exit
+rescue
+	puts $!
+	exit
+end
+
+if filename
+	begin
+		if ienc and ienc!='UTF-8'
+			file = File.open(filename, "r:#{ienc}:UTF-8")
+		else
+			file = File.open(filename, 'r:UTF-8')
+		end
+	rescue
+		$stderr.puts $!
+		exit
+	end
+
+	begin
+		hsh = {}
+		file.each_line do|line|
+			line.split(/[^a-záéíóúüñÁÉÍÓÚÜÑ]+/ui).each do|word|
+				hsh[word] = word.es_stem unless hsh[word]
+			end
+		end
+	rescue
+		puts $!
+		exit
+	ensure
+		file.close
+	end
+else
+	hsh = {}
+	$stdin.each_line do|line|
+		if ienc
+			line = Iconv.conv('UTF-8', ienc, line)
+		else
+			# Just in case the terminal mess with the encoding name.
+			# Por si la terminal juega con el nombre de la codificación.
+			line.force_encoding('UTF-8')
+		end
+
+		begin
+			line.split(/[^a-záéíóúüñÁÉÍÓÚÜÑ]+/ui).each do|word|
+				hsh[word] = word.es_stem unless hsh[word]
+			end
+		rescue Encoding::CompatibilityError
+			if ienc
+				msg = "incompatible encoding, please use option " +
+				      "`--in-inc' correctly. //\n" +
+				      "codificación incompatible, por favor use la " +
+				      "opción `--in-inc' correctamente."
+			else
+				msg="incompatible encoding, please use option `--in-inc'."+
+				    " //\ncodificación incompatible, por favor use la " +
+				    "opción `--in-inc'."
+			end
+
+			if oenc
+				msg = Iconv.conv(oenc, 'UTF-8', msg)
+			end
+
+			$stderr.puts msg
+			exit
+		rescue
+			puts $!
+			exit
+		end
+	end
+end
+
+if oenc
+	begin
+		hsh.each_pair do |k,v|
+			puts Iconv.conv(oenc, 'UTF-8', "#{k} => #{v}")
+		end
+	rescue
+		puts $!
+		exit
+	end
+else
+	hsh.each_pair{ |k,v| puts "#{k} => #{v}" }
+end
+
+__END__
+Use: es_stem [OPTION]...
+
+Options:
+  --help, -h     display this help and exit. // Presenta esta ayuda y termina.
+  --version, -v  output version information and exit //
+                 Muestra la versión y termina.
+  --file, -f     file of words. // fichero de palabras.
+  --in-enc, -i   encoding of the file. // codificación del fichero.
+  --out-enc, -o  output encoding // codificación de salida.
+
+By default UTF-8 is used as input encoding, and if no file is specified,
+standard input will be used instead.
+
+You should set the option `--out-enc' if you are experimenting problems
+visualizing the output text.
+
+//
+
+Por defecto se usará UTF-8 como codificación de entrada, si no se especifica un
+fichero, la entrada estándard se usará en su lugar.
+
+Debería establecer la opción `--out-enc' si está experimentando problemas para
+visualizar el texto de salida.

data/lib/estem.rb

@@ -0,0 +1,196 @@
+# encoding: UTF-8
+#
+# Porter, Spanish stemmer in Ruby.
+#
+# :title: EStem - Ruby based Porter Spanish Stemmer
+
+module EStem
+	def es_stem
+		str = self.dup
+		return remove_accent(str) if str.length == 1
+		tmp = step0(str)
+		str = tmp ? tmp : str
+
+		unless tmp = step1(str)
+			unless tmp = step2a(str)
+				tmp = step2b(str)
+				str = tmp ? tmp : str
+			else
+				str = tmp
+			end
+		end
+		tmp = step3(str)
+		str = tmp.nil? ? str : tmp
+		remove_accent(str)
+	end
+
+	private
+
+	def vowel?(c)
+		VOWEL.include?(c)
+	end
+
+	def consonant?(c)
+		CONSONANT.include?(c)
+	end
+
+	def remove_accent(str)
+		str.tr('áéíóúÁÉÍÓÚ','aeiouAEIOU')
+	end
+
+	def rv(str)
+		if consonant? str[1]
+			i=2
+			i+=1 while str[i] and consonant? str[i]
+			return str.nil? ? str.length-1 : i+1
+		end
+
+		if vowel? str[0] and vowel? str[1]
+			i=2
+			i+=1 while str[i] and vowel? str[i]
+			return str.nil? ? str.length-1 : i+1
+		end
+
+		return 3 if consonant? str[0] and vowel? str[1]
+
+		str.length - 1
+	end
+
+	def r(str, i=0)
+		i+=1 while str[i] and consonant?(str[i])
+		i+=1
+		i+=1 while str[i] and vowel? str[i]
+		str[i].nil? ?  str.length : i+1
+	end
+
+	def r12(str)
+		r1 = r(str)
+		r2 = r(str,r1)
+		[r1,r2]
+	end
+
+	def step0(str)
+		return nil unless str =~ /(se(l[ao]s?)?|l([aeo]s?)|me|nos)$/i
+
+		suffix = $&
+		rv_text = str[rv(str)..-1]
+
+		case rv_text
+		when %r{((?<=i[éÉ]ndo|[áÁ]ndo|[áéíÁÉÍ]r)#{suffix})$}ui
+			str[%r{#$&$}]=''
+			str = remove_accent(str)
+			return str
+		when %r{((?<=iendo|ando|[aei]r)#{suffix})$}i
+			str[%r{#$&$}]=''
+			return str
+		end
+
+		if rv_text =~ /yendo/i and str =~ /uyendo/i
+		      str[suffix]=''
+		      return str
+		end
+		nil
+	end
+
+	#=> new_str or nil
+	def step1(str)
+		r1,r2 = r12(str)
+		r1_text = str[r1..-1]
+		r2_text = str[r2..-1]
+
+		case r2_text
+		when /(anzas?|ic[oa]s?|ismos?|[ai]bles?|istas?|os[oa]s?|[ai]mientos?)$/i
+			str[%r{#$&$}]=''
+			return str
+		when /(ic)?(ador([ae]s?)?|aci[óÓ]n|aciones|antes?|ancias?)$/ui
+			str[%r{#$&$}]=''
+			return str
+		when /log[íÍ]as?/ui
+			str[%r{#$&$}]='log'
+			return str
+		when /(uci([óÓ]n|ones))$/ui
+			str[%r{#$&$}]='u'
+			return str
+		when /(encias?)$/i
+			str[%r{#$&$}]='ente'
+			return str
+		end
+
+		if r2_text =~ /(ativ|iv|os|ic|ad)amente$/i or r1_text =~ /amente$/i
+			str[%r{#$&$}]=''
+			return str
+		end
+
+		case r2_text
+		when /((ante|[ai]ble)?mente)$/i, /((abil|i[cv])?idad(es)?)$/i, /((at)?iv[ao]s?)$/i
+			str[%r{#$&$}]=''
+			return str
+		end
+		nil
+	end
+
+	#=> nil or new_str
+	def step2a(str)
+		rv_pos = rv(str)
+		idx = str[rv_pos..-1] =~ /(y[oóÓ]|ye(ron|ndo)|y[ae][ns]?|ya(is|mos))$/ui
+
+		return nil unless idx
+
+		if 'u' == str[rv_pos+idx-1].downcase
+			str[%r{#$&$}] = ''
+			return str
+		end
+		nil
+	end
+
+	STEP2B_REGEXP = /(
+		ar([áÁ][ns]?|a(n|s|is)?|on)? | ar([éÉ]is|emos|é|É) | ar[íÍ]a(n|s|is|mos)? |
+		er([áÁ][sn]?|[éÉ](is)?|emos|[íÍ]a(n|s|is|mos)?)? |
+		ir([íÍ]a(s|n|is|mos)?|[áÁ][ns]?|emos|[éÉ]|éis)? | aba(s|n|is)? |
+		ad([ao]s?)? | ed | id(a|as|o|os)? | [íÍ]a(n|s|is|mos)? | [íÍ]s |
+		as(e[ns]?|te|eis|teis)? | [áÁ](is|bamos|semos|ramos) | a(n|ndo|mos) |
+		ie(ra|se|ran|sen|ron|ndo|ras|ses|rais|seis) | i(ste|steis|[óÓ]|mos|[éÉ]ramos|[éÉ]semos) |
+		en|es|[éÉ]is|emos
+	)$/xiu
+
+	def step2b(str)
+		rv_pos =  rv(str)
+
+		if idx = str[rv_pos..-1] =~ STEP2B_REGEXP
+			suffix = $&
+			if suffix =~ /^(en|es|[éÉ]is|emos)$/ui
+				str[%r{#{suffix}$}]=''
+				str[rv_pos+idx-1]='' if str[rv_pos+idx-2] =~ /g/i and  str[rv_pos+idx-1] =~ /u/i
+			else
+				str[%r{#{suffix}$}]=''
+			end
+			return str
+		end
+		nil
+	end
+
+	def step3(str)
+		rv_pos = rv(str)
+		rv_text = str[rv_pos..-1]
+
+		if rv_text =~ /(os|[aoáíóÁÍÓ])$/ui
+			str[%r{#$&$}]=''
+			return str
+		elsif idx = rv_text =~ /(u?[eéÉ])$/i
+			if $&[0].downcase == 'u' and str[rv_pos+idx-1].downcase == 'g'
+				str[%r{#$&$}]=''
+			else
+				str.chop!
+			end
+			return str
+		end
+		nil
+	end
+
+	VOWEL = 'aeiouáéíóúüAEIOUÁÉÍÓÚÜ'
+	CONSONANT = "bcdfghjklmnñpqrstvwxyzABCDEFGHIJKLMNÑOPQRSTUVWXYZ"
+end
+
+class String
+	include EStem
+end