raramorph 0.1.0

data/lib/raramorph/arabic_latin_translator.rb

@@ -0,0 +1,42 @@
+# Class For Arabic Latin Transileration
+# Author:: eSpace technologies  www.eSpace.com.eg
+# Copyright:: 2008
+#
+
+class ArabicLatinTranslator 
+  
+  # * Table Used for Tranlation From Arabic To English I.e ( Romanize Word )
+  # * According to  Buckwalter system Dictionary
+  TABLE =   { "\u0621"=> "'" , "\u0622"=> "|" , "\u0623"=> ">" , "\u0624"=> "&" , "\u0625"=> "<" , "\u0626"=> "}" ,
+  "\u0627"=> "A" , "\u0628"=> "b" , "\u0629"=> "p" , "\u062A"=> "t" , "\u062B"=> "v" , "\u062C"=> "j" ,
+  "\u062D"=> "H" , "\u062E"=> "x" , "\u062F"=> "d" , "\u0630"=> "*" , "\u0631"=> "r" , "\u0632"=> "z" ,
+  "\u0633"=> "s" , "\u0634"=> "$" , "\u0635"=> "S" , "\u0636"=> "D" , "\u0637"=> "T" ,"\u0638"=> "Z",
+  "\u0639"=> "E" , "\u063A"=> "g" , "\u0640"=> "_" , "\u0641"=> "f" , "\u0642"=> "q" , "\u0643"=> "k" , 
+  "\u0644"=> "l" , "\u0645"=> "m" , "\u0646"=> "n" , "\u0647"=> "h" , "\u0648"=> "w" , "\u0649"=> "Y", 
+  "\u064A"=> "y" , "\u064B"=> "F" , "\u064C"=> "N" , "\u064D"=> "K" , "\u064E"=> "a" , "\u064F"=> "u" ,
+  "\u0650"=> "i" , "\u0651"=> "~" , "\u0652"=> "o" , "\u0670"=> "`" ,"\u0671"=> "{" , "\u067E"=> "P" ,
+  "\u0686"=> "J" , "\u06A4"=> "V" , "\u06AF"=> "G" , "\u0698"=> "R" , "\u060C" => "," ,"\u061B" => ";",
+  "\u061F" => "?" , "\u0640" => ""   }
+  #Not suitable for morphological analysis : remove all vowels/diacritics, i.e. undo the job !
+  VOWEL_REMOVER = Regexp.compile("[FNKaui~o]")
+  STRIPER =  Regexp.compile("[`\\{]")
+ def initilaize
+   
+ end
+ 
+ # * Translate : Transilerate the arabic word to  Roman lettered Word
+ # * [word] Word String To be processed
+ # * @return transilerated word
+ #
+ def self.translate(word)
+   result = ""
+   word.gsub!(VOWEL_REMOVER , "")
+   word.gsub!(STRIPER , "")
+   word.force_encoding "UTF-8"
+   word.each_char{|char|
+    result+= TABLE[char] ? TABLE[char] : char
+   }   
+   result
+ end
+ 
+end

data/lib/raramorph/dictionary_entry.rb

@@ -0,0 +1,40 @@
+# Class For Storing Dictionary Entries
+# Author:: eSpace technologies  www.eSpace.com.eg
+# Copyright:: 2008
+#
+
+class DictionaryEntry
+        ## Constructs a Dictionary Entry
+        
+       attr_reader :entry , :lemma_id , :vocalization , :morphology , :gloss , :glosses , :pos
+       @@split_regex = Regexp.compile("\\+") 
+       
+	protected  
+  # * Initiliaze New Dict. Entry
+ def initialize( entry,  lemma_id,  vocalization,  morphology,  gloss,  pos)
+               # Instance Variables
+		@entry = entry.strip
+		@lemma_id = lemma_id.strip
+    @vocalization = vocalization.strip
+		@morphology = morphology.strip
+		@gloss = gloss
+    @glosses = []
+    @pos = []              
+    @glosses = fill_instance_array_from_sent_array(gloss.split(@@split_regex))
+	  @pos = fill_instance_array_from_sent_array(pos.split(@@split_regex))              
+ end
+ 
+ private  
+ def fill_instance_array_from_sent_array( sent_array)         	
+  instance_array = []
+ 	sent_array.each do  |value |  
+	 	value = value.strip
+  end
+   sent_array[0] == "" ?  offset = 1 :  offset = 0
+  for i in offset..sent_array.length-1 
+		instance_array[i - offset] = sent_array[i]
+  end
+  instance_array
+ end
+
+end

data/lib/raramorph/in_memory_dictionary_handler.rb

@@ -0,0 +1,287 @@
+# Class For Storing And Loading Dictionaries
+# Author:: eSpace technologies  www.eSpace.com.eg
+# Copyright:: 2008
+#
+
+require 'rubygems'
+class InMemoryDictionaryHandler
+  
+  #Signleton Class
+      ##### Dictionaries ########
+    #### Dictionaries are HASH OF ARRAYS #####
+       @@prefixes = {}
+    #Dictionary of Prefixes
+
+       @@stems = {}
+    #Dictionary of Stems
+ 
+       @@suffixes = {}
+     #Dictionary of Suffixes
+   private_class_method :new
+     
+  # * Loads Dictionaries and initiate variables 
+  def self.create 
+    
+    ### Variables #####
+      @@handler  = nil
+      @@regex = Regexp.compile(".*" + "<pos>(.+?)</pos>" + ".*")
+      @@morphology_regexs=[]
+      #@@leema_starter = Regexp.compile(";; ")
+      @@morphology_regexs[0] = Regexp.compile("^(Pref-0|Suff-0)$")
+      @@morphology_regexs[1] = Regexp.compile("^F" + ".*")
+      @@morphology_regexs[2] = Regexp.compile("^IV" + ".*")
+      @@morphology_regexs[3] = Regexp.compile("^PV" + ".*")
+      @@morphology_regexs[4] = Regexp.compile("^CV" + ".*")
+      @@morphology_regexs[5] = Regexp.compile("^N" + ".*")
+      @@morphology_regexs[6] = Regexp.compile("^[A-Z]" + ".*")
+      @@morphology_regexs[7] = Regexp.compile(".*" + "iy~$")
+      @@compatability_stpliter = Regexp.compile("\\s+")            
+      @@vocalization_array =[]
+      @@vocalization_array[0] = "/FUNC_WORD"
+      @@vocalization_array[1] ="/VERB_IMPERFECT"
+      @@vocalization_array[2] ="/VERB_PERFECT"
+      @@vocalization_array[3] ="/VERB_IMPERATIVE"
+      @@vocalization_array[4] = "/NOUN_PROP"
+      @@vocalization_array[5] ="/NOUN"
+      @@vocalization_array[6] = "/NOUN"
+  
+      @@prefixes_stems_compatibility = Set.new
+    #Changed
+    #Compatibility table for prefixes-stems combinations.
+  
+      @@prefixes_suffixes_compatibility = Set.new
+    #Changed
+    #Compatibility table for prefixes-suffixes combinations.
+  
+      @@stems_suffixes_compatibility = Set.new
+      
+    #Changed
+    #Compatibility table for stem-suffixes combinations.
+
+       puts "Initializing in-memory dictionary handler..."
+       Thread.abort_on_exception = true
+       load_dictionary( @@prefixes , "dictPrefixes"  ,  File.dirname(__FILE__) + "/../dictionaries/dictPrefixes"  )
+       load_stems_marshaled_dictionary
+       load_dictionary(@@suffixes, "dictSuffixes" ,  File.dirname(__FILE__) + "/../dictionaries/dictSuffixes")
+       load_compatibility_table(@@prefixes_stems_compatibility , "prefixes_stems_compatibility" ,  File.dirname(__FILE__) + "/../dictionaries/tableAB")
+       load_compatibility_table(@@prefixes_suffixes_compatibility , "prefixes_suffixes_compatibility" ,  File.dirname(__FILE__) + "/../dictionaries/tableAC")
+       load_compatibility_table(@@stems_suffixes_compatibility , "stems_suffixes_compatibility" ,  File.dirname(__FILE__) + "/../dictionaries/tableBC")
+       puts "... Done ... "
+             @@handler = new unless @@handler
+  end
+  
+  # * load the marshaled stems dictionary if avalaible or load from the origin dictionary if not avalaible
+  def self.load_stems_marshaled_dictionary 
+     if File.exists?( File.dirname(__FILE__) + '/../dictionaries/marshal_stems' ) 
+      File.open( File.dirname(__FILE__) + '/../dictionaries/marshal_stems') do |f|  
+         @@stems =  Marshal.load(f)
+       end
+        puts("#{@@stems.length}  entries totalizing")
+     else
+       reload_stems_dictionary
+     end         
+  end
+  
+  # * Marshal the stems dictionary into a file
+  def self.marshal_stems
+     File.open( File.dirname(__FILE__) + '/../dictionaries/marshal_stems' , 'w+') do |f|  
+        Marshal.dump(@@stems, f)  
+      end   
+  end
+  
+
+  # * Loads Stem dictionary from original file then marshal the dictionary for faster access
+  def self.reload_stems_dictionary
+    load_dictionary(@@stems, "dictStems",  File.dirname(__FILE__) + "/../dictionaries/dictStems") #File.open("dictionaries/dictStems" ,  "r:UTF-8" ))
+    marshal_stems
+  end
+  
+   # * Check if translitered word has prefix
+   # * [translitered] Translitered word to be checked
+  def has_prefix?(translitered)
+   	@@prefixes.has_key?(translitered)
+  end
+
+   # * Check if translitered word has stem
+   # * [translitered] Translitered word to be checked   
+  def has_stem?(translitered)
+    @@stems.has_key?(translitered)
+  end
+  
+   # * Check if translitered word has suffix
+   # * [translitered] Translitered word to be checked
+  def has_suffix?(translitered)
+    @@suffixes.has_key?(translitered)
+  end
+  
+   # * Check if prefix and stem are compatible 
+   # * [prefix] prefix to be checked
+   # * [stem] stem to be checked
+  def prefixes_stems_compatible?(prefix , stem) #String , #String
+    @@prefixes_stems_compatibility.member?(prefix + " " + stem)
+  end
+  
+   # * Check if prefix and suffix are compatible 
+   # * [prefix] prefix to be checked
+   # * [suffix] suffix to be checked
+  def prefixes_suffixes_compatible?(prefix , suffix)
+    @@prefixes_suffixes_compatibility.member?(prefix + " " + suffix)
+  end
+  
+   # * Check if stem and suffix are compatible 
+   # * [stem] stem to be checked
+   # * [suffix] suffix to be checked
+  def stems_suffixes_compatible?(stem , suffix)
+    @@stems_suffixes_compatibility.member?(stem + " " + suffix)
+  end
+  
+  # * Returns the prefixes table
+  def prefixes
+    @@prefixes
+  end
+  
+  def prefixes=(prefixes)
+    @@prefixes = prefixes
+  end
+  
+  # * Returns Stems Dictionary
+  def stems
+    @@stems
+  end
+   
+  def stems=(stems)
+    @@stems = stems
+  end
+  
+  
+ # * Returns Suffixes Dictionary
+  def suffixes
+    @@suffixes
+  end
+  
+  def suffixes=(suffixes)
+    @@suffixes = suffixes
+  end
+  
+ private 
+ 
+  # * load Dictionary from files  
+  # * [dictionary]  Hash of Arrays to store the Dictionary 
+  # * [name] Dictionary Name
+  # * [file] File Path
+  def self.load_dictionary(  dictionary , name , file )
+     lemmas = Set.new
+     forms = 0
+     final  = 0
+     lemma_id = ""
+     puts "Loading Dictionary : #{ name }"
+     #x  = Time.now
+     file = IO.readlines(file)
+     #@loading_secs += Time.now - x
+     line_count = 0 
+   #  leemas = file.select{|line| line.start_with?(@@leema_starter) }
+     file  = file.select{|line| line.start_with?(";; ") or !line.start_with?(";")  }
+    # entries = file.select{|line| !( line.start_with?(@@leema_starter) and line.start_with?(";")) }
+    # read_leemas(leemas)
+    # read_entries(entries)
+
+
+     file.each do |line|
+       # puts "." unless line_count % 1000
+        if line.start_with?(";; ")
+           lemma_id = line[3..line.length]
+          # Raise Exception If non-unique Lemma ID
+            raise ArgumentError.new("Lemma #{lemma_id } in #{name} #{line_count}  isn't unique") if lemmas.member?(lemma_id)
+          # Add The New Lemma
+           lemmas << lemma_id
+        #elsif line.start_with?(";")
+        else           
+           splited_line =  line.split("\t" , -1)
+           raise ArgumentError.new("Entry In #{name} line #{line_count} doesn't have 4 fields ( 3 tabs )") unless splited_line.length == 4   
+           de = self.construct_dictionary_entry(splited_line , name, line_count , lemma_id)
+           if  dictionary.has_key?(de.entry)
+        		dictionary[de.entry] << de
+      	   else 
+                 tmp_array = [] 
+                 tmp_array << de                 
+                 dictionary[de.entry] = tmp_array
+            end  
+           forms+=1; 
+       end
+       line_count+=1
+    end
+  #  file.close() 
+       #puts "Time Taken In If"  + @@if_time.to_s
+       #puts "Time Taken In Sub"  + @@sub_time.to_s
+     
+     puts "#{lemmas.size()}  lemmas and " unless lemma_id == ""
+     puts("#{dictionary.length}  entries totalizing  #{forms}  forms")
+  end 
+   
+  # * Load Compatibilty tables
+  # * [set] Set for Loading Compatibilty Tables
+  # * [name] Table Name
+  # * [file] File Path
+  def self.load_compatibility_table(set, name, file)
+    puts "Loading compatibility table : #{name}  "
+    file = IO.readlines(file)
+     file.each do |line|
+       unless (line.start_with?(";")) #Ignore comments
+		line = line.strip
+		line = line.gsub(@@compatability_stpliter, " ")
+		set << line#line
+      end
+   end
+   	puts  "#{set.size()} entries"
+ end  
+ 
+  # * Construct Dictionary Entry from line 
+  def self.construct_dictionary_entry(splited_line , name  ,line_count , lemma_id)
+             entry  = splited_line[0] 
+             vocalization = splited_line[1]
+             morphology = splited_line[2]
+             gloss_pos = splited_line[3]
+             gloss , pos = "" 
+             # two ways to get the POS info
+             # (1) explicitly, by extracting it from the gloss field:
+            
+             matcher = @@regex.match(gloss_pos) 
+              if matcher
+                 pos = matcher[1] #extract POS from glossPOS
+    		         gloss = gloss_pos #we clean up the gloss later (see below)
+              	# (2) by deduction: use the morphology (and sometimes the voc and gloss) to deduce the appropriate POS                 
+             else
+                 gloss= gloss_pos
+                 # we need the gloss to guess proper name
+                 
+                  if morphology.match(@@morphology_regexs[0])
+                    pos  = ""
+                  elsif morphology.match(@@morphology_regexs[1])
+                    pos = "#{vocalization} #{@@vocalization_array[0]}"
+            		  elsif (morphology.match(@@morphology_regexs[2]))
+            		    pos = "#{vocalization} #{ @@vocalization_array[1]}"
+           	  	  elsif (morphology.match(@@morphology_regexs[3])) 
+            		    pos = "#{vocalization} #{ @@vocalization_array[2]}"
+    		          elsif (morphology.match(@@morphology_regexs[4] )) 
+            		    pos = "#{vocalization} #{@@vocalization_array[3]}"
+           		    elsif (morphology.match(@@morphology_regexs[5])) 
+                            # educated guess (99% correct)  
+            			  if (gloss.match(@@morphology_regexs[6])) 
+         			      pos = "#{vocalization} #{@@vocalization_array[4]}"			
+					    		  #(was NOUN_ADJ: some of these are really ADJ's and need to be tagged manually)
+            			  elsif (vocalization.match(@@morphology_regexs[7]))
+          			     pos = "#{vocalization} #{@@vocalization_array[5]}"
+   		              else 
+          			     pos = "#{vocalization} #{@@vocalization_array[6]}"
+                    end                  
+                  else   raise "No POS can be deduced in #{ name}  (line  #{line_count}"
+                end
+            end   
+            # clean up the gloss: remove POS info and extra space, and convert upper-ASCII  to lower (it doesn't convert well to UTF-8)
+             gloss =gloss.sub(/<pos>.+?<\/pos>/,"")
+       	     gloss = gloss.strip 
+             translotor = Translator.new
+             gloss = translotor.translate(gloss)
+             DictionaryEntry.new(entry, lemma_id, vocalization, morphology, gloss, pos)
+    end
+end

data/lib/raramorph/in_memory_solutions_handler.rb

@@ -0,0 +1,78 @@
+# An in-memory handler for managing solutions found by the morphological analyzer.
+# 
+# Author:: eSpace technologies  www.eSpace.com.eg
+# Copyright:: 2008
+#
+
+class InMemorySolutionsHandler
+  
+  # The unique instance of this handler (singleton pattern)
+  # Constructor to avoid multiple instanciations
+  public_class_method :new
+  @@handler = nil
+  
+  def self.create
+    @@handler= new unless @@handler
+    @@handler
+  end
+    
+  public
+  
+  # Add solutions for a given word
+  # * [translitered] The translitered word.
+  # * [sol] The solution to the translitered word.
+  def add_solutions (translitered, sol)
+    @@solutions[translitered] = sol
+  end
+
+  # Whether or not the word already gave solutions
+  # * [translitered] The translitered word	  
+  #  * @return If it has the solution or not (Boolean).
+  def has_solutions(translitered)
+    @@solutions.has_key?(translitered)
+  end
+
+  # Return the solutions of a given word
+  # * [translitered] The translitered word
+  #  * @return The solution matching the transliterd word.
+  def get_solutions(translitered)
+    if(self.has_solutions(translitered))
+      return @@solutions[translitered]
+    else
+      return nil
+    end
+  end
+
+  # Add alternative spellings for the given word
+  # * [translitered] The translitered word
+  # * [alt] The alternative spelling
+  def add_alternative_spellings(translitered, alt)
+    @@alternative_spellings[translitered] = alt
+  end
+
+  # Whether or not the word already gave alternative spellings
+  # * [translitered] The translitered word
+  #  * @return If the transliterd word has alternative spellings
+  def has_alternative_spellings(translitered)
+    @@alternative_spellings.has_key?(translitered)
+  end
+
+  # Return the alternative spellings of the word
+  # * [translitered] The translitered word
+  #  * @return The alternative spellings matching the transliterd word.
+  def get_alternative_spellings(translitered)
+    if(self.has_alternative_spellings(translitered))
+      return @@alternative_spellings[translitered]
+    else
+      return nil
+    end    
+  end 
+
+private
+    #Hash of solutions for analyzed words
+    @@solutions ={}
+    #Hash of alternative spellings
+    @@alternative_spellings ={}
+
+end
+  

data/lib/raramorph/latin_arabic_translator.rb

@@ -0,0 +1,35 @@
+# Class For  Latin Arabic Transileration
+# Author:: eSpace technologies  www.eSpace.com.eg
+# Copyright:: 2008
+#
+
+class LatinArabicTranslator
+  
+ # * Table Used for Tranlation From Latin Letters To Arabic I.e ( Arabize Word )
+  # * According to  Buckwalter system Dictionary
+  TABLE = {"'" => "\u0621","|" => "\u0622",">" => "\u0623","&" => "\u0624",
+  "<" => "\u0625","}" => "\u0626","A" => "\u0627","b" => "\u0628",
+  "p" => "\u0629","t" => "\u062A","v" => "\u062B","j" => "\u062C",
+  "H" => "\u062D","x" => "\u062E","d" => "\u062F","*" => "\u0630",
+  "r" => "\u0631","z" => "\u0632", "s" => "\u0633","$" => "\u0634","S" => "\u0635",
+  "D" => "\u0636","T" => "\u0637","Z" => "\u0638","E" => "\u0639","g" => "\u063A", 
+  "_" => "\u0640","f" => "\u0641","q" => "\u0642","k" => "\u0643","l" => "\u0644",
+  "m" => "\u0645","n" => "\u0646","h" => "\u0647","w" => "\u0648","Y" => "\u0649","y" => "\u064A",
+  "F" => "\u064B","N" => "\u064C","K" => "\u064D","a" => "\u064E","u" => "\u064F","i" => "\u0650",
+  "~" => "\u0651", "o" => "\u0652",  "`" => "\u0670","{" => "\u0671","P" => "\u067E","J" => "\u0686",
+  "V" => "\u06A4",   "G" => "\u06AF", "R" => "\u0698" ,"," => "\u060C" , ";" => "\u061B" , "?" => "\u061F"
+  }
+  
+ # * Translate : Transilerate the Roman lettered word to  Arabic Word
+ # * [word] Word String To be processed
+ # * @return transilerated word
+ #
+  def self.translate(word)
+   result = ""
+   word.force_encoding "UTF-8"
+   word.each_char{|char|
+    result+= TABLE[char] ? TABLE[char] : char
+   }   
+   result
+  end
+end