shalmaneser 1.2.0.rc4 → 1.2.rc5

data/lib/frprep/FNDatabase.rb

@@ -1,144 +0,0 @@
-# sp 28 06 04
-#
-# this module offers methods to extract gemma corpora from the FrameNet database#
-
-require 'frprep/FrameXML'
-
-class FNDatabase
-    
-  def each_matching_sentence(file_pred,sent_pred)     
-    # fundamental access function to FrameXML files
-
-    # returns file objects where
-    # FrameXMLSentence matches sent_pred
-    # (FrameXMLFile is accessed through FrameXMLSentence.get_file_object and matches file_pred)
-    each_matching_file(file_pred) {|frameNetFile|    
-      frameNetFile.each_sentence {|frameNetSent|
-	if sent_pred.call(frameNetSent)
-	  frameNetSent.verify_annotation
-	  yield frameNetSent
-	end
-      }
-    }
-  end 
-  
-  def each_matching_file(file_pred)   
-    # fundamental access function to FrameXML files
-
-    # returns file (FrameXMLFile) objects which match file_pred
-    each_framexml_file{|frameNetFile|    
-      if file_pred.call(frameNetFile)
-	yield frameNetFile
-      end
-      frameNetFile.close
-    }
-  end
-    
-  def extract_frame(frame,outfile)
-    each_matching_sentence(Proc.new{|fnfile| fnfile.get_frame == frame},
-			   Proc.new{|fnsent| true}) {|fnsent|
-      if fnsent.contains_FE_annotation_and_target
-	fnsent.print_conll_style_to(outfile)
-      end
-    }
-  end
-  
-  def extract_lemma(lemma,outfile) 
-    each_matching_sentence(Proc.new{|fnfile| fnfile.get_lu == lemma},
-			   Proc.new{|fnsent| true}) {|fnsent|
-      if fnsent.contains_FE_annotation_and_target
-	fnsent.print_conll_style_to(outfile)
-      end
-    }
-  end
-
-  def extract_everything(outdirectory)
-    unless outdirectory[-1,1] == "/"
-      outdirectory += "/"
-    end
-
-    outfiles = Hash.new
-    each_matching_sentence(Proc.new{|fnfile| true},
-			   Proc.new{|fnsent| true}) {|fnsent|
-      frame = fnsent.get_file_obj.get_frame
-      unless outfiles.key?(frame)
-	outfiles[frame] = File.new(outdirectory+frame+".tab","w")
-      end
-      if fnsent.contains_FE_annotation_and_target
-	fnsent.print_conll_style_to(outfiles[frame])
-      end
-    }
-    # close output files
-    outfiles.each_value {|file|
-      file.close
-    }
-    # remove zero-size files
-    Dir[outdirectory+"*"].each {|filename|
-      if FileTest.zero?(filename)
-	File.unlink(filename)
-      end
-    }
-  end
-    
-    
-  def initialize(fn_path)
-    unless fn_path[-1,1] == "/"
-      fn_path += "/"
-    end
-    @fn = fn_path
-  end
-  
-  private 
-
-  def each_framexml_file
-    # files might be zipped
-    Dir[@fn+"lu*.xml.gz"].each {|gzfile| 
-      Kernel.system("cp "+gzfile+" /tmp/")
-      Kernel.system("gunzip -f /tmp/"+File.basename(gzfile))
-      gzfile =~ /(.+)\.gz/
-      yield FrameXMLFile.new("/tmp/"+File.basename($1))
-    }
-    # or might not
-    Dir[@fn+"/lu*.xml"].each {|filename|
-      yield FrameXMLFile.new(filename)
-    }
-  end
-
-  # I  don't really remember what this was good for ;-)
-
-#   def browse_everything(allFiles)
-#     if allFiles
-#       Dir[fn+"*.xml.gz"].each {|gzfile|
-# 	Kernel.system("cp "+gzfile+" /tmp/")
-# 	Kernel.system("gunzip -f /tmp/"+File.basename(gzfile))
-# 	gzfile =~ /(.+)\.gz/
-# 	#    STDERR.puts File.basename($1)
-# 	#    STDERR.print "."
-# 	ff = FrameXMLFile.new("/tmp/"+File.basename($1))
-# 	ff.each_sentence {|s|
-# 	  if s.contains_FE_annotation_and_target
-# 	    s.verify_annotation  
-# 	    if s.verify_annotation
-# 	    puts "****************** Error: Still problems after 2nd verification!"
-# 	    end
-# 	    s.print_conll_style
-# 	  end
-# 	}
-#       }
-#     else
-#       ff = FrameXMLFile.new("/tmp/lu1870.xml")
-#       ff.each_sentence {|s|
-# 	if s.contains_FE_annotation_and_target
-# 	  s.verify_annotation
-# 	if s.verify_annotation
-# 	  puts "****************** Error: Still problems after 2nd verification!"
-# 	end
-# 	  #      s.print_layers
-# 	  s.print_conll_style
-# 	end
-#       }
-#     end
-#   end
-  
-end
-

data/lib/frprep/FrameXML.rb

@@ -1,513 +0,0 @@
-# sp 18 06 2004
-#
-# access to FrameNet XML files, sentences, and annotation.
-#
-# sp 10 11 04: only data from the first layer with name XY is
-# used for output. Other data is saved in layer XY.2nd, but is 
-# currently not processed.
-#
-# sp 22 05 04: also, if two labels exist which cover the same span
-# (ie there is a double annotation within the same layer), ignore
-# all but the first label.
-#
-# ke 13 07 05: 
-#   - changed to RegXMl.rb
-#   - fixed two problems in analyse_layer:
-#     - Deleting problematic labels: 
-#       For some reason, thisLayer[i+1..-1].each_index {|other_i|
-#       included the index 0 in any case, resulting in the 1st
-#       label being deleted in any case.
-#     - Deleting problematic labels, checking for label overlap:
-#       The old formulation worked only if labels occurred in the array
-#       in the order they occurred in the sentence, but that was not the case.
-#   - Change in deleting problematic labels:
-#     No longer delete duplicate labels, since e.g. in the PT level there
-#     may be more than one NP label, and we want to keep those
-#
-# KE January 2007:
-# write new adapted FNTab format
-# ( "word", ("pt", "gf", "role", "target", "frame", "stuff")* "ne", "sent_id" )
-
-require 'frprep/Ampersand'
-require 'common/ISO-8859-1'
-require 'common/RegXML'
-
-class FrameXMLFile #  only verified to work for FrameNet v1.1
-
-  def initialize(filename)
-    @filename = filename
-    file = File.new(filename)
-    counter = 0
-    while true
-      counter +=1
-      line = file.gets
-      if line =~ /<lexunit/
-	break
-      end
-      if counter > 3
-	STDERR.puts "Error: File "+filename+" does not conform to FrameNet v1.1 standard (lexunit in 3rd line)"
-	Kernel.exit
-      end
-    end
-    # found lexunit
-    string = line
-    while (line = file.gets())
-      string << line
-    end
-    @lexunit = RegXML.new(string)
-    attributes = @lexunit.attributes()
-    @id = attributes["ID"]
-    attributes["name"] =~ /^([^.]+).([^.]+)$/
-    @lu = $1
-    @pos = $2.upcase
-    if @lu.nil?
-      raise "[framexml] no lemma in header of file #{@filename}"
-    elsif @pos.nil?
-      raise "[framexml] no pos in header of file #{@filename}"
-    end    
-    @frame = attributes["frame"]
-  end
-
-  def get_lu
-    return @lu.gsub(" ","_")
-  end
-
-  def get_lu_id
-    return @id
-  end
-
-  def get_filename
-    return @filename
-  end
-
-  def get_pos
-    return @pos
-  end
-
-  def get_frame
-    return @frame
-  end
-
-  def close
-  end
-
-  def each_sentence
-    @lexunit.children_and_text().each { |subcorpus|
-      subcorpus.children_and_text().each { |annotationSet|
-        if annotationSet.name == "annotationSet"
-          # sentence found
-          yield FrameXMLSentence.new(annotationSet,self)
-        end
-      }
-    }
-  end
-end
-
-class FrameXMLSentence
-  def initialize(annotationSet,file_obj)
-    @file_obj = file_obj
-
-    # layers: hash layer_name -> array:[name, start, stop]
-    #  name: name of the element, string
-    #  start: start character, integer
-    #  stop:  end character, integer
-    @layers = Hash.new
-
-    annotationSet.children_and_text().each { |sentence_or_layer_elt|
-
-      case sentence_or_layer_elt.name 
-      when "sentence"
-        # sentence: has ID, its child is <text>[text]</text>
-        @sent_id = sentence_or_layer_elt.attributes["ID"]
-        text_elt = sentence_or_layer_elt.children_and_text().detect { |child|
-          child.name == "text"
-        }
-        if text_elt
-          # found the text element. its only child should be the text
-          @orig_text = text_elt.children_and_text().detect { |child|
-            child.text?
-          }
-          if @orig_text
-            # take text out of RegXMl object
-            @orig_text = @orig_text.to_s()
-          end
-        end
-
-      when "layers"
-        # contains annotation layers
-        sentence_or_layer_elt.children_and_text().each { |layer|
-          unless layer.name == "layer"
-            # additional material, ignore
-            next
-          end
-          
-          name = layer.attributes["name"]
-          unless name
-            raise "layer without a name"
-          end
-          unless @layers.key?(name)
-            @layers[name] = analyse_layer(layer, name)
-          end
-        }
-      end
-    }
-
-    @pos_text = UtfIso.to_iso_8859_1(@orig_text).split(" ") # text with special characters replaced by iso8859 characters
-    @text = Ampersand.utf8_to_hex(@orig_text).split(" ")  # text with special characters replaced by &...; sequences
-
-    # all text and pos_text have the same number of elements!
-    @start_is = Hash.new # map char indices (start of words) onto word indices
-    @stop_is = Hash.new   # map char indices (end of words) onto word indices
-    @charidx = Array.new # maps word indices on [start,stop]
-    
-    @double_space = Array.new
-    pos = 0
-    while (match = @orig_text.index(/(\s\s+)/,pos))
-    	  @double_space << match
-	  pos = match+1
-    end
-
-
-    # fill start, stop and charidx arrays
-    char_i = 0
-    @pos_text.each_index {|word_i|
-      @start_is[char_i] = word_i
-      startchar = char_i
-      #      puts "Remembering "+char_i.to_s+" as start index of word "+word_i.to_s
-      char_i += our_length(@pos_text[word_i])
-      @stop_is[char_i-1] = word_i
-
-      stopchar = char_i-1
-
-      #      puts "Remembering "+(char_i-1).to_s+" as stop index of word "+word_i.to_s
-      
-      @charidx << [startchar,stopchar]
-      
-      # separators
-      if @double_space.include?(char_i) then
-	char_i += 2
-      else	 
-	char_i += 1
-      end
-    }
-  end
-
-  def get_file_obj
-    return @file_obj
-  end
-
-  def get_sent_id
-    return @sent_id
-  end
-  
-  def print_text
-    puts "("+@id+ ")\t"+@text
-  end
-  
-  def contains_FE_annotation_and_target
-    target_info = @layers["Target"][0]
-    unless target_info[0] == "Target"
-      STDERR.puts "Error in sentence from "+filename+": No target" # strictly speaking, no target at pos 0 in @layers["Target"]
-      STDERR.puts "Sentence: "+@text
-      return false
-    else
-      return (@layers.key?("FE") and target_info[2] != 0)
-    end  
-  end
-
-  # we only verify the interesting layers (FE,GF,Target)
-  # if there is weird stuff going on on e.g. the Noun or Adj layer, we don't care.
-
-  def verify_annotation # returns true if some change has taken place
-    change = false
-    @layers.each_pair {|layername,l|
-      
-      if layername=="FE" or layername=="GF" or layername=="PT" or layername=="Target" # only verify the "important" layers
-	
-	l.each_index {|i|
-	  
-	  element,start,stop = l[i]
-	  
-	  newstart = start
-	  newstop = stop
-	
-	  @charidx.each_index{|j|
-	    unless j== 0
-	      pstartidx, pstopidx = @charidx[j-1]
-	    end
-	    startidx, stopidx = @charidx[j]
-	    
-	    if (start > startidx and start <= stopidx) or 
-		(j != 0 and start > pstopidx and start < startidx)
-	      newstart = startidx
-	    end
-	    
-	    if (stop >= startidx and stop < stopidx) 
-	      newstop = stopidx
-	    elsif (j != 0 and stop > pstopidx and stop < startidx)
-	      newstop = pstopidx
-	    end 
-
-	  }
-	  if start != newstart or stop != newstop
-	    change = true
-	    @layers[layername][i] = [element,newstart,newstop]
-	    STDERR.puts "Heuristics has changed element "+element+" from ["+[start,stop].join(",")+"] to ["+[newstart,newstop].join(",")+"] in file "+@file_obj.get_filename+"."
-	    markable_as_string(layername,element).each {|string|
-	      STDERR.puts "New markable: "+string
-	    }
-	    STDERR.puts "Sentence: "+@pos_text.join(" ")
-	    puts
-	  end
-	}
-      end
-    }
-    return change
-  end
-
-  def print_conll_style
-    print_conll_style_to(STDOUT)
-  end
-
-  # CHANGED KE January 2007:
-  # write new adapted FNTab format
-  # ( "word", ("pt", "gf", "role", "target", "frame", "stuff")* "ne", "sent_id" )
-  def print_conll_style_to(out)
-    
-    # even though in principle there might be multiple 
-    # labels for one span [i.e. in one value of the 
-    # {gf,fe,pt} hashes], we only ever record one
-
-    gf = Hash.new
-    add_all_to_hash(gf,"GF")
-    fe = Hash.new
-    add_all_to_hash(fe,"FE")
-    pt = Hash.new
-    add_all_to_hash(pt,"PT")
-    target = Hash.new
-    add_all_to_hash(target,"Target")
-
-    in_target = false
-
-    @pos_text.each_index {|i|
-      # write format: 
-      #  "word" "pt", "gf", "role", "target", "frame", "stuff" "ne", "sent_id" 
-      line = Array.new
-      # word
-      word = @pos_text[i]
-      line << word
-
-      start, stop = @charidx[i]
-      # "pt", "gf", "role", 
-      [pt,gf,fe].each {|hash|	
-	token = Array.new
-	if hash.key?([start,"start"])
-	  markables = hash.delete([start,"start"])
-	  markables.each {|element|
-	    token << "B-"+element
-	  }
-	end
-	if hash.key?([stop,"stop"])
-	  markables = hash.delete([stop,"stop"])
-	  markables.each {|element|
-	    token << "E-"+element
-	  }
-	end
-	if token.empty?
-	  line << "-"
-	else
-	  line << token.sort.join(":")
-	end
-      }
-      # "target"  
-      if target.key?([start,"start"])
-	target.delete([start,"start"])
-      	in_target = true
-      end
-      if in_target
-	line << @file_obj.get_lu+"."+@file_obj.get_pos
-      else
-	line << "-"
-      end
-      if target.key?([stop,"stop"])
-	target.delete([stop,"stop"])
-	in_target = false
-      end
-      # "frame"
-      line << @file_obj.get_frame
-
-      # "stuff" "ne", 
-      line << "-"
-      line << "-"
-
-      # "sent_id" 
-      line << @file_obj.get_lu_id+"-"+@sent_id
-      
-      out.puts line.join("\t")
-    }
-
-    out.puts
-    
-    [gf,fe,pt,target].each {|hash|
-      unless hash.empty?
-	STDERR.puts @file_obj.get_filename
-	raise "**** Error: Hash not empty after creation of Sentence in CoNLL-Format (could not find matching words for some markup element)!"
-      end
-    }
-  end
-
-
-  def print_layers
-    @layers.each {|ln,l|
-      puts "Layer "+ln+":"
-      l.each {|element,start,stop|
-	puts "\t"+element+": "+start.to_s+" -- "+stop.to_s
-      }
-      puts "***"
-    }
-  end
-
-  
-  private
-
-
-  def our_length(string)   # (1) replace &...; with 1 char and " with two chars
-    return string.gsub(/&(.+?);/,"X").length
-  end
-      
-  def is_fe(fename)
-    @layers["FE"].each {|name,start,stop|
-      if fename == name
-	return true
-      end
-    }
-    return false
-  end
-
-
-  def markable_as_string(layername,markup_name) # returns an array of all markables with this name
-    
-    result = Array.new
-    
-    festart = nil
-    festop = nil
-    @layers[layername].each {|name,start,stop|
-      if markup_name == name
-	fe = Array.new
-	infe = false
-	@charidx.each_index {|i|
-	  startidx,stopidx = @charidx[i]
-	  if startidx == start
-	    infe = true
-	  end
-	  if infe
-	    fe << @pos_text[i]
-	  end
-	  if stopidx == stop
-	    result << (fe.join(" ")+"["+start.to_s+","+stop.to_s+", VERIFIED]")
-	    break
-	  elsif stopidx > stop
-	    result <<  (fe.join(" ")+"["+start.to_s+","+stop.to_s+",ERROR]")
-	    break
-	  end
-	}
-      end
-    }
-    return result
-  end
-
-  def add_to_hash(hash,key,name)
-    exists = false
-    if hash.key?(key)
-      exists = true
-    else
-      hash[key] = Array.new
-      hash[key] << name
-    end
-    return exists
-  end
-
-  def add_all_to_hash(hash,layername)    
-    # use "uniq" to remove wrong double annotations
-    @layers[layername].uniq.each {|element,start,stop|
-      exists = add_to_hash(hash,[start, "start"],element)
-      if exists
-	STDERR.puts "Warning ["+@file_obj.get_filename+"]: In layer "+layername+", two elements start at position "+start.to_s+". Only using first. Layer as read from FrameXML: "+@layers[layername].map {|element,start,stop| element+" ("+start.to_s+","+stop.to_s+")"}.join(" ")
-      else
-	add_to_hash(hash,[stop, "stop"],element)	
-      end
-    }
-  end
-
-
-  def analyse_layer(layer_elt,name) # read layer information from file and store in @layers
-    if name.nil?
-      STDERR.puts "Error: layer line "+line+" with empty name."
-    end
-    
-    # thisLayer, retv: array:[name(string), start(integer), end(integer)]
-    thisLayer = Array.new
-    retv = Array.new
-
-    labels_elt = layer_elt.children_and_text.detect { |child| child.name == "labels"}
-    unless labels_elt
-      # no labels found, return empty array
-      return thisLayer
-    end
-
-    labels_elt.children_and_text.each { |label|
-      unless label.name == "label"
-        # some other markup, ignore
-        next
-      end
-
-      attributes = label.attributes()
-      if attributes["itype"]
-        # null instantiation, don't retain
-        next
-      end
-      if not(attributes["start"]) and not(attributes["end"])
-        # no start and end labels
-        next
-      end
-      thisLayer << [attributes["name"], attributes["start"].to_i, attributes["end"].to_i]
-    }
-
-    # sanity check: verify that 
-    # 1. we don't have overlapping labels
-
-    deleteHash = Hash.new # keep track of the labels which are to be deleted
-                          # i -> Boolean
-
-    thisLayer.each_index {|i|
-      # efficiency: skip already delete labels
-      if deleteHash[i]
-        next
-      end
-      this_label, this_from , this_to = thisLayer[i] 
-      
-      # compare with all remaining labels
-      (i+1..thisLayer.length()-1).to_a.each { |other_i|
-        other_label,other_from,other_to = thisLayer[other_i]        
-
-        # overlap? Throw out the later FE
-        if this_from <= other_from and other_from <= this_to
-          $stderr.puts "Warning: Label overlap, deleting #{other_label}"
-          deleteHash[other_i] = true
-        elsif this_from <= other_to and other_to <= this_to
-          $stderr.puts "Warning: Label overlap, deleting #{this_label}"
-          deleteHash[i] = true
-        end
-      }
-      # matched with all other labels. If "keep", return
-      
-      if deleteHash[i]
-#	$stderr.puts " deleting entry #{i}" 
-      else
-        retv << thisLayer[i]
-      end
-    }
-    
-    return retv
-  end
-end