rbbt-text 0.6.3 → 1.0.0

data/lib/rbbt/ner/oscar3.rb

@@ -8,14 +8,17 @@ require 'rbbt/util/log'
 class OSCAR3 < NER
   Rbbt.claim Rbbt.software.opt.OSCAR3, :install, Rbbt.share.install.software.OSCAR3.find
 
-  @@TextToSciXML   = Rjb::import('uk.ac.cam.ch.wwmm.ptclib.scixml.TextToSciXML')
-  @@ProcessingDocumentFactory   = Rjb::import('uk.ac.cam.ch.wwmm.oscar3.recogniser.document.ProcessingDocumentFactory')
-  @@MEMMSingleton = Rjb::import('uk.ac.cam.ch.wwmm.oscar3.recogniser.memm.MEMMSingleton')
-  @@DFANEFinder = Rjb::import('uk.ac.cam.ch.wwmm.oscar3.recogniser.finder.DFANEFinder')
-  @@MEMM = @@MEMMSingleton.getInstance();
-  @@DFA  = @@DFANEFinder.getInstance();
+  def self.init
+    @@TextToSciXML              ||= Rjb::import('uk.ac.cam.ch.wwmm.ptclib.scixml.TextToSciXML')
+    @@ProcessingDocumentFactory ||= Rjb::import('uk.ac.cam.ch.wwmm.oscar3.recogniser.document.ProcessingDocumentFactory')
+    @@MEMMSingleton             ||= Rjb::import('uk.ac.cam.ch.wwmm.oscar3.recogniser.memm.MEMMSingleton')
+    @@DFANEFinder               ||= Rjb::import('uk.ac.cam.ch.wwmm.oscar3.recogniser.finder.DFANEFinder')
+    @@MEMM                      ||= @@MEMMSingleton.getInstance();
+    @@DFA                       ||= @@DFANEFinder.getInstance();
+  end
 
   def self.match(text,  type = nil, memm =  false)
+    self.init
     doc  = @@ProcessingDocumentFactory.getInstance().makeTokenisedDocument(@@TextToSciXML.textToSciXML(text), true, false, false);
     mentions = []
     it = doc.getTokenSequences().iterator

data/lib/rbbt/ner/oscar4.rb

@@ -8,15 +8,25 @@ require 'rbbt/util/log'
 class OSCAR4 < NER
   Rbbt.claim Rbbt.software.opt.OSCAR4, :install, Rbbt.share.install.software.OSCAR4.find
 
-  Rjb::load(nil, jvmargs = ['-Xms128m','-Xmx2048m'])
-  @@OSCAR = Rjb::import('uk.ac.cam.ch.wwmm.oscar.Oscar')
+  def self.init
+    Rjb::load(nil, jvmargs = ['-Xms1G','-Xmx2G']) unless Rjb.loaded?
 
-  def self.match(text,  type = nil, memm =  false)
+    @@OSCAR      ||= Rjb::import('uk.ac.cam.ch.wwmm.oscar.Oscar')
+    @@FormatType ||= Rjb::import('uk.ac.cam.ch.wwmm.oscar.chemnamedict.entities.FormatType')
+  end
+
+  def self.tagger
+    @@tagger ||= @@OSCAR.new()
+  end
+
+  def self.match(text,  type = nil)
+    self.init
 
     return [] if text.nil? or text.strip.empty?
 
-    oscar = @@OSCAR.new();
-    entities = oscar.findAndResolveNamedEntities(text);
+    oscar = tagger
+    #entities = oscar.findAndResolveNamedEntities(text);
+    entities = oscar.findNamedEntities(text);
     it = entities.iterator
 
     result = []
@@ -24,9 +34,13 @@ class OSCAR4 < NER
     while it.hasNext
       entity = it.next
       mention = entity.getSurface
-      result << mention
+      #inchi = entity.getFirstChemicalStructure(@@FormatType.INCHI)
+      #inchi = inchi.getValue() unless inchi.nil?
+      inchi = nil
+
+      NamedEntity.setup mention, entity.getStart, entity.getType, inchi, entity.getConfidence
 
-      NamedEntity.setup mention, entity.getStart, entity.getType, nil, entity.getNamedEntity.getConfidence
+      result << mention
     end
 
     result

data/lib/rbbt/ner/rnorm.rb

@@ -13,30 +13,28 @@ class Normalizer
   # score above the minimum. Otherwise it return an empty list.
   def self.get_best(values, min)
     return [] if values.empty?
-    best = values.collect{|p| p[1]}.max
+    best = values.collect{|p| p[1] }.max
     return [] if best < min
     values.select{|p| p[1] == best}
   end
 
   # Compares the tokens and gives each candidate a score based on the
   # commonalities and differences amongst the tokens.
-  def token_score(candidates, mention)
-    candidates.collect{|code|
-      next if @synonyms[code].nil?
-      value = @synonyms[code].select{|name| name =~ /\w/}.collect{|name|
-        case 
-        when mention == name
-          100
-        when mention.downcase == name.downcase
-          90
-        when mention.downcase.gsub(/\s/,'') == name.downcase.gsub(/\s/,'')
-          80
-        else
-          @tokens.evaluate(mention, name)
-        end
-      }.max
-      [code, value]
-    }.compact
+  def token_score(code, mention)
+    return nil if @synonyms[code].nil?
+    @synonyms[code].select{|name| name =~ /[a-zA-Z]/ }.collect{|name|
+      value = case 
+              when mention == name
+                100
+              when mention.downcase == name.downcase
+                90
+              when mention.downcase.gsub(/\s/,'') == name.downcase.gsub(/\s/,'')
+                80
+              else
+                @tokens.evaluate(mention, name)
+              end
+      [value, name]
+    }.sort_by{|value, name| value }.last
   end
 
   # Order candidates with the number of words in common between the text
@@ -46,7 +44,7 @@ class Normalizer
   # been a Proc or a Hash.
   def entrez_score(candidates, text, to_entrez = nil)
       code2entrez = {}
-      candidates.each{|code|
+      candidates.each{|code, score|
         if to_entrez.is_a? Proc
           entrez = to_entrez.call(code)
         elsif to_entrez.is_a? Hash
@@ -72,24 +70,37 @@ class Normalizer
   # positions. This is based on the idea that synonym lists order their
   # synonyms by importance.
   def appearence_order(candidates, mention)
-    positions = candidates.collect{|code|
+    positions = candidates.collect{|code, score, name|
       next unless @synonyms[code]
       pos = nil
       @synonyms[code].each_with_index{|list,i|
         next if pos
         pos = i if list.include? mention
       }
-      pos
+      pos 
     }
+
     return nil if positions.compact.empty?
-    best = candidates.zip(positions).sort{|a,b| a[1] <=> b[1]}.first[1]
-    candidates.zip(positions).select{|p| p[1] == best}.collect{|p| p[0]}
+
+    best = candidates.zip(positions).sort{|a,b| 
+      case
+      when (a[1].nil? and b[1].nil?)
+        0
+      when b[1].nil?
+        1
+      when a[1].nil?
+        -1
+      else
+        a[1] <=> b[1]
+      end
+    }.first[1]
+    candidates.zip(positions).select{|p| p[1] == best}
   end
 
 
 
   def initialize(lexicon, options = {})
-    @synonyms = TSV.open(lexicon, :flat)
+    @synonyms = TSV === lexicon ? lexicon : TSV.open(lexicon, :type => :flat, :unnamed => true)
 
     @index = CueIndex.new
     @index.load(lexicon, options[:max_candidates])
@@ -103,30 +114,43 @@ class Normalizer
   end
 
   def select(candidates, mention, text = nil, options = {})
-    threshold  = options[:threshold] || 0
-    max_candidates  = options[:max_candidates] || 200
-    max_entrez  = options[:max_entrez] || 10
+    options = Misc.add_defaults options, :threshold => 0, :max_candidates => 20, :max_entrez => 10, :keep_matches => false
+    threshold, max_candidates, max_entrez, keep_matches = Misc.process_options options, :threshold, :max_candidates, :max_entrez, :keep_matches
 
     # Abort if too ambigous
     return [] if candidates.empty?
     return [] if candidates.length > max_candidates
 
-    scores = token_score(candidates, mention)
-    best_codes = Normalizer::get_best(scores, threshold).collect{|p| p[0]}
+    scores = candidates.zip(candidates.collect{|candidate| token_score(candidate, mention)}).collect{|v| v.flatten}
+    scores.delete_if{|candidate, score, name| score.nil? or score <= threshold}
+
+    best_codes = Normalizer::get_best(scores, threshold)
 
     # Abort if too ambigous
     return [] if best_codes.length > max_entrez
 
     if best_codes.length > 1 and text
-      scores = entrez_score(best_codes, text, @to_entrez)
+      scores = entrez_score(best_codes.collect{|c| c.first}, text, @to_entrez)
 
-      Normalizer::get_best(scores, 0).collect{|p| p[0]}
+      if keep_matches
+        Normalizer::get_best(scores, 0)
+      else
+        Normalizer::get_best(scores, 0).collect{|p| p[0]}
+      end
     else
       orders = appearence_order(best_codes, mention)
       if orders 
-        orders
+        if keep_matches
+          orders.collect{|p| p[0]}
+        else
+          orders.collect{|p| p[0][0]}
+        end
       else
-        best_codes
+        if keep_matches
+          best_codes
+        else
+          best_codes.collect{|p| p[0]}
+        end
       end
     end
 

data/lib/rbbt/ner/rnorm/cue_index.rb

@@ -17,6 +17,7 @@ class CueIndex
 
     file ||= Rbbt.share.rnorm.cue_default.produce if !file && !block
 
+    file = file.find if file.respond_to? :find
     load_config(:define, file, &block)
   end
 
@@ -47,12 +48,12 @@ class CueIndex
   
   def load(file, max_candidates = 50)
     @indexes = Array.new(@rules.size){Hash.new}
-    data = TSV.open(file, :flat)
+    data = TSV === file ? file : TSV.open(file, :type => :flat, :unnamed => true)
     data.each{|code, values|
       values.each{|value|
         cues(value).each_with_index{|cue_list,i|
           cue_list.each{|cue|
-            @indexes[i][cue] ||= []
+            @indexes[i][cue] ||= Set.new
             @indexes[i][cue]  << code unless @indexes[i][cue].include? code
           }
         }
@@ -69,7 +70,7 @@ class CueIndex
     @indexes.each_with_index{|index,i|
       best = []
       cues[i].each{|cue|
-        best << index[cue] if index[cue]
+        best << index[cue].to_a if index[cue]
       }
       return best.flatten if best.any?
     }

data/lib/rbbt/ner/rnorm/tokens.rb

@@ -33,7 +33,8 @@ class Tokenizer
 
     def method_missing(name, *args, &bloc)
       @token = name.to_sym
-      @value = *args.first
+      value = args.first
+      @value = value
       self
     end
 
@@ -123,8 +124,12 @@ class Tokenizer
 
   #{{{ Metaprogramming hooks
   def define_tokens(name, *args, &block)
-    action = *args[0] || block ||  /#{name.to_s}s?/i
-      raise "Wrong format" unless (action.is_a?(Proc) || action.is_a?(Regexp))
+    action = args[0] || block ||  /#{name.to_s}s?/i
+
+    #HACK: Misterious error where *args[0] returns an array [/regexp/i] for
+    #example
+    #action = action.first if Array === action
+    raise "Wrong format" unless (action.is_a?(Proc) || action.is_a?(Regexp))
 
     @types[name.to_sym] = action
     @order.push name.to_sym
@@ -160,6 +165,7 @@ class Tokenizer
     @transforms = []
 
     file ||= Rbbt.share.rnorm.tokens_default.produce if !file && !block
+    file = file.find if file.respond_to? :find
     load_config :main, file, &block
   end
 
@@ -200,7 +206,7 @@ class Tokenizer
   #{{{ Comparisons
 
   def evaluate_tokens(list1, list2)
-    @operations.inject(0){| acc, o|
+    @operations.inject(0){|acc, o|
       acc + o.eval(list1, list2)
     }
   end

data/lib/rbbt/ner/segment.rb

@@ -9,11 +9,16 @@ module Segment
     @offset = offset.nil? ? nil : offset.to_i
   end
 
+
+  def segment_length
+    self.length
+  end
+
   #{{{ Ranges
 
   def end
     return nil if offset.nil?
-    offset + length - 1
+    offset + segment_length - 1
   end
 
   def range
@@ -41,8 +46,14 @@ module Segment
     self
   end
 
-  def make_relative(segments)
-    segments.collect{|s| s.push offset}
+  def make_relative(segments, &block)
+    if block_given?
+      segments.each{|s| s.push offset}
+      yield(segments)
+      segments.each{|s| s.pull offset}
+    else
+      segments.each{|s| s.push offset}
+    end
   end
 
   def range_in(container = nil)
@@ -72,7 +83,7 @@ module Segment
         when (not a.range.include? b.offset and not b.range.include? a.offset)
           a.offset <=> b.offset
         else
-          a.length <=> b.length
+          a.segment_length <=> b.segment_length
         end
       end
     else
@@ -125,7 +136,7 @@ module Segment
         chunks << chunk
       end
 
-      segment_end = offset + segment.length - 1
+      segment_end = offset + segment.segment_length - 1
 
       if not skip_segments
         chunk = text[offset..segment_end]
@@ -138,7 +149,7 @@ module Segment
 
     end
 
-    if not text.nil? and text.any?
+    if not text.nil? and not text.empty?
       chunk = text.dup
       Segment.setup(chunk, text_offset)
       chunks << chunk
@@ -156,8 +167,8 @@ module Segment
       offset = text.index part
       next if offset.nil?
       Segment.setup(part, pre_offset + offset)
-      pre_offset += offset + part.length - 1
-      text = text[(offset + part.length - 1)..-1]
+      pre_offset += offset + part.segment_length - 1
+      text = text[(offset + part.segment_length - 1)..-1]
     end
   end
 

data/lib/rbbt/ner/segment/docid.rb

@@ -0,0 +1,46 @@
+require 'rbbt/ner/segment'
+
+module SegmentWithDocid 
+  extend Annotation
+
+  self.annotation :docid
+
+  def masked?
+    self[0..5] == "MASKED"
+  end
+
+  def mask
+    return self if masked?
+    raise "Cannot mask an array of elements, they must be masked individually" if Array === self
+    raise "Cannot mask a segment with no docid" if not self.respond_to? :docid or docid.nil?
+    raise "Cannot mask a segment with no offset" if offset.nil?
+    textual_position = ["MASKED", length] * ":"
+    self.replace(textual_position)
+    self
+  end
+
+  def unmasked_text
+    return self unless masked?
+    tag, length = self.split(":")
+    Document.setup(docid).text[offset..(offset+length.to_i-1)]
+  end
+
+  def unmask
+    return self unless masked?
+    self.replace(unmasked_text)
+    self
+  end
+
+  def str_length
+    self.length
+  end
+
+  def masked_length
+    self.split(":").last.to_i
+  end
+
+  def segment_length
+    masked? ? masked_length : str_length
+  end
+end
+

data/lib/rbbt/ner/segment/named_entity.rb

@@ -5,7 +5,7 @@ module NamedEntity
   extend Entity
   include Segment
 
-  self.annotation :type, :code, :score, :docid
+  self.annotation :type, :code, :score
 
   def report
     <<-EOF

data/lib/rbbt/ner/segment/transformed.rb

@@ -11,7 +11,7 @@ module Transformed
     text
   end
  
-  def self.with_transform(text, segments, replacement)
+  def self.with_transform(text, segments, replacement = nil)
 
     text.extend Transformed
     text.replace(segments, replacement)
@@ -119,7 +119,7 @@ module Transformed
         new = replacement.call(segment)
       end
 
-      diff = new.length - segment.length
+      diff = new.length - segment.segment_length
 
       self[updated_begin..updated_end] = new
 
@@ -146,7 +146,9 @@ module Transformed
     end
   end
 
-  def restore(segments, first_only = false)
+  # Restore the sentence from all transformation. Segments that are passed as
+  # parameters are restored from transformed space to original space
+  def restore(segments = [], first_only = false)
     return segments if @transformation_stack.empty?
 
     if first_only