rbbt-text 0.2.1 → 0.5.0

data/lib/rbbt/ner/token_trieNER.rb

@@ -1,105 +1,214 @@
 require 'rbbt-util'
 require 'rbbt/util/tsv'
 require 'rbbt/ner/annotations'
+require 'rbbt/ner/annotations/token'
 require 'rbbt/ner/NER'
 
 class TokenTrieNER < NER
   def self.clean(token)
     if token.length > 3
-      token.downcase
+      token.downcase.sub(/-/,'')
     else
       token
     end
   end
 
-  def self.prepare_token(token, start)
-    Token.annotate(clean(token), start, token)
+  def self.prepare_token(token, start, extend_to_token = true, no_clean = false)
+    if no_clean
+      if extend_to_token
+        Token.annotate(clean(token), start, token)
+      else
+        clean(token)
+      end
+    else
+      if extend_to_token
+        Token.annotate(clean(token), start, token)
+      else
+        token
+      end
+    end
   end
 
-  def self.tokenize(text, split_at = /\s|(\(|\)|[-."':,])/, start = 0)
+  def self.tokenize(text, extend_to_token = true, split_at = nil, no_clean = false, start = 0)
+    split_at = /\s|(\(|\)|[-."':,])/ if split_at.nil?
 
     tokens = []
     while matchdata = text.match(split_at)
-      tokens << prepare_token(matchdata.pre_match, start) unless matchdata.pre_match.empty?
-      tokens << prepare_token(matchdata.captures.first, start + matchdata.begin(1)) if matchdata.captures.any? and not matchdata.captures.first.empty?
+      tokens << prepare_token(matchdata.pre_match, start, extend_to_token, no_clean) unless matchdata.pre_match.empty?
+      tokens << prepare_token(matchdata.captures.first, start + matchdata.begin(1), extend_to_token, no_clean) if matchdata.captures.any? and not matchdata.captures.first.empty?
       start += matchdata.end(0)
       text = matchdata.post_match
     end
-    tokens << prepare_token(text, start) unless text.empty?
+     
+    tokens << prepare_token(text, start, extend_to_token) unless text.empty?
 
     tokens
   end
 
   #{{{ Process dictionary
 
+  module EnumeratedArray
+    attr_accessor :pos
+    def self.extended(array)
+      array.pos = 0
+    end
+
+    def last?
+      @pos == length - 1
+    end
+
+    def advance
+      @pos += 1
+    end
+
+    def back
+      @pos -= 1
+    end
+
+    def next
+      e = self[@pos]
+      advance
+      e 
+    end
+
+    def peek
+      self[@pos]
+    end
+
+    def left?
+      @pos < length
+    end
+
+  end
+
+
   class Code
-    attr_accessor :value, :type
-    def initialize(value, type = nil)
-      @value = value
+    attr_accessor :code, :type
+    def initialize(code, type = nil)
+      @code = code
       @type = type
     end
 
     def to_s
-      [type, value] * ":"
+      [type, code] * ":"
     end
   end
 
-  def self.index_for_tokens(tokens, code, type = nil)
-    if tokens.empty?
-      {:END => [Code.new code, type]}
+  def self.index_for_tokens(tokens, code, type = nil, slack = nil)
+    if not tokens.left?
+      {:END => [Code.new(code, type)]}
     else
-      {tokens.shift => index_for_tokens(tokens, code, type)}
+      head = tokens.next
+      if (slack.nil? or not slack.call(head))
+        res = {head => index_for_tokens(tokens, code, type, slack)}
+      else
+        res = {head => index_for_tokens(tokens, code, type, slack)}.merge(index_for_tokens(tokens, code, type, slack))
+      end
+      tokens.back
+      res
     end
   end
-  
+
   def self.merge(index1, index2)
+    index1.write if index1.respond_to? :write
     index2.each do |key, new_index2|
       case
       when key == :END
-        index1[:END] ||= []
-        index1[:END] += new_index2.reject{|new| index1[:END].collect{|e| e.to_s }.include? new.to_s }
-        index1[:END].uniq!
+        end1 = index1[:END] || []
+        end1 += new_index2.reject{|new| end1.collect{|e| e.to_s }.include? new.to_s }
+        end1.uniq!
+        index1[:END] = end1
       when index1.include?(key)
-        merge(index1[key], new_index2)
+        index1[key] = merge(index1[key], new_index2)
       else
         index1[key] = new_index2
       end
     end
+    index1.read if index1.respond_to? :read
+
+    index1
   end
 
-  def self.process(hash, type = nil)
-    index = {}
-    hash.each do |code, names|
-      names.flatten.each do |name|
-        next if name.empty? or name.length < 2
-        tokens = tokenize name
+  def self.process(index, hash, type = nil, slack = nil, split_at = nil, no_clean = false)
 
-        merge(index, index_for_tokens(tokens, code, type)) unless tokens.empty?
+    chunk_size = hash.size / 100
+    items_in_chunk = 0
+    tmp_index = {}
+    hash.send(hash.respond_to?(:through)? :through : :each) do |code, names|
+      names = Array === names ? names : [names]
+      names.flatten! if Array === names.first and not Token === names.first.first
+      names.each do |name|
+        next if name.empty? or (String === name and name.length < 2)
+
+        tokens = Array === name ? name : tokenize(name, false, split_at, no_clean) 
+        tokens.extend EnumeratedArray
+
+        tmp_index = merge(tmp_index, index_for_tokens(tokens, code, type, slack)) unless tokens.empty?
+        items_in_chunk += 1
+
+        if items_in_chunk > chunk_size
+          index = merge(index, tmp_index)
+          tmp_index = {}
+          items_in_chunk = 0
+        end
       end
     end
+    index = merge(index, tmp_index)
+
     index
   end
 
   #{{{ Matching
   
-  def self.find(index, tokens, longest_match = true)
-    return nil unless index.include? tokens.first
+  def self.follow(index, head)
+    res = nil
+
+    if index.include? head
+      return index[head]
+    end
+
+    return nil unless (not TCHash === index ) and index.include? :PROCS
 
-    head = tokens.shift
-    next_index = index[head]
+    index[:PROCS].each do |key,value|
+      return value if key.call(head)
+    end
 
-    if tokens.empty?
+    nil
+  end
+  
+  def self.find_fail(index, tokens, head, longest_match, slack, first)
+    if Proc === slack and not first and not head.nil? and tokens.left? and slack.call(head) 
+      matches = find(index, tokens, longest_match, slack, false) # Recursion
+      if not matches.nil?
+        matches.last.unshift head
+        return matches
+      end
+    end
+
+    tokens.back
+    return nil
+  end
+ 
+  def self.find(index, tokens, longest_match = true, slack = nil, first = true)
+    head = tokens.next
+    
+    next_index = follow(index, head)
+
+
+    return find_fail(index, tokens, head, longest_match, slack, first) if next_index.nil?
+
+    if not tokens.left?
       if next_index.include? :END
         return [next_index[:END], [head]]
       else
-        tokens.unshift head
-        return nil
+        return find_fail(index, tokens, head, longest_match, slack, first)
       end
     else
 
       return [next_index[:END], [head]] if next_index.include?(:END) and not longest_match
 
-      matches = find(next_index, tokens)
+      matches = find(next_index, tokens, longest_match, slack, false) # Recursion
+
       if not matches.nil?
         matches.last.unshift head
         return matches
@@ -107,8 +216,7 @@ class TokenTrieNER < NER
       
       return [next_index[:END], [head]] if next_index.include?(:END)
 
-      tokens.unshift head
-      return nil
+      return find_fail(index, tokens, head, longest_match, slack, first)
     end
   end
 
@@ -117,20 +225,35 @@ class TokenTrieNER < NER
     match_offset = match_tokens.first.offset
     match_tokens.each{|t| 
       match << " " * (t.offset - (match_offset + match.length)) if t.offset > (match_offset + match.length)
-      match << t.original
+      match << (t.respond_to?(:original) ? t.original : t)
     }
 
     NamedEntity.annotate(match, match_tokens.first.offset, type, codes)
   end
 
-  attr_accessor :index, :longest_match, :type
-  def initialize(file, type = nil, options = {})
-    options = Misc.add_defaults options, :flatten => true, :longest_match => true
+  attr_accessor :index, :longest_match, :type, :slack, :split_at, :no_clean
+  def initialize(type = nil, file = nil, options = {})
+    options = Misc.add_defaults options, :longest_match => true, :no_clean => false, :slack => nil, :split_at => nil,
+      :persistence => false
+    @slack = slack
     @longest_match = options.delete :longest_match
+    @split_at = options.delete :split_at
+    @no_clean = options.delete :no_clean
 
+    file = [] if file.nil?
     file = [file] unless Array === file
-    @index = {}
-    file.each do |f| TokenTrieNER.merge(@index, TokenTrieNER.process(TSV.new(f, options), type)) end
+    @index = Persistence.persist(file, :TokenTRIE, :tsv, options) do |file, options, filename, persistecen_file|
+      if persistecen_file.nil?
+        @index = {}
+      else
+        FileUtils.mkdir_p File.dirname(persistecen_file) unless File.exists? File.dirname(persistecen_file)
+        @index = TCHash.get persistecen_file, true, :marshal
+      end
+      file.each do |f| 
+        merge(f, type)
+      end
+      @index
+    end
   end
 
   def merge(new, type = nil)
@@ -140,24 +263,36 @@ class TokenTrieNER < NER
     when Hash === new
       TokenTrieNER.merge(@index, new)
     when TSV === new
-      TokenTrieNER.merge(@index, TokenTrieNER.process(new,type))
+      old_unnamed = new.unnamed
+      old_monitor = new.monitor
+      new.unnamed = true
+      new.monitor = {:step => 1000, :desc => "Processing TSV into TokenTrieNER"}
+      TokenTrieNER.process(@index, new, type, slack, split_at, no_clean)
+      new.unnamed = old_unnamed
+      new.monitor = old_monitor
     when String === new
-      TokenTrieNER.merge(@index, TokenTrieNER.process(TSV.new(new, :flatten => true), type))
+      new = TSV.new(new, :flat)
+      new.unnamed = true
+      new.monitor = {:step => 1000, :desc => "Processing TSV into TokenTrieNER"}
+      TokenTrieNER.process(@index, new, type, slack, split_at, no_clean)
     end
   end
 
   def match(text)
-    tokens = TokenTrieNER.tokenize text
+    tokens = Array === text ? text : TokenTrieNER.tokenize(text, true, split_at, no_clean)
+
+    tokens.extend EnumeratedArray
+    tokens.pos = 0
 
     matches = []
-    while tokens.any?
-      new_matches = TokenTrieNER.find(@index, tokens, longest_match) 
+    while tokens.left?
+      new_matches = TokenTrieNER.find(@index, tokens, longest_match, slack) 
 
       if new_matches
         codes, match_tokens = new_matches
-        matches << TokenTrieNER.make_match(match_tokens, codes.collect{|c| c.type}, codes.collect{|c| c.value})
+        matches << TokenTrieNER.make_match(match_tokens, codes.collect{|c| c.type}, codes.collect{|c| c.code})
       else
-        tokens.shift
+        tokens.advance
       end
     end
 
@@ -165,4 +300,3 @@ class TokenTrieNER < NER
   end
 
 end
-

data/lib/rbbt/nlp/genia/sentence_splitter.rb

@@ -0,0 +1,214 @@
+module NLP
+  def self.returnFeatures(prevWord, delimiter, nextWord)
+    if nextWord.match(/__ss__/)
+      nw = nextWord.sub(/__ss__/, "")
+    else
+      nw = nextWord
+    end
+
+    str = ""
+    # prev. word, next word
+    str += "pw_" + prevWord.downcase
+    str += "\tnw_" + nw.downcase
+
+    # delimiter
+    str += "\td_" + delimiter
+
+    # capitalized first char in next word
+    # capital in next word excluding first char.
+    if nw[0].chr == nw[0].chr.capitalize
+      str += "\tnfc_y"
+      nwExcluginFirst = nw[1 ... -1]
+      if nwExcluginFirst == nil
+        str += "\tnwcef_n"
+      elsif nwExcluginFirst.downcase == nwExcluginFirst
+        str += "\tnwcef_n"
+      else
+        str += "\tnwcef_y"
+      end
+    else
+      if nw.downcase == nw
+        str += "\tnwcef_n"
+      else
+        str += "\tnwcef_y"
+      end
+      str += "\tnfc_n"
+    end
+
+    # prev. word capital
+    if prevWord.downcase == prevWord
+      str += "\tpwc_n"
+    else
+      str += "\tpwc_y"
+    end
+
+    # number in prev. word, in next word
+    if prevWord.match(/[0-9]/)
+      str += "\tpwn_y"
+    else
+      str += "\tpwn_n"
+    end
+    if nw.match(/[0-9]/)
+      str += "\tnwn_y"
+    else
+      str += "\tnwn_n"
+    end
+
+    # prev., next word excluding braket, camma, etc.
+    prevWordEx = prevWord.gsub(/[()'",\[\]]/, "")
+    nwEx = nw.gsub(/[()'",\[\]]/, "")
+    str += "\tpwex_" + prevWordEx.downcase
+    str += "\tnwex_" + nwEx.downcase
+
+    # bracket or quatation in prev. word
+    if prevWord.match(/()'"/)
+      str += "\tpwcbq_y"
+    else
+      str += "\tpwcbq_n"
+    end
+    # camma in prev., next word
+    if prevWord.match(/,/)
+      str += "\tpwcc_y"
+    else
+      str += "\tpwcc_n"
+    end
+    if nw.match(/,/)
+    else
+      str += "\tnwcc_n"
+    end
+
+    # prev. word + delimiter
+    str += "\tpw_" + prevWord + "_d_" + delimiter
+    # prev. word ex. +  delimiter + next word ex.
+    str += "\tpwex_" + prevWordEx + "_d_" + delimiter + "_nwex_" + nwEx
+    #str +=
+    #str +=
+    #str +=
+    str += "\n"
+  end
+
+  def self.event_extraction(text)
+    events = ""
+    marks = ""
+
+    eventCount = 0
+
+    pat = / [^ ]+[.!\?\)\]\"]( +)[^ ]+ /
+    for line in text.split(/\n/) do
+      while line.match(pat) do
+        line.sub!(/ ([^ ]+)([.!\?\)\]\"])( +)([^ ]+) /){
+          a, b, d, c = $1, $2, $3, $4
+          events << eventCount.to_s  << "\t"
+          events << returnFeatures(a, b, c)
+          (" " + a + b + "__" + eventCount.to_s + "____" + d + "__" + c + " ")
+        }
+        eventCount += 1
+      end
+      marks << line
+    end
+
+    [events, marks]
+  end
+
+  def self.process_labels(marked_text, labels)
+    out = ""
+
+    count = 0
+    text_lines = marked_text.split(/\n/)
+    line = text_lines.shift
+    for label in labels
+      pat = "__" + count.to_s + "__"
+      until(line.match(pat)) do
+        out << line
+        line = text_lines.shift
+      end
+      splitted = label.chomp.to_i
+
+      line.sub!(pat){
+        if splitted == 1
+          "__\n__"
+        else
+          "____"
+        end
+      }
+      line.sub!(/__\n____ +__/, "\n")
+      line.sub!(/______( +)__/){
+        $1
+      }
+      count += 1
+    end
+
+    out << line
+
+    out << text_lines * ""
+
+    out
+  end
+
+  def self.geniass_sentence_splitter_extension(text)
+    require Rbbt.software.opt.Geniass.ruby["Geniass.so"].find
+    geniass = Geniass.new
+    if not geniass.geniass_is_loaded
+      Misc.in_dir Rbbt.software.opt.Geniass.find do
+        geniass.load_geniass
+      end
+    end
+
+    cleaned = text.gsub("\n",NEW_LINE_MASK)
+    events, marks = event_extraction(cleaned)
+
+    labels = events.split(/\n/).collect{|line| 
+      geniass.label(line)
+    }
+
+    out = process_labels(marks, labels)
+
+    offsets = []
+
+    inTxtStrict = StringIO.new text
+    inTxtNew = StringIO.new out.gsub("\n", '|').gsub(NEW_LINE_MASK, "\n")
+
+    marker = "|"[0]
+    position = 0
+    sentenceCount = 1
+    target = ''
+    targetNew = ''
+    start = 0
+    finish = 0
+
+    while(!inTxtNew.eof?) do
+      targetNew = inTxtNew.getc
+      target = inTxtStrict.getc
+      position += 1
+      if targetNew == marker
+        sentenceCount += 1
+        finish = position - 1
+        offsets << [start, finish] if finish - start > 10
+        if targetNew == target
+          start = position
+        else
+          targetNew = inTxtNew.getc
+          while targetNew != target do
+            target = inTxtStrict.getc
+            position += 1
+          end
+          start = position - 1
+        end
+      end
+    end
+
+    finish = position - 1
+    offsets << [start, finish] if finish > start
+
+    inTxtStrict.close
+    inTxtNew.close
+
+    offsets.collect do |s,e|
+      sentence = text[s..e]
+      next if sentence.nil?
+      Segment.annotate sentence, s
+      sentence
+    end
+
+  end
+end