rbbt-text 1.1.8 → 1.1.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ea1646b5f32644bb5872f57422534b49955f988df26df4a65c8dda592515eac3
-  data.tar.gz: 3f6bc60546b79c76b6b35840712453616c377fcc088f321e95847f116776bef1
+  metadata.gz: 77391b4691e4ea2a6e5da918bc40820bae8175ff1d82f9c96a1685986605dfd7
+  data.tar.gz: a83dd9236502d1787f1040fb4c60a6160086515713282283e434b589c1425743
 SHA512:
-  metadata.gz: 9376c68bad67733b5771b57ead7c962d45ff29c44362d1c51bf3480d3c3bf9f1f75284e40044fc4ed95bd94a03ab0759b3b7320bf1e3da00a0cdd82255c9395c
-  data.tar.gz: cd25a9cd91fde366be195801d45238d555edfc94f2b06391db7db2d9f4781b34dd599514385782d6c7e22af2841c5f3322ba74bf0a3a9c1fdbe308a255f00098
+  metadata.gz: f69d7eb10741d2b3c7735e8e29f29625567775647d16d0261b42cce108d2f8309a2e938dad3360842a964a9c5d4fd5a2197c72618ab40971f7a65306e9c6936a
+  data.tar.gz: dec802a15cfc7c8c9a90ee8ec0c83af88c881ee16e071776a995554aa0661603bdd6cb7bf30162c43beccf1a423a2e8d26afc15f92544ccc08284a87a038a1b2

data/lib/rbbt/ner/NER.rb

@@ -1,6 +1,6 @@
-require 'rbbt/ner/segment'
-require 'rbbt/ner/segment/named_entity'
-require 'rbbt/ner/segment/segmented'
+require 'rbbt/text/segment'
+require 'rbbt/text/segment/named_entity'
+require 'rbbt/text/segment/segmented'
 
 class NER
   def entities(text, protect = false, *args)

data/lib/rbbt/ner/abner.rb

@@ -1,7 +1,7 @@
 require 'rbbt'
 require 'rjb'
-require 'rbbt/ner/segment'
 require 'rbbt/resource'
+require 'rbbt/text/segment'
 require 'rbbt/ner/NER'
 
 # Offers a Ruby interface to the Abner Named Entity Recognition Package
@@ -31,10 +31,10 @@ class Abner < NER
   # Given a chunk of text, it finds all the mentions appearing in it. It
   # returns all the mentions found, regardless of type, to be coherent
   # with the rest of NER packages in Rbbt.
-  def match(text)
+  def match(text, fix_encode = true)
     return [] if text.nil? or text.empty?
 
-    text = text.encode('utf-8', 'binary', :invalid => :replace, :undef => :replace, :replace => '')
+    text = text.encode('utf-8', 'binary', :invalid => :replace, :undef => :replace, :replace => '') if fix_encode
     res = @tagger.getEntities(text)
     types = res[1]
     strings = res[0]

data/lib/rbbt/ner/banner.rb

@@ -1,7 +1,7 @@
 require 'rbbt'
 require 'rjb'
-require 'rbbt/ner/segment'
 require 'rbbt/ner/NER'
+require 'rbbt/text/segment'
 
 # Offers a Ruby interface to the Banner Named Entity Recognition Package
 # in Java. Banner[http://banner.sourceforge.net/].

data/lib/rbbt/ner/brat.rb

@@ -1,5 +1,5 @@
-require 'rbbt/ner/segment/named_entity'
-require 'rbbt/ner/segment/relationship'
+require 'rbbt/text/segment/named_entity'
+require 'rbbt/text/segment/relationship'
 module Brat
   Rbbt.claim Rbbt.software.opt.Brat, :install, "https://github.com/nlplab/brat.git"
 

data/lib/rbbt/ner/chemical_tagger.rb

@@ -1,6 +1,6 @@
 require 'rbbt'
 require 'rjb'
-require 'rbbt/ner/segment'
+require 'rbbt/text/segment'
 require 'rbbt/ner/NER'
 require 'rbbt/util/log'
 

data/lib/rbbt/ner/linnaeus.rb

@@ -1,6 +1,6 @@
 require 'rjb'
 require 'rbbt'
-require 'rbbt/ner/segment/named_entity'
+require 'rbbt/text/segment/named_entity'
 
 module Linnaeus
 

data/lib/rbbt/ner/ngram_prefix_dictionary.rb

@@ -1,8 +1,8 @@
 require 'rbbt'
 require 'rbbt/util/misc'
 require 'rbbt/tsv'
-require 'rbbt/ner/segment'
-require 'rbbt/ner/segment/token'
+require 'rbbt/text/segment'
+require 'rbbt/text/segment/token'
 require 'rbbt/ner/NER'
 require 'inline'
 

data/lib/rbbt/ner/oscar3.rb

@@ -1,7 +1,7 @@
 require 'rbbt'
 require 'rjb'
 require 'libxml'
-require 'rbbt/ner/segment'
+require 'rbbt/text/segment'
 require 'rbbt/ner/NER'
 require 'rbbt/util/log'
 

data/lib/rbbt/ner/oscar4.rb

@@ -1,7 +1,7 @@
 require 'rbbt'
 require 'rjb'
 require 'libxml'
-require 'rbbt/ner/segment'
+require 'rbbt/text/segment'
 require 'rbbt/ner/NER'
 require 'rbbt/util/log'
 

data/lib/rbbt/ner/patterns.rb

@@ -1,7 +1,7 @@
-require 'rbbt/ner/segment/named_entity'
-require 'rbbt/ner/segment/segmented'
-require 'rbbt/ner/segment/transformed'
-require 'rbbt/ner/segment/relationship'
+require 'rbbt/text/segment/named_entity'
+require 'rbbt/text/segment/segmented'
+require 'rbbt/text/segment/transformed'
+require 'rbbt/text/segment/relationship'
 require 'rbbt/ner/regexpNER'
 require 'rbbt/ner/token_trieNER'
 require 'rbbt/nlp/nlp'

data/lib/rbbt/ner/regexpNER.rb

@@ -1,4 +1,4 @@
-require 'rbbt/ner/segment'
+require 'rbbt/text/segment'
 require 'rbbt/ner/NER'
 require 'rbbt/util/simpleDSL'
 

data/lib/rbbt/ner/token_trieNER.rb

@@ -1,7 +1,7 @@
 require 'rbbt'
 require 'rbbt/tsv'
-require 'rbbt/ner/segment'
-require 'rbbt/ner/segment/token'
+require 'rbbt/text/segment'
+require 'rbbt/text/segment/token'
 require 'rbbt/ner/NER'
 
 class TokenTrieNER < NER

data/lib/rbbt/nlp/genia/sentence_splitter.rb

@@ -1,5 +1,5 @@
 require 'rbbt/nlp/nlp'
-require 'rbbt/ner/segment'
+require 'rbbt/text/segment'
 module NLP
   Rbbt.claim Rbbt.software.opt.Geniass, :install, Rbbt.share.install.software.Geniass.find
 

data/lib/rbbt/nlp/nlp.rb

@@ -2,8 +2,8 @@ require 'rbbt'
 require 'rbbt/util/tmpfile'
 require 'rbbt/persist'
 require 'rbbt/resource'
-require 'rbbt/ner/segment'
-require 'rbbt/ner/segment/segmented'
+require 'rbbt/text/segment'
+require 'rbbt/text/segment/segmented'
 require 'rbbt/nlp/genia/sentence_splitter'
 require 'digest/md5'
 

data/lib/rbbt/nlp/open_nlp/sentence_splitter.rb

@@ -1,6 +1,6 @@
 require 'rbbt'
 require 'rjb'
-require 'rbbt/ner/segment'
+require 'rbbt/text/segment'
 require 'rbbt/resource'
 
 module OpenNLP

data/lib/rbbt/{corpus → text}/corpus.rb

@@ -1,8 +1,17 @@
-require 'rbbt/corpus/document'
-require 'rbbt/corpus/document_repo'
+require 'rbbt/text/corpus/document'
+require 'rbbt/text/corpus/document_repo'
 
 class Corpus
+  class << self
+    attr_accessor :claims
+    def claim(namespace, &block)
+      @@claims = {}
+      @@claims[namespace] = block
+    end
+
+  end
   attr_accessor :corpora_path, :document_repo, :persistence_dir, :global_annotations
+
   def initialize(corpora_path = nil)
     @corpora_path = case
                    when corpora_path.nil?
@@ -24,6 +33,7 @@ class Corpus
  
     Misc.lock(@corpora_path.document_repo) do
       @document_repo   = DocumentRepo.open_tokyocabinet @corpora_path.document_repo, false
+      @document_repo.close
     end
 
  end
@@ -32,35 +42,65 @@ class Corpus
     File.join(persistence_dir, docid)
   end
 
+
+  def docid(docid)
+    begin
+      if @document_repo.include?(docid)
+        Document.new(persistence_for(docid), docid, @document_repo[docid], @global_annotations, self)
+      else
+        namespace, id, type = docid.split(":")
+        if @@claims.include?(namespace)
+
+          docid = self.instance_exec id, type, &(@@claims[namespace])
+          docid = docid.first if Array === docid
+          self.docid(docid)
+        else
+          raise "Document '#{ docid }' was not found." unless @document_repo.include? docid
+        end
+      end
+    ensure
+      @document_repo.close
+    end
+  end
+
   def document(namespace, id, type, hash)
     docid = [namespace, id, type, hash] * ":"
-    raise "Document '#{ docid }' was not found." unless @document_repo.include? docid
-    Document.new(persistence_for(docid), docid, @document_repo[docid], @global_annotations)
+    self.docid(docid)
   end
 
-  def docid(docid)
-    raise "Document '#{ docid }' was not found." unless @document_repo.include? docid
-    Document.new(persistence_for(docid), docid, @document_repo[docid], @global_annotations)
+  def add_document(text, namespace = nil, id = nil, type = nil)
+    text = Misc.fixutf8(text)
+    hash = Digest::MD5.hexdigest(text)
+    @document_repo.add(text, namespace, id, type, hash)
   end
 
-  def add_document(text, namespace, id, type = nil)
-    hash = Digest::MD5.hexdigest(text)
+  def add_docid(text, docid)
+    namespace, id, type, hash = docid.split(":")
     @document_repo.add(text, namespace, id, type, hash)
   end
 
+
   def find(namespace=nil, id = nil, type = nil, hash = nil)
     @document_repo.find(namespace, id, type, hash).collect{|docid|
-      Document.new(persistence_for(docid), docid, @document_repo[docid], @global_annotations)
+      self.docid(docid)
     }
   end
 
   def find_docid(docid)
     @document_repo.find_docid(docid).collect{|docid|
-      Document.new(persistence_for(docid), docid, @document_repo[docid], @global_annotations)
+      self.docid(docid)
     }
   end
 
   def exists?(namespace=nil, id = nil, type = nil, hash = nil)
     find(namespace, id, type, hash).any?
   end
+
+  def [](docid)
+    self.docid(docid)
+  end
+  
+  def include?(id)
+    @document_repo.include? id
+  end
 end

data/lib/rbbt/text/corpus/document.rb

@@ -0,0 +1,361 @@
+require 'rbbt/text/segment'
+require 'rbbt/text/segment/segmented'
+require 'rbbt/tsv'
+require 'rbbt/resource/path'
+require 'rbbt/persist/tsv'
+require 'rbbt/util/misc'
+require 'rbbt/text/document'
+require 'json'
+
+class Corpus
+  class Document
+    
+    class MultipleEntity < Exception; end
+
+    attr_accessor :text, :docid, :namespace, :id, :type, :hash, :segments, :segment_indices, :persist_dir, :global_persistence, :corpus
+
+    attr_accessor :multiple_result
+    def initialize(persist_dir = nil, docid = nil, text = nil, global_persistence = nil, corpus = nil)
+      @segments = {}
+      @segment_indices = {}
+      @corpus = corpus
+
+      if not persist_dir.nil?
+        @persist_dir = persist_dir
+        @persist_dir = Path.setup(@persist_dir) if not Path == @persist_dir
+      end
+
+      @global_persistence = global_persistence
+
+      if not docid.nil?
+        @docid = docid 
+        update_docid
+      end
+      @text = text unless text.nil?
+    end
+
+    def update_docid
+      @namespace, @id, @type, @hash = docid.split(":", -1)
+    end
+
+    def docid=(docid)
+      @docid = docid 
+      update_docid
+    end
+
+    def self.define(entity, &block)
+      send :define_method, "produce_#{entity}", &block
+
+      self.class_eval <<-EOC, __FILE__, __LINE__
+        def load_#{entity}(raw = false)
+          return if segments.include? "#{ entity }"
+          if self.respond_to?("load_with_persistence_#{entity}") and not @persist_dir.nil?
+            segments["#{entity}"] = load_with_persistence_#{entity}(raw)
+          else
+            segments["#{ entity }"] = produce_#{entity}
+          end
+        end
+
+        def #{entity}(raw = false)
+          begin
+            entities = segments["#{ entity }"]
+            if entities.nil?
+              load_#{entity}(raw)
+              entities = segments["#{ entity }"]
+            end
+          end
+
+          entities
+        end
+
+        def #{entity}_at(pos, persist = false)
+          segment_index("#{ entity }", persist ? File.join(@persist_dir, 'ranges') : nil)[pos]
+        end
+
+      EOC
+    end
+
+    def self.define_multiple(entity, &block)
+      send :define_method, "produce_#{entity}" do
+        return self.multiple_result[entity] if self.multiple_result && self.multiple_result[entity]
+        raise MultipleEntity, "Entity #{entity} runs with multiple documents, please prepare beforehand with prepare_multiple: #{self.docid}"
+      end
+
+      name = "multiple_produce_#{entity}"
+      class << self
+        self
+      end.send :define_method, name, &block
+
+      self.class_eval <<-EOC, __FILE__, __LINE__
+        def load_#{entity}(raw = false)
+          return if segments.include? "#{ entity }"
+          if self.respond_to?("load_with_persistence_#{entity}") and not @persist_dir.nil?
+            segments["#{entity}"] = load_with_persistence_#{entity}(raw)
+          else
+            segments["#{ entity }"] = produce_#{entity}
+          end
+        end
+
+        def #{entity}(raw = false)
+          begin
+            entities = segments["#{ entity }"]
+            if entities.nil?
+              load_#{entity}(raw)
+              entities = segments["#{ entity }"]
+            end
+          end
+
+          entities
+        end
+
+        def #{entity}_at(pos, persist = false)
+          segment_index("#{ entity }", persist ? File.join(@persist_dir, 'ranges') : nil)[pos]
+        end
+
+      EOC
+    end
+
+    def self.prepare_multiple(docs, entity)
+      missing = []
+      docs.each do |doc|
+        begin
+          doc.send(entity)
+        rescue MultipleEntity
+          missing << doc
+        end
+      end
+      res = self.send("multiple_produce_#{entity.to_s}", missing)
+      case res
+      when Array
+        res.each_with_index do |res,i|
+          missing[i].multiple_result ||= {}
+          missing[i].multiple_result[entity] = res
+        end
+      when Hash
+        res.each do |document,res|
+          case document
+          when Corpus::Document
+            document.multiple_result[entity] = res
+          when String
+            document = missing.select{|d| d.docid == document}.first
+            document.multiple_result[entity] = res
+          end
+        end
+      end
+      missing.each{|doc| doc.send entity }
+    end
+
+
+    #{{{ PERSISTENCE
+
+    TSV_REPOS = {}
+    FIELDS_FOR_ENTITY_PERSISTENCE = {}
+    def self.persist(entity, fields = nil)
+
+      if not fields.nil?
+        fields = [fields] if not Array === fields
+        fields = fields.collect{|f| f.to_s}
+        FIELDS_FOR_ENTITY_PERSISTENCE[entity.to_s] = fields
+      end
+
+      self.class_eval <<-EOC, __FILE__, __LINE__
+        def load_with_persistence_#{entity}(raw = false)
+          fields = FIELDS_FOR_ENTITY_PERSISTENCE["#{ entity }"]
+
+          tsv_file = File.join(@persist_dir.find, "#{ entity }")
+
+          return nil if raw == :check and File.exists? tsv_file
+
+          annotations = Persist.persist("Entity[#{ entity }]", :tsv, :file => tsv_file) do 
+            segments = produce_#{entity}
+            tsv = Segment.tsv(segments, fields)
+          end
+
+          return annotations if raw
+
+          annotations.unnamed = true
+          annotations.collect{|id, annotation| 
+            Segment.load_tsv_values(text, annotation, annotations.fields)
+          }
+        end
+            EOC
+    end
+
+    def self.persist_in_tsv(entity, tsv = nil, fields = nil)
+      tsv = TSV.setup(Persist.open_tokyocabinet(tsv, false, :list), :key => "ID", :fields => ["Start", "End", "JSON", "Document ID", "Entity Type"]).tap{|t| t.unnamed = true, t.close} if Path === tsv
+
+      if ! tsv.nil? && ! tsv.respond_to?(:keys)
+        fields = tsv
+        tsv = nil
+      end
+
+      TSV_REPOS[entity.to_s] = tsv
+
+      if ! fields.nil?
+        fields = [fields] if not Array === fields
+        fields = fields.collect{|f| f.to_s}
+        FIELDS_FOR_ENTITY_PERSISTENCE[entity.to_s] = fields unless fields.nil?
+      end
+
+      self.class_eval <<-EOC, __FILE__, __LINE__
+        def load_with_persistence_#{entity}(raw = false)
+          repo = TSV_REPOS["#{ entity }"]
+          if repo.nil?
+            raise "No persistence file or persistence dir for persist_in_tsv" if persist_dir.nil?
+            repo = Persist.open_tokyocabinet(persist_dir.annotations_by_type.find, true, :marshal_tsv)
+          end
+
+          fields = FIELDS_FOR_ENTITY_PERSISTENCE["#{ entity }"]
+          begin
+            if ! repo.include?("#{ entity }")
+              segments = produce_#{entity}
+              repo.write_and_read do
+                repo["#{entity}"] = Segment.tsv(segments, fields) if segments.any?
+              end
+            else
+              if raw == :check
+                repo.close
+                return nil
+              end
+            end
+            
+            annotations = repo["#{entity}"]
+
+            repo.close
+
+            return annotations if raw
+
+            annotations.unnamed = true
+            annotations.collect{|id, annotation| 
+              Segment.load_tsv_values(text, annotation, annotations.fields)
+            }
+          ensure
+            repo.close
+          end
+        end
+      EOC
+    end
+    
+    def self.persist_in_global_tsv(entity, tsv = nil, fields = nil, doc_field = nil, entity_field = nil)
+      tsv = TSV.setup(Persist.open_tokyocabinet(tsv, false, :list), :key => "ID", :fields => (fields || ["Start", "End", "JSON", "Document ID", "Entity Type"])).tap{|t| t.unnamed = true, t.close} if Path === tsv
+
+      doc_field ||= "Document ID" 
+      entity_field ||= "Entity Type"
+
+      TSV_REPOS[entity.to_s] = tsv
+
+      if not fields.nil?
+        fields = [fields] if not Array === fields
+        fields = fields.collect{|f| f.to_s}
+      else
+        fields = nil
+      end
+
+      FIELDS_FOR_ENTITY_PERSISTENCE[entity.to_s] = fields 
+
+      self.class_eval <<-EOC, __FILE__, __LINE__
+        def load_with_persistence_#{entity}(raw = false)
+          fields = FIELDS_FOR_ENTITY_PERSISTENCE["#{ entity }"]
+
+          data = TSV_REPOS["#{ entity }"] || @global_persistence
+          
+          begin
+
+            data.read true
+
+            fields = data.fields if fields.nil? and data.respond_to? :fields
+
+
+            if data.respond_to? :persistence_path and String === data.persistence_path
+              data.filter(data.persistence_path + '.filters')
+            end
+
+            data.add_filter("field:#{ doc_field }", @docid) if data.fields.include?("#{doc_field}")
+            data.add_filter("field:#{ entity_field }", "#{ entity }") if data.fields.include?("#{entity_field}")
+            keys = data.keys
+            data.pop_filter if data.fields.include?("#{entity_field}")
+            data.pop_filter if data.fields.include?("#{doc_field}")
+
+            if keys.empty?
+              segments = produce_#{entity}
+              segments << Segment.setup("No #{entity} found in document " + @docid.to_s, -1) if segments.empty?
+              tsv = Segment.tsv(segments, *fields.reject{|f| ["#{doc_field}", "#{entity_field}", "Start", "End", "annotation_types"].include? f})
+
+              tsv.add_field "#{ doc_field }" do
+                @docid
+              end
+
+              tsv.add_field "#{ entity_field }" do
+                "#{ entity }"
+              end
+
+              data.add_filter("field:#{ doc_field }", @docid) if data.fields.include?("#{doc_field}")
+              data.add_filter("field:#{ entity_field }", "#{ entity }") if data.fields.include?("#{entity_field}")
+              data.write true
+              keys = tsv.collect do |key, value|
+                data[key] = value
+                key
+              end
+              data.pop_filter if data.fields.include?("#{entity_field}")
+              data.pop_filter if data.fields.include?("#{doc_field}")
+              data.read
+
+            else
+              if raw == :check
+                data.close
+                return nil
+              end
+            end
+
+            return data.values if raw
+
+            start_pos = data.identify_field "Start"
+            segments = data.values_at(*keys).collect{|annotation| 
+                pos = annotation[start_pos]
+                Segment.load_tsv_values(text, annotation, data.fields) unless [-1, "-1", [-1], ["-1"]].include? pos
+            }.compact
+            data.close
+
+            segments
+          ensure
+            data.close
+          end
+
+        end
+        EOC
+    end
+
+    def segment_index(name, persist_dir = nil)
+      @segment_indices[name] ||= Segment.index(self.send(name), persist_dir.nil? ? :memory : File.join(persist_dir, name + '.range'))
+    end
+
+    def load_into(segment, *annotations)
+      options = annotations.pop if Hash === annotations.last
+      options ||= {}
+
+      if options[:persist] and not @persist_dir.nil?
+        persist_dir = File.join(@persist_dir, 'ranges')
+      else
+        persist_dir = nil
+      end
+
+      Segmented.setup(segment, {})
+      annotations.collect do |name|
+        name = name.to_s
+        index = segment_index(name, persist_dir)
+        annotations = index[segment.range]
+        segment.segments[name] ||= {}
+        segment.segments[name] = annotations
+        class << segment
+          self
+        end.class_eval "def #{ name }; @segments['#{ name }']; end", __FILE__, __LINE__
+      end
+
+      segment
+    end
+
+    def entity
+      Object::Document.setup(self.docid, corpus)
+    end
+  end
+end