rbbt-text 1.1.8 → 1.1.9

data/test/rbbt/text/corpus/test_document.rb

@@ -0,0 +1,52 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
+require 'rbbt/text/corpus/document'
+
+class TestCorpusDocument < Test::Unit::TestCase
+  def setup
+    Log.severity = 0
+
+    Corpus::Document.define :words do
+      words = self.text.split(" ")
+      Segment.align(self.text, words)
+    end
+
+    Open.mkdir Rbbt.tmp.test.annotations.find
+    Corpus::Document.persist_in_global_tsv(:words, Rbbt.tmp.test.anotations.words.find)
+
+
+    Corpus::Document.define_multiple :words2 do |documents|
+      documents.collect do |doc|
+        words = doc.text.split(" ")
+        Segment.align(doc.text, words)
+      end
+    end
+
+    Corpus::Document.persist_in_global_tsv(:words2, Rbbt.tmp.test.anotations.counts.find)
+  end
+
+  def test_words
+    text = "This is a test document"
+    document = Corpus::Document.new(Rbbt.tmp.test.persist, "TEST:test_doc", text)
+    assert_equal Segment.sort(document.words), text.split(" ")
+  end
+
+  def test_words_multiple
+    document1 = Corpus::Document.new(Rbbt.tmp.test.persist, "TEST:test_doc:1", "This is a test document")
+    document2 = Corpus::Document.new(Rbbt.tmp.test.persist, "TEST:test_doc2:2", "This is a another test document")
+
+    docs = [document1, document2]
+
+    Corpus::Document.prepare_multiple(docs, :words2)
+    
+    assert_equal document1.words2, document1.text.split(" ")
+    assert_equal document2.words2, document2.text.split(" ")
+
+    document1 = Corpus::Document.new(Rbbt.tmp.test.persist, "TEST:test_doc:1", "This is a test document")
+    document2 = Corpus::Document.new(Rbbt.tmp.test.persist, "TEST:test_doc2:2", "This is a another test document")
+
+    docs = [document1, document2]
+
+    Corpus::Document.prepare_multiple(docs, :words2)
+  end
+end
+

data/test/rbbt/{ner → text}/segment/test_named_entity.rb

@@ -1,6 +1,6 @@
 require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
-require 'rbbt/ner/segment'
-require 'rbbt/ner/segment/named_entity'
+require 'rbbt/text/segment'
+require 'rbbt/text/segment/named_entity'
 
 class TestClass < Test::Unit::TestCase
   def test_info

data/test/rbbt/{ner → text}/segment/test_segmented.rb

@@ -1,5 +1,5 @@
 require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
-require 'rbbt/ner/segment/segmented'
+require 'rbbt/text/segment/segmented'
 
 class TestClass < Test::Unit::TestCase
   def test_split

data/test/rbbt/{ner → text}/segment/test_transformed.rb

@@ -1,6 +1,6 @@
 require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
-require 'rbbt/ner/segment/transformed'
-require 'rbbt/ner/segment/named_entity'
+require 'rbbt/text/segment/transformed'
+require 'rbbt/text/segment/named_entity'
 require 'rexml/document'
 
 class TestClass < Test::Unit::TestCase
@@ -98,7 +98,6 @@ More recently, PPAR activators were shown to inhibit the activation of inflammat
 
     assert_equal original, a
 
-
     assert_equal original, a
 
     exp1, exp2 = nil, nil
@@ -286,5 +285,99 @@ More recently, PPAR activators were shown to inhibit the activation of inflammat
     end
 
   end
+
+  def test_by_sentence
+    a = "This is a first sentences. ILF can bind to purine-rich regulatory motifs such as the human T-cell leukemia virus-long terminal region and the interleukin-2 promoter."
+
+    sentence_pos = a.index('.')+2
+    sentence = a[sentence_pos..-1]
+    Segment.setup sentence, sentence_pos
+
+    gene1 = "ILF"
+    gene1.extend NamedEntity
+    gene1.offset = a.index gene1
+    gene1.type = "Gene"
+
+    Transformed.with_transform(sentence, [gene1], "[G]") do 
+      assert_equal sentence.sub("ILF", "[G]"), sentence
+    end
+  end
+
+  def test_collisions
+    text =<<-EOF.chomp
+This is another sentence. Protein (nsp1), helicase (nsp13).
+    EOF
+
+    sentence_pos = text.index(".") + 2
+    sentence = Segment.setup(text[sentence_pos..-1], sentence_pos)
+
+    viral = %w(nsp1 nsp13)
+    human = %w(helicase)
+
+    viral = viral.collect do |e|
+      next unless text.index(e)
+      NamedEntity.setup(e, text.index(e), "VirGene")
+    end.compact
+
+    human = human.collect do |e|
+      next unless text.index(e)
+      NamedEntity.setup(e, text.index(e), "HumGene")
+    end
+
+    clean = human.reject{|s| s.overlaps(viral).any?}
+
+    Transformed.with_transform(sentence, viral, Proc.new{|e| "[VIRAL=#{e}]"}) do
+      assert_equal sentence, "Protein ([VIRAL=nsp1]), helicase ([VIRAL=nsp13])."
+      Transformed.with_transform(sentence, clean, Proc.new{|e| "[HUMAN=#{e}]"}) do
+        assert_equal sentence, "Protein ([VIRAL=nsp1]), [HUMAN=helicase] ([VIRAL=nsp13])."
+      end
+    end
+  end
+
+
+  def test_collisions2
+    text =<<-EOF.chomp
+This is another sentence. Among the nonstructural proteins, the leader protein (nsp1), the papain-like protease (nsp3), the nsp4, the 3C-like protease (nsp5), the nsp7, the nsp8, the nsp9, the nsp10, the RNA-directed RNA polymerase (nsp12), the helicase (nsp13), the guanine-N7 methyltransferase (nsp14), the uridylate-specific endoribonuclease (nsp15), the 2'-O-methyltransferase (nsp16), and the ORF7a protein could be built on the basis of homology templates.
+    EOF
+
+    sentence_pos = text.index(".") + 2
+    sentence = Segment.setup(text[sentence_pos..-1], sentence_pos)
+
+    target = sentence.dup
+
+    viral = %w(nsp1 nsp4 nsp5 nsp7 nsp8 nsp9 nsp10 nsp12 nsp13 nsp14 nsp15 ORF7a spike)
+    human = %w(helicase nsp5 nsp4 nsp3)
+
+    viral = viral.collect do |e|
+      next unless text.index(e)
+      NamedEntity.setup(e, text.index(e), "VirGene")
+    end.compact
+
+    human = human.collect do |e|
+      next unless text.index(e)
+      NamedEntity.setup(e, text.index(e), "HumGene")
+    end
+
+    clean = human.reject{|s| s.overlaps(viral).any?}
+
+    tag = Misc.digest("TAG")
+
+    viral.each do |e|
+      target.gsub!(/\b#{e}\b/, "[VIRAL=#{e}-#{tag}]")
+    end
+
+    target_tmp = target.dup
+
+    clean.each do |e|
+      target.gsub!(/\b#{e}\b/, "[HUMAN=#{e}-#{tag}]")
+    end
+
+    Transformed.with_transform(sentence, viral, Proc.new{|e| "[VIRAL=#{e}-#{tag}]"}) do
+      assert_equal sentence, target_tmp
+      Transformed.with_transform(sentence, clean, Proc.new{|e| "[HUMAN=#{e}-#{tag}]"}) do
+        assert_equal sentence, target
+      end
+    end
+  end
 end
 

data/test/rbbt/text/test_corpus.rb

@@ -0,0 +1,34 @@
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'test/unit'
+require 'rbbt-util'
+require 'rbbt/text/corpus'
+
+class Corpus::Document
+
+  define :words do
+    text.split(" ")
+  end
+end
+
+class TestClass < Test::Unit::TestCase
+  def test_document
+    Log.severity = 0
+    text = "This is a test document"
+
+    docid = nil
+    TmpFile.with_file do |dir|
+      corpus = Corpus.new dir
+      docid = corpus.add_document text, :TEST, :test_doc
+      document = corpus.docid(docid)
+      assert_equal text, document.text
+
+      corpus = Corpus.new dir
+      document = corpus.docid(docid)
+      assert_equal text, document.text
+      document = corpus.find(:TEST, :test_doc).first
+      assert_equal text, document.text
+    end
+  end
+end
+

data/test/rbbt/text/test_document.rb

@@ -0,0 +1,58 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
+require 'rbbt/text/document'
+require 'rbbt/text/corpus/sources/pmid'
+
+class TestDocument < Test::Unit::TestCase
+  def setup
+    Log.severity = 0
+    Document.corpus = Corpus.new Rbbt.tmp.test.document_corpus
+
+    Corpus::Document.define :words do
+      words = self.text.split(" ")
+      Segment.align(self.text, words)
+    end
+
+    Corpus::Document.define :genes do
+      require 'rbbt/ner/banner'
+      Banner.new.match(self.text)
+    end
+
+    Corpus::Document.persist_in_global_tsv("genes")
+    Corpus::Document.persist_in_global_tsv(:words)
+  end
+
+  def test_title_and_text
+    document = Document.setup('PMID:32272262')
+
+    assert document.text.downcase.include?("covid")
+    assert_equal "High-resolution Chest CT Features and Clinical Characteristics of Patients Infected with COVID-19 in Jiangsu, China.", document.title
+  end
+
+  def test_full_text
+    document = Document.setup('PMID:4304705')
+    assert document.text.length < document.full_text.length
+  end
+
+  def test_words
+    document = Document.setup('PMID:32272262')
+    words = document.entities :words
+    assert words.first.respond_to?(:offset)
+  end
+
+  def test_genes
+    text = "This is a mention to TP53, a gene that should be found"
+    document = Document.setup(Document.corpus.add_document(text, "TEST"))
+    genes = document.entities :genes
+
+    assert_equal "TP53", genes.first
+    assert genes.first.respond_to?(:offset)
+
+    text = "This is a mention to TP53, a gene that should be found"
+    document = Document.setup(Document.corpus.add_document(text, "TEST"))
+    genes = document.entities :genes
+
+    assert_equal "TP53", genes.first
+    assert genes.first.respond_to?(:offset)
+  end
+end
+

data/test/rbbt/{ner → text}/test_segment.rb

@@ -1,7 +1,7 @@
 require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
-require 'rbbt/ner/segment'
+require 'rbbt/text/segment'
 
-class TestClass < Test::Unit::TestCase
+class TestSegment < Test::Unit::TestCase
   def test_info
     a = "test"
     a.extend Segment

data/test/test_helper.rb

@@ -6,7 +6,7 @@ require 'rbbt'
 require 'rbbt/persist'
 require 'rbbt/util/tmpfile'
 require 'rbbt/util/log'
-require 'rbbt/corpus/document_repo'
+require 'rbbt/text/corpus'
 
 class Test::Unit::TestCase
   def get_test_datafile(file)
@@ -22,8 +22,8 @@ class Test::Unit::TestCase
     FileUtils.rm_rf Rbbt.tmp.test.find :user
     Persist::CONNECTIONS.values.each do |c| c.close end
     Persist::CONNECTIONS.clear
-    DocumentRepo::TC_CONNECTIONS.values.each do |c| c.close end
-    DocumentRepo::TC_CONNECTIONS.clear
+    Corpus::DocumentRepo::TC_CONNECTIONS.values.each do |c| c.close end
+    Corpus::DocumentRepo::TC_CONNECTIONS.clear
   end
 
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-text
 version: !ruby/object:Gem::Version
-  version: 1.1.8
+  version: 1.1.9
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-01-31 00:00:00.000000000 Z
+date: 2020-04-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rbbt-util
@@ -78,11 +78,6 @@ files:
 - lib/rbbt/bow/bow.rb
 - lib/rbbt/bow/dictionary.rb
 - lib/rbbt/bow/misc.rb
-- lib/rbbt/corpus/corpus.rb
-- lib/rbbt/corpus/document.rb
-- lib/rbbt/corpus/document_repo.rb
-- lib/rbbt/corpus/sources/pubmed.rb
-- lib/rbbt/entity/document.rb
 - lib/rbbt/ner/NER.rb
 - lib/rbbt/ner/abner.rb
 - lib/rbbt/ner/banner.rb
@@ -99,17 +94,22 @@ files:
 - lib/rbbt/ner/rnorm.rb
 - lib/rbbt/ner/rnorm/cue_index.rb
 - lib/rbbt/ner/rnorm/tokens.rb
-- lib/rbbt/ner/segment.rb
-- lib/rbbt/ner/segment/docid.rb
-- lib/rbbt/ner/segment/named_entity.rb
-- lib/rbbt/ner/segment/relationship.rb
-- lib/rbbt/ner/segment/segmented.rb
-- lib/rbbt/ner/segment/token.rb
-- lib/rbbt/ner/segment/transformed.rb
 - lib/rbbt/ner/token_trieNER.rb
 - lib/rbbt/nlp/genia/sentence_splitter.rb
 - lib/rbbt/nlp/nlp.rb
 - lib/rbbt/nlp/open_nlp/sentence_splitter.rb
+- lib/rbbt/text/corpus.rb
+- lib/rbbt/text/corpus/document.rb
+- lib/rbbt/text/corpus/document_repo.rb
+- lib/rbbt/text/corpus/sources/pmid.rb
+- lib/rbbt/text/document.rb
+- lib/rbbt/text/segment.rb
+- lib/rbbt/text/segment/docid.rb
+- lib/rbbt/text/segment/named_entity.rb
+- lib/rbbt/text/segment/relationship.rb
+- lib/rbbt/text/segment/segmented.rb
+- lib/rbbt/text/segment/token.rb
+- lib/rbbt/text/segment/transformed.rb
 - share/install/software/ABNER
 - share/install/software/BANNER
 - share/install/software/ChemicalTagger
@@ -129,10 +129,6 @@ files:
 - test/rbbt/bow/test_dictionary.rb
 - test/rbbt/bow/test_misc.rb
 - test/rbbt/entity/test_document.rb
-- test/rbbt/ner/segment/test_named_entity.rb
-- test/rbbt/ner/segment/test_relationship.rb
-- test/rbbt/ner/segment/test_segmented.rb
-- test/rbbt/ner/segment/test_transformed.rb
 - test/rbbt/ner/test_NER.rb
 - test/rbbt/ner/test_abner.rb
 - test/rbbt/ner/test_banner.rb
@@ -146,11 +142,19 @@ files:
 - test/rbbt/ner/test_patterns.rb
 - test/rbbt/ner/test_regexpNER.rb
 - test/rbbt/ner/test_rnorm.rb
-- test/rbbt/ner/test_segment.rb
 - test/rbbt/ner/test_token_trieNER.rb
 - test/rbbt/nlp/genia/test_sentence_splitter.rb
 - test/rbbt/nlp/open_nlp/test_sentence_splitter.rb
 - test/rbbt/nlp/test_nlp.rb
+- test/rbbt/text/corpus/sources/test_pmid.rb
+- test/rbbt/text/corpus/test_document.rb
+- test/rbbt/text/segment/test_named_entity.rb
+- test/rbbt/text/segment/test_relationship.rb
+- test/rbbt/text/segment/test_segmented.rb
+- test/rbbt/text/segment/test_transformed.rb
+- test/rbbt/text/test_corpus.rb
+- test/rbbt/text/test_document.rb
+- test/rbbt/text/test_segment.rb
 - test/test_helper.rb
 homepage: http://github.com/mikisvaz/rbbt-util
 licenses: []
@@ -178,6 +182,15 @@ test_files:
 - test/rbbt/nlp/test_nlp.rb
 - test/rbbt/nlp/open_nlp/test_sentence_splitter.rb
 - test/rbbt/nlp/genia/test_sentence_splitter.rb
+- test/rbbt/text/test_document.rb
+- test/rbbt/text/corpus/sources/test_pmid.rb
+- test/rbbt/text/corpus/test_document.rb
+- test/rbbt/text/test_segment.rb
+- test/rbbt/text/test_corpus.rb
+- test/rbbt/text/segment/test_transformed.rb
+- test/rbbt/text/segment/test_relationship.rb
+- test/rbbt/text/segment/test_named_entity.rb
+- test/rbbt/text/segment/test_segmented.rb
 - test/rbbt/bow/test_bow.rb
 - test/rbbt/bow/test_misc.rb
 - test/rbbt/bow/test_dictionary.rb
@@ -194,11 +207,6 @@ test_files:
 - test/rbbt/ner/test_banner.rb
 - test/rbbt/ner/test_token_trieNER.rb
 - test/rbbt/ner/test_finder.rb
-- test/rbbt/ner/test_segment.rb
 - test/rbbt/ner/test_linnaeus.rb
-- test/rbbt/ner/segment/test_transformed.rb
-- test/rbbt/ner/segment/test_relationship.rb
-- test/rbbt/ner/segment/test_named_entity.rb
-- test/rbbt/ner/segment/test_segmented.rb
 - test/rbbt/ner/test_oscar4.rb
 - test/test_helper.rb

data/lib/rbbt/corpus/document.rb

@@ -1,266 +0,0 @@
-require 'rbbt/ner/segment'
-require 'rbbt/ner/segment/segmented'
-require 'rbbt/tsv'
-require 'rbbt/resource/path'
-require 'rbbt/persist/tsv'
-require 'rbbt/util/misc'
-require 'json'
-
-class Document
-
-  attr_accessor :text, :docid, :namespace, :id, :type, :hash, :segments, :segment_indices, :persist_dir, :global_persistence
-  def initialize(persist_dir = nil, docid = nil, text = nil, global_persistence = nil)
-    @segments = {}
-    @segment_indices = {}
-
-    if not persist_dir.nil?
-      @persist_dir = persist_dir
-      @persist_dir = Path.setup(@persist_dir) if not Path == @persist_dir
-    end
-
-    @global_persistence = global_persistence
-
-    if not docid.nil?
-      @docid = docid 
-      update_docid
-    end
-    @text = text unless text.nil?
-  end
-
-  def update_docid
-    @namespace, @id, @type, @hash = docid.split(":", -1)
-  end
-
-  def docid=(docid)
-    @docid = docid 
-    update_docid
-  end
-
-  #{{{ PERSISTENCE
-
-  TSV_REPOS = {}
-  FIELDS_FOR_ENTITY_PERSISTENCE = {}
-  def self.persist(entity, fields = nil)
-
-    if not fields.nil?
-      fields = [fields] if not Array === fields
-      fields = fields.collect{|f| f.to_s}
-      FIELDS_FOR_ENTITY_PERSISTENCE[entity.to_s] = fields
-    end
-
-    self.class_eval <<-EOC
-      def load_with_persistence_#{entity}(raw = false)
-        fields = FIELDS_FOR_ENTITY_PERSISTENCE["#{ entity }"]
-
-        tsv_file = File.join(@persist_dir.find, "#{ entity }")
-
-        return nil if raw == :check and File.exists? tsv_file
-
-        annotations = Persist.persist("Entity[#{ entity }]", :tsv, :file => tsv_file) do 
-          segments = produce_#{entity}
-          tsv = Segment.tsv(segments, fields)
-        end
-
-        return annotations if raw
-
-        annotations.unnamed = true
-        annotations.collect{|id, annotation| 
-          Segment.load_tsv_values(text, annotation, annotations.fields)
-        }
-      end
-          EOC
-  end
-
-  def self.persist_in_tsv(entity, tsv = nil, fields = nil)
-    if not tsv.nil? and not tsv.respond_to?(:keys)
-      fields = tsv
-      tsv = nil
-    end
-
-    TSV_REPOS[entity.to_s] = tsv
-
-    if not fields.nil?
-      fields = [fields] if not Array === fields
-      fields = fields.collect{|f| f.to_s}
-      FIELDS_FOR_ENTITY_PERSISTENCE[entity.to_s] = fields unless fields.nil?
-    end
-
-    self.class_eval <<-EOC
-      def load_with_persistence_#{entity}(raw = false)
-        repo = TSV_REPOS["#{ entity }"]
-        if repo.nil?
-          raise "No persistence file or persistence dir for persist_in_tsv" if persist_dir.nil?
-          repo = Persist.open_tokyocabinet(persist_dir.annotations_by_type.find, true, :marshal_tsv)
-        end
-
-        fields = FIELDS_FOR_ENTITY_PERSISTENCE["#{ entity }"]
-        if not repo.include? "#{ entity }"
-          segments = produce_#{entity}
-          repo.write
-          repo["#{entity}"] = Segment.tsv(segments, fields)
-          repo.read
-        else
-          if raw == :check
-            repo.close
-            return nil
-          end
-        end
-
-        
-        annotations = repo["#{entity}"]
-
-        repo.close
-
-
-        return annotations if raw
-
-        annotations.unnamed = true
-        annotations.collect{|id, annotation| 
-          Segment.load_tsv_values(text, annotation, annotations.fields)
-        }
-      end
-    EOC
-  end
-  
-  def self.persist_in_global_tsv(entity, tsv = nil, fields = nil, doc_field = nil, entity_field = nil)
-    doc_field ||= "Document ID" 
-    entity_field ||= "Entity Type"
-
-    TSV_REPOS[entity.to_s] = tsv
-
-    if not fields.nil?
-      fields = [fields] if not Array === fields
-      fields = fields.collect{|f| f.to_s}
-    else
-      fields = nil
-    end
-
-    FIELDS_FOR_ENTITY_PERSISTENCE[entity.to_s] = fields 
-
-    self.class_eval <<-EOC
-      def load_with_persistence_#{entity}(raw = false)
-        fields = FIELDS_FOR_ENTITY_PERSISTENCE["#{ entity }"]
-
-        data = TSV_REPOS["#{ entity }"] || @global_persistence
-
-        data.read true
-
-        fields = data.fields if fields.nil? and data.respond_to? :fields
-
-
-        if data.respond_to? :persistence_path and String === data.persistence_path
-          data.filter(data.persistence_path + '.filters')
-        end
-        data.add_filter("field:#{ doc_field }", @docid)
-        data.add_filter("field:#{ entity_field }", "#{ entity }")
-        keys = data.keys
-        data.pop_filter
-        data.pop_filter
-
-        if keys.empty?
-          segments = produce_#{entity}
-          segments << Segment.setup("No #{entity} found in document " + @docid.to_s, -1) if segments.empty?
-          tsv = Segment.tsv(segments, *fields.reject{|f| ["#{doc_field}", "#{entity_field}", "Start", "End", "annotation_types"].include? f})
-
-          tsv.add_field "#{ doc_field }" do
-            @docid
-          end
-
-          tsv.add_field "#{ entity_field }" do
-            "#{ entity }"
-          end
-
-          data.add_filter("field:#{ doc_field }", @docid)
-          data.add_filter("field:#{ entity_field }", "#{ entity }")
-          data.write true
-          keys = tsv.collect do |key, value|
-            data[key] = value
-            key
-          end
-          data.pop_filter
-          data.pop_filter
-          data.read
-
-        else
-          if raw == :check
-            data.close
-            return nil
-          end
-        end
-
-        return data.values if raw
-
-        start_pos = data.identify_field "Start"
-        segments = data.values_at(*keys).collect{|annotation| 
-            pos = annotation[start_pos]
-            Segment.load_tsv_values(text, annotation, data.fields) unless [-1, "-1", [-1], ["-1"]].include? pos
-         }.compact
-        data.close
-
-        segments
-      end
-      EOC
-  end
-
-
-  def self.define(entity, &block)
-    send :define_method, "produce_#{entity}", &block
-
-    self.class_eval <<-EOC
-      def load_#{entity}(raw = false)
-        return if segments.include? "#{ entity }"
-        if self.respond_to?("load_with_persistence_#{entity}") and not @persist_dir.nil?
-          segments["#{entity}"] = load_with_persistence_#{entity}(raw)
-        else
-          segments["#{ entity }"] = produce_#{entity}
-        end
-      end
-
-      def #{entity}(raw = false)
-        begin
-          entities = segments["#{ entity }"]
-          if entities.nil?
-            load_#{entity}(raw)
-            entities = segments["#{ entity }"]
-          end
-        end
-
-        entities
-      end
-
-      def #{entity}_at(pos, persist = false)
-        segment_index("#{ entity }", persist ? File.join(@persist_dir, 'ranges') : nil)[pos]
-      end
-
-    EOC
-  end
-
-  def segment_index(name, persist_dir = nil)
-    @segment_indices[name] ||= Segment.index(self.send(name), persist_dir.nil? ? :memory : File.join(persist_dir, name + '.range'))
-  end
-
-  def load_into(segment, *annotations)
-    options = annotations.pop if Hash === annotations.last
-    options ||= {}
-
-    if options[:persist] and not @persist_dir.nil?
-      persist_dir = File.join(@persist_dir, 'ranges')
-    else
-      persist_dir = nil
-    end
-
-    Segmented.setup(segment, {})
-    annotations.collect do |name|
-      name = name.to_s
-      index = segment_index(name, persist_dir)
-      annotations = index[segment.range]
-      segment.segments[name] ||= {}
-      segment.segments[name] = annotations
-      class << segment
-        self
-      end.class_eval "def #{ name }; @segments['#{ name }']; end"
-    end
-
-    segment
-  end
-end