rbbt-text 0.5.0 → 0.6.0

data/lib/rbbt/ner/NER.rb

@@ -1,6 +1,6 @@
-require 'rbbt/ner/annotations'
-require 'rbbt/ner/annotations/named_entity'
-require 'rbbt/ner/annotations/annotated'
+require 'rbbt/ner/segment'
+require 'rbbt/ner/segment/named_entity'
+require 'rbbt/ner/segment/segmented'
 
 class NER
   def entities(text, protect = false, *args)
@@ -13,7 +13,7 @@ class NER
         }
         matches
       end.flatten
-    when (Annotated === text and protect)
+    when (Segmented === text and protect)
       entities(text.split_segments(true), protect, *args)
     else
       match(text, *args)

data/lib/rbbt/ner/abner.rb

@@ -1,13 +1,14 @@
 require 'rbbt'
 require 'rjb'
-require 'rbbt/ner/annotations'
+require 'rbbt/ner/segment'
+require 'rbbt/resource'
 require 'rbbt/ner/NER'
 
 # Offers a Ruby interface to the Abner Named Entity Recognition Package
 # in Java Abner[http://www.cs.wisc.edu/~bsettles/abner/].
 class Abner < NER
 
-  Rbbt.software.opt.ABNER.define_as_install Rbbt.share.install.software.ABNER.find
+  Rbbt.claim Rbbt.software.opt.ABNER, :install, Rbbt.share.install.software.ABNER.find
 
   @@JFile   = Rjb::import('java.io.File')
   @@Tagger  = Rjb::import('abner.Tagger')
@@ -38,9 +39,9 @@ class Abner < NER
       mention = mention.to_s; 
       offset = text.index(mention)
       if offset.nil?
-        NamedEntity.annotate(mention, nil, type.to_s)
+        NamedEntity.setup(mention, nil, type.to_s)
       else
-        NamedEntity.annotate(mention, offset + global_offset, type.to_s)
+        NamedEntity.setup(mention, offset + global_offset, type.to_s)
         text = text[offset + mention.length..-1]
         global_offset += offset + mention.length
       end

data/lib/rbbt/ner/banner.rb

@@ -1,13 +1,13 @@
 require 'rbbt'
 require 'rjb'
-require 'rbbt/ner/annotations'
+require 'rbbt/ner/segment'
 require 'rbbt/ner/NER'
 
 # Offers a Ruby interface to the Banner Named Entity Recognition Package
 # in Java. Banner[http://banner.sourceforge.net/].
 class Banner < NER
 
-  Rbbt.software.opt.BANNER.define_as_install Rbbt.share.install.software.BANNER.find
+  Rbbt.claim Rbbt.software.opt.BANNER, :install, Rbbt.share.install.software.BANNER.find
 
   @@JFile = Rjb::import('java.io.File')
   @@SimpleTokenizer = Rjb::import('banner.tokenization.SimpleTokenizer')
@@ -68,7 +68,7 @@ class Banner < NER
       mention.sub!(/^\s*/,'')
       mention.sub!(/\s*$/,'')
       offset = text.index(mention)
-      NamedEntity.annotate(mention, offset, 'GENE')
+      NamedEntity.setup(mention, offset, 'GENE')
       mention
     }
     res

data/lib/rbbt/ner/chemical_tagger.rb

@@ -1,11 +1,11 @@
 require 'rbbt'
 require 'rjb'
-require 'rbbt/ner/annotations'
+require 'rbbt/ner/segment'
 require 'rbbt/ner/NER'
 require 'rbbt/util/log'
 
 class ChemicalTagger < NER
-  Rbbt.software.opt.ChemicalTagger.define_as_install Rbbt.share.install.software.ChemicalTagger.find
+  Rbbt.claim Rbbt.software.opt.ChemicalTagger, :install, Rbbt.share.install.software.ChemicalTagger.find
 
   Rjb::load(nil, jvmargs = ['-Xms128m','-Xmx2048m'])
 
@@ -24,7 +24,7 @@ class ChemicalTagger < NER
 
     matches.collect do |mention|
       offset = text.index mention
-      NamedEntity.annotate mention, offset, "Chemical Mention", nil, nil
+      NamedEntity.setup mention, offset, "Chemical Mention", nil, nil
     end
   end
 

data/lib/rbbt/ner/ngram_prefix_dictionary.rb

@@ -1,7 +1,8 @@
-require 'rbbt-util'
-require 'rbbt/util/tsv'
-require 'rbbt/ner/annotations'
-require 'rbbt/ner/annotations/token'
+require 'rbbt'
+require 'rbbt/util/misc'
+require 'rbbt/tsv'
+require 'rbbt/ner/segment'
+require 'rbbt/ner/segment/token'
 require 'rbbt/ner/NER'
 require 'inline'
 
@@ -41,9 +42,28 @@ VALUE fast_start_with(VALUE str, VALUE cmp, int offset)
     end
   end
 
-  def self.process(hash)
+  def self.process_stream(stream)
     index = {}
-    hash.each do |code, names|
+    while line = stream.gets
+      names = line.split(/\t|\|/).select{|n| not n.empty?}.compact
+      code = names.shift
+      
+      names.each do |name|
+        ngram = name[0..2].strip
+        index[ngram] ||= []
+        index[ngram] << [name, code]
+      end
+    end
+    index
+ 
+  end
+
+  def self.process_hash(hash)
+    index = {}
+    hash.monitor = true if hash.respond_to? :monitor
+    hash.unnamed = true if hash.respond_to? :unnamed
+    method = hash.respond_to?(:through)? :through : :each
+    hash.send(method) do |code, names|
       names.each do |name|
         ngram = name[0..2].strip
         index[ngram] ||= []
@@ -94,15 +114,30 @@ VALUE fast_start_with(VALUE str, VALUE cmp, int offset)
 
   attr_accessor :index, :type
   def initialize(file, type = nil)
-    tsv = TSV.new(file, :flat)
     @type = type
-    tsv.unnamed = true
-    @index = NGramPrefixDictionary.process(tsv)
+    case
+    when (TSV === file or Hash === file)
+      Log.debug("Ngram Prefix Dictionary. Loading of lexicon hash started.")
+      @index = NGramPrefixDictionary.process_hash(file)
+    when Path === file
+      Log.debug("Ngram Prefix Dictionary. Loading of lexicon file started: #{ file }.")
+      @index = NGramPrefixDictionary.process_stream(file.open)
+    when Misc.is_filename?(file)
+      Log.debug("Ngram Prefix Dictionary. Loading of lexicon file started: #{ file }.")
+      @index = NGramPrefixDictionary.process_stream(Open.open(file))
+    when StreamIO === file
+      Log.debug("Ngram Prefix Dictionary. Loading of lexicon stream started.")
+      @index = NGramPrefixDictionary.process_stream(file)
+    else
+      raise "Format of lexicon not understood: #{file.inspect}"
+    end
+
+    Log.debug("Ngram Prefix Dictionary. Loading done.")
   end
 
   def match(text)
     NGramPrefixDictionary.match(index, text).collect{|name, code, offset|
-      NamedEntity.annotate(name, offset, type, code)
+      NamedEntity.setup(name, offset, type, code)
     }
   end
 end

data/lib/rbbt/ner/oscar3.rb

@@ -1,12 +1,12 @@
 require 'rbbt'
 require 'rjb'
 require 'libxml'
-require 'rbbt/ner/annotations'
+require 'rbbt/ner/segment'
 require 'rbbt/ner/NER'
 require 'rbbt/util/log'
 
 class OSCAR3 < NER
-  Rbbt.software.opt.OSCAR3.define_as_install Rbbt.share.install.software.OSCAR3.find
+  Rbbt.claim Rbbt.software.opt.OSCAR3, :install, Rbbt.share.install.software.OSCAR3.find
 
   @@TextToSciXML   = Rjb::import('uk.ac.cam.ch.wwmm.ptclib.scixml.TextToSciXML')
   @@ProcessingDocumentFactory   = Rjb::import('uk.ac.cam.ch.wwmm.oscar3.recogniser.document.ProcessingDocumentFactory')
@@ -50,7 +50,7 @@ class OSCAR3 < NER
         next unless type.nil? or type.include? mention_type
         score  = memm ? entities.get(key).to_string.to_f : nil
 
-        NamedEntity.annotate mention, rstart.to_i + offset, mention_type, nil, score
+        NamedEntity.setup mention, rstart.to_i + offset, mention_type, nil, score
         
         mentions << mention unless mentions.collect{|m| m.to_s}.include? mention.to_s
       end

data/lib/rbbt/ner/oscar4.rb

@@ -1,12 +1,12 @@
 require 'rbbt'
 require 'rjb'
 require 'libxml'
-require 'rbbt/ner/annotations'
+require 'rbbt/ner/segment'
 require 'rbbt/ner/NER'
 require 'rbbt/util/log'
 
 class OSCAR4 < NER
-  Rbbt.software.opt.OSCAR4.define_as_install Rbbt.share.install.software.OSCAR4.find
+  Rbbt.claim Rbbt.software.opt.OSCAR4, :install, Rbbt.share.install.software.OSCAR4.find
 
   Rjb::load(nil, jvmargs = ['-Xms128m','-Xmx2048m'])
   @@OSCAR = Rjb::import('uk.ac.cam.ch.wwmm.oscar.Oscar')
@@ -26,7 +26,7 @@ class OSCAR4 < NER
       mention = entity.getSurface
       result << mention
 
-      NamedEntity.annotate mention, entity.getStart, entity.getType, nil, entity.getNamedEntity.getConfidence
+      NamedEntity.setup mention, entity.getStart, entity.getType, nil, entity.getNamedEntity.getConfidence
     end
 
     result

data/lib/rbbt/ner/patterns.rb

@@ -1,7 +1,7 @@
-require 'rbbt/ner/annotations/named_entity'
-require 'rbbt/ner/annotations/annotated'
-require 'rbbt/ner/annotations/transformed'
-require 'rbbt/ner/annotations/relations'
+require 'rbbt/ner/segment/named_entity'
+require 'rbbt/ner/segment/segmented'
+require 'rbbt/ner/segment/transformed'
+require 'rbbt/ner/segment/relationship'
 require 'rbbt/ner/regexpNER'
 require 'rbbt/ner/token_trieNER'
 require 'rbbt/nlp/nlp'
@@ -12,7 +12,9 @@ class PatternRelExt
     patterns = Array === patterns ? patterns : [patterns]
     type ||= "Simple Pattern"
     regexpNER = RegExpNER.new type => patterns.collect{|p| /#{p}/}
-    Transformed.with_transform(sentence, sentence.annotations, Proc.new{|s| s.type.to_s.upcase}) do |sentence|
+    segments = sentence.segments
+    segments = segments.values.flatten if Hash === segments
+    Transformed.with_transform(sentence, segments, Proc.new{|s| s.type.to_s.upcase}) do |sentence|
       regexpNER.entities(sentence)
     end
   end
@@ -23,23 +25,23 @@ class PatternRelExt
     when key =~ /(.*)\[entity:(.*)\]/
       chunk_type, chunk_value = $1, $2
       annotation_types = chunk_value.split(",")
-      Proc.new{|chunk| (chunk_type == "all" or chunk.type == chunk_type) and 
-        ((Hash === chunk.annotations ? chunk.annotations.values.flatten : chunk.annotations).flatten.select{|a| NamedEntity === a}.collect{|a| a.type.to_s}.flatten & annotation_types).any? }
+      Proc.new{|chunk| (chunk_type == "all" or (Array === chunk.type ? chunk.type.include?(chunk_type) : chunk.type == chunk_type)) and 
+        ((Hash === chunk.segments ? chunk.segments.values.flatten : chunk.segments).flatten.select{|a| NamedEntity === a}.collect{|a| a.type.to_s}.flatten & annotation_types).any? }
 
     when key =~ /(.*)\[code:(.*)\]/
       chunk_type, chunk_value = $1, $2
       annotation_codes = chunk_value.split(",")
-      Proc.new{|chunk| (chunk_type == "all" or chunk.type == chunk_type) and 
-        ((Hash === chunk.annotations ? chunk.annotations.values.flatten : chunk.annotations).select{|a| NamedEntity === a}.collect{|a| a.code}.flatten & annotation_codes).any? }
+      Proc.new{|chunk| (chunk_type == "all" or (Array === chunk.type ? chunk.type.include?(chunk_type) : chunk.type == chunk_type)) and 
+        ((Hash === chunk.segments ? chunk.segments.values.flatten : chunk.segments).select{|a| NamedEntity === a}.collect{|a| a.code}.flatten & annotation_codes).any? }
 
     when key =~ /(.*)\[stem:(.*)\]/
       chunk_type, chunk_value = $1, $2
-      Proc.new{|chunk| (chunk_type == "all" or chunk.type == chunk_type) and 
+      Proc.new{|chunk| (chunk_type == "all" or (Array === chunk.type ? chunk.type.include?(chunk_type) : chunk.type == chunk_type)) and 
         chunk.split(/\s+/).select{|w| w.stem == chunk_value.stem}.any?}
 
     when key =~ /(.*)\[(.*)\]/
       chunk_type, chunk_value = $1, $2
-      Proc.new{|chunk| (chunk_type == "all" or chunk.type == chunk_type) and 
+      Proc.new{|chunk| (chunk_type == "all" or (Array === chunk.type ? chunk.type.include?(chunk_type) : chunk.type == chunk_type)) and 
         chunk.parts.values.select{|a| a == chunk_value}.any?}
 
     else
@@ -120,9 +122,9 @@ class PatternRelExt
     sentence_chunks = NLP.gdep_chunk_sentences(sentences)
 
     sentences.zip(sentence_chunks).collect do |sentence, chunks|
-      annotation_index = Segment.index(sentence.annotations)
+      annotation_index = Segment.index(sentence.segments)
       chunks.each do |chunk|
-        Annotated.annotate(chunk, annotation_index[chunk.range])
+        Segmented.setup(chunk, annotation_index[chunk.range])
       end
 
       match_chunks(chunks)

data/lib/rbbt/ner/regexpNER.rb

@@ -1,4 +1,4 @@
-require 'rbbt/ner/annotations'
+require 'rbbt/ner/segment'
 require 'rbbt/ner/NER'
 require 'rbbt/util/simpleDSL'
 
@@ -23,7 +23,7 @@ class RegExpNER < NER
       end
 
       if match and not match.empty?
-        NamedEntity.annotate(match, start + pre.length, type)
+        NamedEntity.setup(match, start + pre.length, type)
         matches << match
       end
 
@@ -86,6 +86,7 @@ class RegExpNER < NER
 
   def match(text)
     matches = RegExpNER.match_regexp_hash(text, @regexps)
+    matches
   end
 
 end

data/lib/rbbt/ner/rnorm.rb

@@ -1,7 +1,7 @@
 require 'rbbt/ner/rnorm/cue_index'
 require 'rbbt/ner/rnorm/tokens'
 require 'rbbt/util/open'
-require 'rbbt/util/tsv'
+require 'rbbt/tsv'
 require 'rbbt/sources/entrez'
 require 'rbbt/bow/bow.rb'
 
@@ -89,7 +89,7 @@ class Normalizer
 
 
   def initialize(lexicon, options = {})
-    @synonyms = TSV.new(lexicon, :flat)
+    @synonyms = TSV.open(lexicon, :flat)
 
     @index = CueIndex.new
     @index.load(lexicon, options[:max_candidates])

data/lib/rbbt/ner/rnorm/cue_index.rb

@@ -1,4 +1,4 @@
-require 'rbbt-util'
+require 'rbbt'
 require 'rbbt/util/misc'
 require 'rbbt/util/simpleDSL'
 
@@ -47,7 +47,7 @@ class CueIndex
   
   def load(file, max_candidates = 50)
     @indexes = Array.new(@rules.size){Hash.new}
-    data = TSV.new(file, :flat)
+    data = TSV.open(file, :flat)
     data.each{|code, values|
       values.each{|value|
         cues(value).each_with_index{|cue_list,i|

data/lib/rbbt/ner/{annotations.rb → segment.rb}

@@ -1,63 +1,59 @@
+require 'rbbt/annotations'
+require 'rbbt/fix_width_table'
+
 module Segment 
-  attr_accessor :offset, :docid
-
-  def self.included(base)
-    if base.instance_methods.include? "segment_types"
-      class << base
-        self.module_eval do 
-          define_method "extended" do |object|
-            object.segment_types ||= []
-            object.segment_types << self.to_s unless object.segment_types.include? self.to_s
-          end
-        end
-      end
-    end
-  end
+  extend Annotation
+  self.annotation :offset
 
-  def self.annotate(string, offset = nil, docid = nil)
-    string.extend Segment
-    string.offset = offset
-    string.docid = docid
-    string
-  end
+  #{{{ Ranges
 
-  def id
-    new = info.dup
-    Digest::MD5.hexdigest(Misc.hash2string(new) << self << (offset || 0).to_s)
+  def end
+    return nil if offset.nil?
+    offset + length - 1
   end
- 
-  SKIP = %w(docid offset)
-  def info
-    equal_ascii = "="[0]
-    info = {}
-    singleton_methods.select{|method| method[-1] == equal_ascii}.
-      collect{|m| m[(0..-2)]}.each{|m| info[m] = self.send(m) if self.respond_to?(m) and not SKIP.include? m.to_s}
-    info
-    info.delete_if{|k,v| v.nil?}
-    info
+
+  def range
+    raise "No offset specified" if offset.nil?
+    (offset..self.end)
   end
 
-  def self.load(text, start, eend, info, docid = nil)
-    string = text[start.to_i..eend.to_i] if start and eend
-    string ||= info[:literal]
-    string.extend Segment
+  def pull(offset)
+    if self.offset.nil? or offset.nil?
+      self.offset = nil
+    else
+      self.offset += offset 
+    end
 
-    # add types
-    types = info.delete("segment_types")|| info.delete(:segment_types) || []
-    types.each do |type| string.extend Misc.string2const(type) end
+    self
+  end
 
-    # set info data
-    info.each do |key,value|
-      string.send key + '=', value if string.respond_to? key.to_sym
+  def push(offset)
+    if self.offset.nil? or offset.nil?
+      self.offset = nil
+    else
+      self.offset -= offset 
     end
 
-    string.docid = docid
-    string.offset = start.to_i
+    self
+  end
 
-    string
+  def make_relative(segments)
+    segments.collect{|s| s.push offset}
   end
 
-  # {{{ Sorting and splitting
+  def range_in(container = nil)
+    raise "No offset specified" if offset.nil?
+    case
+    when (Segment === container and not container.offset.nil?)
+      ((offset - container.offset)..(self.end - container.offset))
+    when Integer === container
+      ((offset - container)..(self.end - container))
+    else
+      range
+    end
+  end
+
+  #{{{ Sorting
 
   def self.sort(segments, inline = true)
     if inline
@@ -102,13 +98,15 @@ module Segment
     sorted
   end
 
+  #{{{ Splitting
+
   def self.split(text, segments, skip_segments = false)
     sorted_segments = clean_sort segments
 
     chunks      = []
     segment_end = 0
     text_offset = 0
-    sorted_segments.reverse.each do |segment|
+    sorted_segments.each do |segment|
       return chunks if text.nil? or text.empty?
       next if segment.offset.nil?
       offset = segment.offset - text_offset
@@ -119,7 +117,7 @@ module Segment
         next
       when offset > 0 # Save pre
         chunk = text[0..offset - 1]
-        Segment.annotate(chunk, text_offset)
+        Segment.setup(chunk, text_offset)
         chunks << chunk
       end
 
@@ -127,7 +125,7 @@ module Segment
 
       if not skip_segments
         chunk = text[offset..segment_end]
-        Segment.annotate(chunk, text_offset + offset)
+        Segment.setup(chunk, text_offset + offset)
         chunks << chunk
       end
 
@@ -138,72 +136,29 @@ module Segment
 
     if not text.nil? and text.any?
       chunk = text.dup
-      Segment.annotate(chunk, text_offset)
+      Segment.setup(chunk, text_offset)
       chunks << chunk
     end
 
     chunks
   end
 
-  # {{{ Ranges and manipulation
 
-  def pull(offset)
-    if self.offset.nil? or offset.nil?
-      self.offset = nil
-    else
-      self.offset += offset 
-    end
-
-    self
-  end
-
-  def push(offset)
-    if self.offset.nil? or offset.nil?
-      self.offset = nil
-    else
-      self.offset -= offset 
-    end
-
-    self
-  end
-
-  def make_relative(segments)
-    segments.collect{|s| s.push offset}
-  end
-
-  def end
-    return nil if offset.nil?
-    offset + length - 1
-  end
-
-  def range
-    raise "No offset specified" if offset.nil?
-    (offset..self.end)
-  end
-
-  def range_in(container = nil)
-    raise "No offset specified" if offset.nil?
-    case
-    when (Segment === container and not container.offset.nil?)
-      ((offset - container.offset)..(self.end - container.offset))
-    when Integer === container
-      ((offset - container)..(self.end - container))
-    else
-      range
-    end
-  end
+  #{{{ Align
 
   def self.align(text, parts)
     pre_offset = 0
     parts.each do |part|
       offset = text.index part
       next if offset.nil?
-      Segment.annotate(part, pre_offset + offset)
+      Segment.setup(part, pre_offset + offset)
       pre_offset += offset + part.length - 1
       text = text[(offset + part.length - 1)..-1]
     end
   end
 
+  #{{{ Index
+
   class Index
     attr_accessor :index, :data
     def initialize(index, data)
@@ -216,12 +171,11 @@ module Segment
     end
   end
 
-  def self.index(segments, persistence_file = :memory)
-
+  def self.index(segments, persist_file = :memory)
     segments = segments.values.flatten if Hash === segments
 
     annotation_index = 
-      Persistence.persist("Index", :Index, :fwt, :persistence => (! (persistence_file.nil? or persistence_file == :memory)), :persistence_file => persistence_file, :range => true) do
+      Persist.persist("Segment_index", :fwt, :persist => (! (persist_file.nil? or persist_file == :memory)), :file => persist_file) do
 
         value_size = 0
         index_data = segments.collect{|segment| 
@@ -233,6 +187,7 @@ module Segment
 
         fwt = FixWidthTable.get :memory, value_size, true
         fwt.add_range index_data
+
         fwt
       end
 
@@ -241,14 +196,111 @@ module Segment
     Index.new annotation_index, data
   end
 
-end
+  #{{{ Save and load
+
+  def self.tsv_values_for_segment(segment, fields)
+    info = segment.info
+    values = []
+
+    fields.each do |field|
+      values << case
+      when field == "JSON"
+        info.to_json
+      when field == "literal"
+        segment.gsub(/\n|\t/, ' ')
+      when field == "Start"
+        segment.offset
+      when field == "End"
+        segment.end
+      else
+        info.delete(field.to_sym)
+      end
+    end
+
+    values
+  end
+
+  def self.load_tsv_values(text, values, fields)
+    info = {}
+    literal_pos = fields.index "literal"
+
+    object = if literal_pos.nil?
+               ""
+             else
+               v = values[literal_pos]
+               v = v.first if Array === v
+               v
+             end
+
+    fields.each_with_index do |field, i|
+      if field == "JSON"
+        JSON.parse(values[i]).each do |key, value|
+          info[key.to_sym] = value
+        end
+      else
+        info[field.to_sym] = values[i]
+      end
+    end
+
+    start = info.delete(:Start)
+    if not (start.nil? or ((Array === start or String === start) and start.empty?))
+      if Array === start
+        start = start.first
+      end
+      start = start.to_i
+      info[:offset] = start
+
+      eend = info.delete(:End)
+      if Array === eend
+        eend = eend.first
+      end
+      eend = eend.to_i
+
+      if object.empty?
+        object.replace text[start..eend]
+      end
+    end
+
+    info[:annotation_types] = [Segment] unless info.include? :annotation_types
 
-module Comment
-  include Segment
-  attr_accessor :comment
-  def self.annotate(text, comment = nil)
-    text.extend Comment
-    text.comment = (comment.nil? ? text : comment)
-    text
+    Annotated.load(object, info)
   end
+
+  def self.set_tsv_fields(fields, segments)
+    tsv_fields = []
+    add_types = ! (fields.delete(:no_types) || fields.delete("no_types") || fields.include?(:JSON) || fields.include?("JSON"))
+    literal = (fields.delete(:literal) || fields.delete("literal"))
+    tsv_fields << "Start" << "End"
+    tsv_fields << :annotation_types if add_types
+    tsv_fields << :literal if literal
+
+    if fields.any? and not (fields == [:all] or fields == ["all"])
+      tsv_fields.concat fields 
+    else
+      tsv_fields.concat segments.first.annotations if segments.any?
+    end
+    tsv_fields
+    tsv_fields.collect!{|f| f.to_s}
+    tsv_fields.delete "offset"
+    tsv_fields
+  end
+
+  def self.tsv(segments, *fields)
+    fields = set_tsv_fields fields, segments
+    tsv = TSV.setup({}, :key_field => "ID", :fields => fields, :type => :double)
+
+    segments.each do |segment|
+      tsv[segment.id] = self.tsv_values_for_segment(segment, fields)
+    end
+
+    tsv
+  end
+
+  def self.load_tsv(tsv)
+    tsv.collect do |id, values|
+      Annotated.load_tsv_values(id, values, tsv.fields)
+    end
+  end
+
 end
+