rbbt-text 1.2.0 → 1.3.4

data/lib/rbbt/text/corpus/document_repo.rb

@@ -1,68 +0,0 @@
-require 'rbbt/util/misc'
-require 'tokyocabinet'
-
-class Corpus
-  module DocumentRepo
-    class OpenError < StandardError;end
-    class KeyFormatError < StandardError;end
-
-    TC_CONNECTIONS = {}
-    def self.open_tokyocabinet(path, write)
-      database = Persist.open_tokyocabinet(path, write, :single, TokyoCabinet::BDB)
-      database.extend DocumentRepo
-      database
-    end
-
-    def docid2fields(docid)
-      docid.split(":", -1).values_at 0,1,2,3
-    end
-
-    def fields2docid(namespace = nil, id = nil, type = nil, hash = nil)
-      [namespace, id, type, hash] * ":"
-    end
-
-    def docid(docid)
-      get(docid)
-    end
-
-    def add(text, namespace, id, type, hash)
-      docid = fields2docid(namespace, id, type, hash)
-
-      return docid if self.include?(docid)
-
-      write_and_close do
-        self[docid] = text
-      end
-
-      docid
-    end
-
-    def find(namespace=nil, id = nil, type = nil, hash = nil)
-      case
-      when namespace.nil?
-        self.keys
-      when id.nil?
-        range_start = [namespace] * ":" + ':'
-        range_end   = [namespace] * ":" + ';'
-        self.range(range_start, true, range_end, false)
-      when (type and hash)
-        [[namespace, id, type, hash] * ":"]
-      when hash
-        [[namespace, id, "", hash] * ":"]
-      when type
-        range_start = [namespace, id, type] * ":" + ':'
-        range_end   = [namespace, id, type] * ":" + ';'
-        self.range(range_start, true, range_end, false)
-      else
-        range_start = [namespace, id] * ":" + ':'
-        range_end   = [namespace, id] * ":" + ';'
-        self.range(range_start, true, range_end, false)
-      end
-    end
-
-    def find_docid(docid)
-      find(*docid2fields(docid))
-    end
-
-  end
-end

data/lib/rbbt/text/corpus/sources/pmid.rb

@@ -1,34 +0,0 @@
-require 'rbbt/sources/pubmed'
-
-class Corpus
-
-  NAMESPACES = {} unless defined? NAMESPACES
-  NAMESPACES[:pubmed] = :add_pmid
-
-  def add_pmid(pmid, type = nil)
-    pmids = Array === pmid ? pmid : [pmid]
-    type = nil if String === type and type.empty?
-
-    PubMed.get_article(pmids).collect do |pmid, article|
-      Log.debug "Loading pmid #{pmid}"
-      if type.nil? || type.to_sym == :abstract
-        add_document(article.abstract || "", :PMID, pmid, :abstract)
-      elsif type.to_sym == :title
-        add_document(article.title, :PMID, pmid, :title)
-      else
-        raise "No FullText available for #{ pmid }" if article.full_text.nil?
-        add_document(article.full_text, :PMID, pmid, :fulltext)
-      end
-    end
-  end
-  
-  def add_pubmed_query(query, max = 3000, type = nil)
-    pmids = PubMed.query(query, max)
-    add_pmid(pmids, type)
-  end
-
-  self.claim "PMID" do |id, type|
-    Log.debug "Claiming #{id}"
-    self.add_pmid(id, type)
-  end
-end

data/lib/rbbt/text/document.rb

@@ -1,39 +0,0 @@
-require 'rbbt-util'
-require 'rbbt/entity'
-
-require 'rbbt/text/corpus'
-
-module Document
-  extend Entity
-  class << self
-    attr_accessor :corpus
-  end
-
-  property :document => :single do
-    Document.corpus.docid(self)
-  end
-
-  property :type => :single do |type|
-    self.annotate((self.split(":").values_at(0,1)) * ":" + ":" + type.to_s)
-  end
-
-  property :title => :single do
-    type(:title).text
-  end
-
-  property :full_text => :single do
-    type(:full_text).text
-  end
-
-  property :abstract => :single do
-    type(:abstract).text
-  end
-
-  property :text => :single do
-    document.text
-  end
-
-  property :entities => :single do |type,*args|
-    document.method(type).call *args
-  end
-end

data/lib/rbbt/text/segment.rb

@@ -1,363 +0,0 @@
-require 'rbbt/annotations'
-require 'rbbt/fix_width_table'
-
-module Segment 
-  extend Annotation
-  self.annotation :offset, :docid
-
-  def segment_length
-    begin
-      super()
-    rescue
-      self.length
-    end
-  end
-
-  #{{{ Ranges
-
-  def end
-    return nil if offset.nil?
-    offset.to_i + segment_length - 1
-  end
-
-  def range
-    raise "No offset specified" if offset.nil?
-    (offset.to_i..self.end)
-  end
-
-  def pull(offset)
-    if self.offset.nil? or offset.nil?
-      self.offset = nil
-    else
-      self.offset += offset 
-    end
-
-    self
-  end
-
-  def push(offset)
-    if self.offset.nil? or offset.nil?
-      self.offset = nil
-    else
-      self.offset -= offset 
-    end
-
-    self
-  end
-
-  def make_relative(segments, &block)
-    if block_given?
-      segments.each{|s| s.push offset}
-      yield(segments)
-      segments.each{|s| s.pull offset}
-    else
-      segments.each{|s| s.push offset}
-    end
-  end
-
-  def range_in(container = nil)
-    raise "No offset specified" if offset.nil?
-    case
-    when (Segment === container and not container.offset.nil?)
-      ((offset - container.offset)..(self.end - container.offset))
-    when Integer === container
-      ((offset - container)..(self.end - container))
-    else
-      range
-    end
-  end
-
-  def includes?(segment)
-    (segment.offset.to_i >= self.offset.to_i) and
-    (segment.offset.to_i + segment.segment_length.to_i <= self.offset.to_i + self.segment_length.to_i)
-  end
-
-  def overlaps?(segment)
-    segment.offset.to_i >= self.offset.to_i && segment.offset.to_i <= self.end || 
-    self.offset.to_i >= segment.offset.to_i && self.offset.to_i <= segment.end
-  end
-
-  def overlaps(segments)
-    segments.select{|s| self.overlaps?(s)}
-  end
-
-
-  def self.collisions(main, secondary)
-    secondary.select do |ss|
-      main.select{|ms| ms.overlaps? ss }.any?
-    end
-  end
-
-  #{{{ Sorting
-
-  def self.sort(segments, inline = true)
-    if inline
-      segments.sort do |a,b| 
-        case
-        when ((a.nil? and b.nil?) or (a.offset.nil? and b.offset.nil?))
-          0
-        when (a.nil? or a.offset.nil?)
-          -1
-        when (b.nil? or b.offset.nil?)
-          +1
-        when (not a.range.include? b.offset.to_i and not b.range.include? a.offset.to_i)
-          a.offset.to_i <=> b.offset.to_i
-        else
-          a.segment_length <=> b.segment_length
-        end
-      end
-    else
-      segments.sort_by do |segment| segment.offset.to_i || 0 end.reverse
-    end
-  end
-
-  def self.overlaps(sorted_segments)
-    last = nil
-    overlaped = []
-
-    sorted_segments.reverse.each do |segment| 
-      overlaped << segment if (not last.nil?) and segment.range.end > last 
-      last = segment.range.begin
-    end
-
-    overlaped
-  end
-
-  def self.clean_sort(segments)
-    sorted = sort(segments).reject{|s| s.offset.nil?}
-    overlaps = overlaps(sorted)
-    overlaps.each do |s|
-      sorted.delete s
-    end
-
-    sorted
-  end
-
-  #{{{ Splitting
-
-  def self.split(text, segments, skip_segments = false)
-    sorted_segments = clean_sort segments
-
-    chunks      = []
-    segment_end = 0
-    text_offset = 0
-    sorted_segments.each do |segment|
-      return chunks if text.nil? or text.empty?
-      next if segment.offset.nil?
-      offset = segment.offset - text_offset
-
-      # Consider segment offset. Save pre, or skip if overlap
-      case
-      when offset < 0 # Overlap, skip
-        next
-      when offset > 0 # Save pre
-        chunk = text[0..offset - 1]
-        Segment.setup(chunk, text_offset)
-        chunks << chunk
-      end
-
-      segment_end = offset + segment.segment_length - 1
-
-      if not skip_segments
-        chunk = text[offset..segment_end]
-        Segment.setup(chunk, text_offset + offset)
-        chunks << chunk
-      end
-
-      text_offset += segment_end + 1
-      text = text[segment_end + 1..-1]
-
-    end
-
-    if not text.nil? and not text.empty?
-      chunk = text.dup
-      Segment.setup(chunk, text_offset)
-      chunks << chunk
-    end
-
-    chunks
-  end
-
-
-  #{{{ Align
-
-  def self.align(text, parts)
-    pre_offset = 0
-    parts.each do |part|
-      offset = text.index part
-      next if offset.nil?
-      Segment.setup(part, pre_offset + offset)
-      pre_offset += offset + part.segment_length - 1
-      text = text[(offset + part.segment_length - 1)..-1]
-    end
-  end
-
-  #{{{ Index
-
-  class Index
-    attr_accessor :index, :data
-    def initialize(index, data)
-      @index = index
-      @data = data
-    end
-
-    def [](pos)
-      index[pos].collect{|id| data[id]}
-    end
-  end
-
-  def self.index(segments, persist_file = :memory)
-    segments = segments.values.flatten if Hash === segments
-
-    annotation_index = 
-      Persist.persist("Segment_index", :fwt, :persist => (! (persist_file.nil? or persist_file == :memory)), :file => persist_file) do
-
-        value_size = 0
-        index_data = segments.collect{|segment| 
-          next if segment.offset.nil?
-          range = segment.range
-          value_size = [segment.id.length, value_size].max
-          [segment.id, [range.begin, range.end]]
-        }.compact
-
-        fwt = FixWidthTable.get :memory, value_size, true
-        fwt.add_range index_data
-
-        fwt
-      end
-
-    data = {}
-    segments.each do |segment| data[segment.id] = segment end
-    Index.new annotation_index, data
-  end
-
-  #{{{ Save and load
-
-  def self.tsv_values_for_segment(segment, fields)
-    info = segment.info
-    values = []
-
-    fields.each do |field|
-      values << case
-      when field == "JSON"
-        info.to_json
-      when field == "literal"
-        segment.gsub(/\n|\t/, ' ')
-      when field == "Start"
-        segment.offset
-      when field == "End"
-        segment.end
-      else
-        info.delete(field.to_sym)
-      end
-    end
-
-    values
-  end
-
-  def self.load_tsv_values(text, values, fields)
-    info = {}
-    literal_pos = fields.index "literal"
-
-    object = if literal_pos.nil?
-               ""
-             else
-               v = values[literal_pos]
-               v = v.first if Array === v
-               v
-             end
-
-    fields.each_with_index do |field, i|
-      if field == "JSON"
-        JSON.parse(values[i]).each do |key, value|
-          info[key.to_sym] = value
-        end
-      else
-        info[field.to_sym] = values[i]
-      end
-    end
-
-    start = info.delete(:Start)
-    if not (start.nil? or ((Array === start or String === start) and start.empty?))
-      if Array === start
-        start = start.first
-      end
-      start = start.to_i
-      info[:offset] = start
-
-      eend = info.delete(:End)
-      if Array === eend
-        eend = eend.first
-      end
-      eend = eend.to_i
-
-      if object.empty?
-        object.replace text[start..eend]
-      end
-    end
-
-    info[:annotation_types] = [Segment] unless info.include? :annotation_types
-
-    Annotated.load_entity(object, info)
-  end
-
-  def self.set_tsv_fields(fields, segments)
-    tsv_fields = []
-    add_types = ! (fields.delete(:no_types) || fields.delete("no_types") || fields.include?(:JSON) || fields.include?("JSON"))
-    literal = (fields.delete(:literal) || fields.delete("literal"))
-    tsv_fields << "Start" << "End"
-    tsv_fields << :annotation_types if add_types
-    tsv_fields << :literal if literal
-
-    if fields.any? and not (fields == [:all] or fields == ["all"])
-      tsv_fields.concat fields 
-    else
-      tsv_fields.concat segments.first.annotations if segments.any?
-    end
-    tsv_fields
-    tsv_fields.collect!{|f| f.to_s}
-    tsv_fields.delete "offset"
-    tsv_fields
-  end
-
-  def self.tsv(segments, *fields)
-    fields = set_tsv_fields fields, segments
-    tsv = TSV.setup({}, :key_field => "ID", :fields => fields, :type => :double)
-
-    segments.each do |segment|
-      tsv[segment.segment_id] = self.tsv_values_for_segment(segment, fields)
-    end
-
-    tsv
-  end
-
-  def self.load_tsv(tsv)
-    fields = tsv.fields
-    tsv.with_unnamed do
-      tsv.collect do |id, values|
-        Annotated.load_tsv_values(id, values, fields)
-      end
-    end
-  end
-
-  def ansi(color)
-    Log.color color, self
-  end
-
-  def locus
-    [offset, self.end] * ".."
-  end
-
-  def segment_id
-    if self.respond_to?(:docid)
-      [docid, locus, Misc.obj2digest(info)] * ":"
-    else
-      Misc.obj2digest(info)
-    end
-  end
-
-  #def ==(other)
-  #  self.text == other.text
-  #end
-end
-