rbbt-text 1.1.8 → 1.1.9

data/lib/rbbt/corpus/document_repo.rb

@@ -1,137 +0,0 @@
-require 'rbbt/util/misc'
-require 'tokyocabinet'
-
-module DocumentRepo
-  class OpenError < StandardError;end
-  class KeyFormatError < StandardError;end
-
-  TC_CONNECTIONS = {}
-  def self.open_tokyocabinet(path, write)
-    write = true if not File.exists?(path)
-    flags = (write ? TokyoCabinet::BDB::OWRITER | TokyoCabinet::BDB::OCREAT : TokyoCabinet::BDB::OREADER)
-
-    FileUtils.mkdir_p File.dirname(path) unless File.exists?(File.dirname(path))
-
-    database = TC_CONNECTIONS[path] ||= TokyoCabinet::BDB.new
-    database.close
-
-    if !database.open(path, flags)
-      ecode = database.ecode
-      raise "Open error: #{database.errmsg(ecode)}. Trying to open file #{path}"
-    end
-
-    class << database
-      attr_accessor :writable, :persistence_path
-
-      def read
-        return if not @writable
-        self.close
-        if !self.open(@persistence_path, TokyoCabinet::BDB::OREADER)
-          ecode = self.ecode
-          raise "Open error: #{self.errmsg(ecode)}. Trying to open file #{@persistence_path}"
-        end
-        @writable = false
-        self
-      end
-
-      def write
-        return if @writable
-        self.close
-        if !self.open(@persistence_path, TokyoCabinet::BDB::OWRITER | TokyoCabinet::BDB::OCREAT)
-          ecode = self.ecode
-          raise "Open error: #{self.errmsg(ecode)}. Trying to open file #{@persistence_path}"
-        end
-        @writable = true
-        self
-      end
-
-      def write?
-        @writable
-      end
-
-      def collect
-        res = []
-        each do |key, value|
-          res << if block_given?
-                   yield key, value
-          else
-            [key, value]
-          end
-        end
-        res
-      end
-
-      def delete(key)
-        out(key)
-      end
-
-      def values_at(*keys)
-        keys.collect do |key|
-          self[key]
-        end
-      end
-
-      def merge!(hash)
-        hash.each do |key,values|
-          self[key] = values
-        end
-      end
-
-    end
-
-    database.persistence_path ||= path
-
-    database.extend DocumentRepo
-
-    database
-  end
-
-  def docid2fields(docid)
-    docid.split(":", -1).values_at 0,1,2,3
-  end
-
-  def fields2docid(namespace = nil, id = nil, type = nil, hash = nil)
-    [namespace, id, type, hash] * ":"
-  end
-
-  def docid(docid)
-    get(docid)
-  end
-
-  def add(text, namespace, id, type, hash)
-    read
-    write unless write?
-    docid = fields2docid(namespace, id, type, hash)
-    self[docid] = text unless self.include? docid
-    read
-    docid
-  end
-
-  def find(namespace=nil, id = nil, type = nil, hash = nil)
-    case
-    when namespace.nil?
-      self.keys
-    when id.nil?
-      range_start = [namespace] * ":" + ':'
-      range_end   = [namespace] * ":" + ';'
-      self.range(range_start, true, range_end, false)
-    when (type and hash)
-      [[namespace, id, type, hash] * ":"]
-    when hash
-      [[namespace, id, "", hash] * ":"]
-    when type
-      range_start = [namespace, id, type] * ":" + ':'
-      range_end   = [namespace, id, type] * ":" + ';'
-      self.range(range_start, true, range_end, false)
-    else
-      range_start = [namespace, id] * ":" + ':'
-      range_end   = [namespace, id] * ":" + ';'
-      self.range(range_start, true, range_end, false)
-    end
-  end
-
-  def find_docid(docid)
-    find(*docid2fields(docid))
-  end
-
-end

data/lib/rbbt/corpus/sources/pubmed.rb

@@ -1,27 +0,0 @@
-require 'rbbt/sources/pubmed'
-
-class Corpus
-
-  NAMESPACES = {} unless defined? NAMESPACES
-  NAMESPACES[:pubmed] = :add_pmid
-
-  def add_pmid(pmid, type = nil)
-    pmids = Array === pmid ? pmid : [pmid]
-    type = nil if String === type and type.empty?
-
-    PubMed.get_article(pmids).collect do |pmid, article|
-      add_document(article.title, :pubmed, pmid, :title)
-      if (type.nil? and article.pdf_url.nil?) or (not type.nil? and type.to_sym === :abstract)
-        add_document(article.abstract || "", :pubmed, pmid, :abstract)
-      else
-        raise "No FullText available for #{ pmid }" if article.pdf_url.nil?
-        add_document(article.full_text, :pubmed, pmid, :fulltext)
-      end
-    end
-  end
-  
-  def add_pubmed_query(query, max, type = nil)
-    pmids = PubMed.query(query, max)
-    add_pmid(pmids, type)
-  end
-end

data/lib/rbbt/entity/document.rb

@@ -1,75 +0,0 @@
-require 'rbbt/entity'
-require 'rbbt/ner/segment/docid'
-
-module Document
-  extend Entity
-
-  class << self
-    attr_accessor :corpus
-  end
-
-  property :docid => :single2array do |*args|
-    @docid ||= if self =~ /^text:/
-                 self
-               else
-                 ["text", Misc.digest(self.inspect)] * ":"
-               end
-    @docid
-  end
-
-  property :annotation_id => :both do |*args|
-    if Array === self
-      Misc.hash2md5(info.merge(:self => self))
-    else
-      docid(*args)
-    end
-  end
-
-  property :_get_text => :single do
-    self
-  end
-
-  property :text => :array2single do |*args|
-    article_text = {}
-    missing = []
-
-    if Document.corpus.nil?
-      self._get_text(*args)
-    else
-
-      Document.corpus.read_and_close do
-        self.each do |doc|
-          id = doc.docid(*args)
-          case
-          when Document.corpus.include?(doc) 
-            article_text[doc] =  Document.corpus[doc] 
-          when Document.corpus.include?(id) 
-            article_text[doc] =  Document.corpus[id] 
-          else
-            missing << doc
-          end
-
-        end
-      end
-
-      if missing.any?
-        missing.first.annotate missing
-        missing_text = Misc.process_to_hash(missing){|list| list._get_text(*args)}
-
-        Misc.lock(Document.corpus.respond_to?(:persistence_path) ? Document.corpus.persistence_path : nil) do
-          Document.corpus.write_and_close do
-
-            missing_text.each do |doc, doc_text|
-              doc = self.annotate doc.dup
-              Document.corpus[doc.docid(*args)] = doc_text
-              article_text[doc] = doc_text
-            end
-          end
-        end
-      end
-
-      article_text.values_at *self
-    end
-  end
-end
-