rbbt-sent 1.0.0 → 1.0.1

data/LICENSE

@@ -11,6 +11,16 @@ the following conditions:
 The above copyright notice and this permission notice shall be
 included in all copies or substantial portions of the Software.
 
+Any academic publication involving the use of this software in any way 
+should include a reference to the following publication:
+
+Vazquez, M. and Carmona-Saez, P. and Nogales-Cadenas, R. and Chagoyen, M. and
+Tirado, F. and Carazo, J.M. and Pascual-Montano, A.: "SENT: semantic features in
+text". Nucleic Acids Research 2009, doi:10.1093/nar/gkp392
+
+Any comercial use of the software must be notified for aproval to the
+holder of the copyright, listed above.
+
 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND

data/bin/sent_config

@@ -13,10 +13,10 @@ $USAGE =<<EOT
   actions:
     * configure:   Set paths for data, work, and tmp directories
 
-    * install:
+    * prepare:
         * analysis: Install configuration to perform analysis
     
-    * update:
+    * install:
         * metadocs: Generate metadocs for all organisms
     
     * init:
@@ -48,14 +48,14 @@ class Controller < SimpleConsole::Controller
     render :action => :usage
   end
 
-  def install
+  def prepare
     raise "Run #{__FILE__} configure first to configure sent" if $noconfig
 
     case params[:id]
     when "analysis"
       @tasks = %w(analysis)
     when nil
-      redirect_to :action => :help, :id => :install
+      redirect_to :action => :help, :id => :prepare
     else
       @tasks = [params[:id]]
     end
@@ -65,14 +65,14 @@ class Controller < SimpleConsole::Controller
 
   end
 
-  def update
+  def install
     raise "Run #{__FILE__} configure first to configure rbbt" if $noconfig
 
     case params[:id]
     when "metadocs"
       @location = File.join(Sent.datadir,'analysis')
     else
-      redirect_to :action => :help, :id => :update
+      redirect_to :action => :help, :id => :install
     end
 
     $force = true if params[:force]
@@ -90,7 +90,7 @@ class View < SimpleConsole::View
     puts $USAGE
   end
 
-  def install
+  def prepare
     require 'rake'
     load File.join(Sent.rootdir, 'tasks/install.rake')
 
@@ -100,7 +100,7 @@ class View < SimpleConsole::View
     }
   end
 
-  def update
+  def install
     require 'rake'
 
     puts "Changing directory to #{@location}"
@@ -108,7 +108,7 @@ class View < SimpleConsole::View
 
     load "./Rakefile"
 
-    Rake::Task['update'].invoke
+    Rake::Task['default'].invoke
   end
 
 

data/install_scripts/analysis/Rakefile

@@ -1,139 +1,97 @@
-require 'sent'
+require 'rake_pipeline'
 require 'sent/main'
 require 'rbbt/sources/organism'
 require 'progress-monitor'
 
+include Rake::Pipeline
 
-$list = ENV['list']
-$kstart = ENV['kstart'] || ENV['k']
-$kend = ENV['kend'] || $kstart
 
-rule (/summary\/(.*)/) => lambda {|n| n.sub(/summary/,'NMF') } do |t|
-  nmf = t.name.sub(/summary/,'NMF')
+def org_description(org, associations)
+  name = Organism.name(org)
+  supported_ids = Organism.supported_ids(org, :examples => true)
 
-  Sent.analyze(nmf, t.name)
-  FileUtils.touch t.name
+  <<-EOT
+Name: #{ name }
+Organism: #{ name }
+Description: #{associations.values.flatten.length} associations for #{associations.keys.length} genes and #{associations.values.flatten.uniq.length} articles
+ID Format: #{supported_ids.collect{|p| "#{ p[0] } (#{ p[1] })"}.join(", ")}
+  EOT
 end
 
-rule (/NMF\/(.*)/) => lambda {|n| n.sub(/NMF/,'matrices') } do |t|
-  matrix = t.name.sub(/NMF/,'matrices')
-
-  k = $kstart
-  if $kstart < $kend
-    best = 0
-    ccc = Sent.CCC(matrix, $kstart, $kend)
-    ccc.each_with_index{|v,i|
-      if v.to_i > best
-        k = $kstart.to_i + i
-      end
-    }
-  end
 
-  Sent.NMF(matrix, t.name, k.to_i, 10)
-  FileUtils.touch t.name
-end
-
-rule (/matrices\/(.*)/) => lambda {|n| n.sub(/matrices/,'metadocs') } do |t|
-  metadocs = t.name.sub(/matrices/,'metadocs')
-  
-  list = nil
-  list = Open.read($list).collect{|l| l.chomp} if $list
+desc "Find gene mentions in text"
+step_def :mentions do
   
-  Sent.matrix(metadocs, t.name, list)
+  associations = Sent.mentions(job_name)
+
+  associations.collect {|code, pmids| "%s\t%s" % [code, pmids.uniq.join('|')] } * "\n"
 end
 
-rule (/metadocs\/(.*)/) => lambda {|n| n.sub(/metadocs/,'associations') } do |t|
-  assocfile = t.name.sub(/metadocs/,'associations')
+desc "Join gene mention associations, entrez GeneRif associations, and GO associations" 
+step_def :associations do |t|
+  associations = Open.to_hash(StringIO.new(input), :flatten => true)
 
-  Sent.metadocs(assocfile, t.name)
-end
+  associations.merge!(Organism.gene_literature(job_name))
+  associations.merge!(Organism.gene_literature_go(job_name))
 
-rule(/associations\/(.*)_text/) do |t|
-  org = File.basename(t.name).sub(/_text/,'')
-
-  ner   = Organism.ner(org, :rner)
-  norm  = Organism.norm(org)
-  pmids = Organism.literature(org)
-
-  fout = File.open(t.name, 'w')
-  chunks = pmids.chunk(100)
-
-  Progress.monitor("Finding gene-article associations in text", 1000)
-  chunks.each{|chunk|
-    PubMed.get_article(chunk).each{|pmid, article|
-      text = article.text
-      
-      mentions = ner.extract(text)
-
-      Progress.monitor("Resolving mentions", 1000)
-      codes = mentions.collect{|mention| 
-        matches = norm.match(mention)
-        norm.select(matches,mention,text)
-      }.flatten.uniq.sort
-      
-      codes.each{|code|
-        fout.puts "#{ code }\t#{pmid}"
-      }
-
-    }
-  }
-  fout.close
-end
+  File.open(t.name + '.description', 'w') do |f| f.puts org_description(job_name, associations) end
 
-rule (/associations\/(.*)/) => lambda{|n| n + '_text'} do |t|
-  org = File.basename(t.name)
+  associations.collect {|code, pmids| "%s\t%s" % [code, pmids.uniq.join('|')] } * "\n"
+end
 
-  fout = File.open(t.name, 'w')
-  fout.write Open.read(t.name + '_text')
+desc "Build dictionary"
+step_def :dictionary do |t|
+  associations = Open.to_hash(StringIO.new(input), :flatten => true)
 
-  associations = Organism.gene_literature(org)
-  associations.each{|gene, pmids|
-    pmids.each{|pmid|
-      fout.puts "#{ gene }\t#{pmid}"
-    }
-  }
+  Progress.monitor("Building dictionary for #{job_name}")
+  term_weights, stems = Sent.dictionary(associations)
 
-  associations = Organism.gene_literature_go(org)
-  associations.each{|gene, pmids|
-    pmids.each{|pmid|
-      fout.puts "#{ gene }\t#{pmid}"
-    }
-  }
-  fout.close
+  File.open(t.name + '.stems', 'w') do |f| f.puts stems.collect{|stem, words| "%s\t%s" % [stem, words.join("|")] } end
 
-  name = Organism.name(org)
-  supported_ids = Organism.supported_ids(org, :examples => true)
-  associations = Open.to_hash(t.name, :flatten => true)
+  term_weights.collect {|term, weight| "%s\t%s" % [term, weight] } * "\n"
+end
 
-  description =<<-EOT
-Name: #{ name }
-Organism: #{ name }
-Description: #{associations.values.flatten.length} associations for #{associations.keys.length} genes and #{associations.values.flatten.uniq.length} articles
-ID Format: #{supported_ids.collect{|p| "#{ p[0] } (#{ p[1] })"}.join(", ")}
-  EOT
+desc "Build metadocument"
+step_def :metadoc do
+  terms = input.collect{|l| l.split("\t").first}
+  associations = Open.to_hash(StringIO.new(input(:associations)), :flatten => true)
 
-  Open.write(t.name + '.description', description)
+  Progress.monitor("Building metadoc for #{job_name}")
+  Sent.metadoc(associations, terms)
 end
 
-task 'clean' do
-  FileUtils.rm Dir.glob("associations/*")
+desc "Selecting gene metadocs"
+step_def :matrix, [] do |t|
+  $org          ||= ENV['organism']
+  $genes        ||= STDIN.read.split(/\n/)
+  $metadoc_dir  ||= step_dir(:metadoc)
+
+  metadoc_file = File.join($metadoc_dir, $org)
+
+  Sent.matrix(metadoc_file, $genes)
 end
 
-task 'all' do
-  Organism.all.each{|org|
-    `rake metadocs/#{ org }`
-  }
+desc "Performing NMF factorization"
+step_def :NMF do |t|
+  $factors   ||= ENV['factors']
+  Sent.NMF(input, t.name, $factors.to_i)
 end
 
-task 'update' do
-  if $org
-    FileUtils.rm Dir.glob("**/#{$org}.*") if $force
-    Rake::Task["metadocs/#{$org}"].invoke
-  else
-    Rake::Task['clean'].invoke if $force
-    Rake::Task['all'].invoke
-  end
+
+desc "Analyze results"
+step_def :summary do 
+  $factors   ||= ENV['factors']
+  Sent.analyze(input_filename, output_filename, $factors)
 end
 
 
+task 'all' do
+
+  Progress.monitor("Processing organisms", :announcement => Proc.new{|org| "Organism #{ org }" }, :stack_depth => 0)
+  Organism.all.each do |org|
+    Rake::Task[File.join('metadoc',org)].invoke
+  end
+
+end
 
+task 'default' => 'all'

data/lib/sent/main.rb

@@ -1,3 +1,4 @@
+require 'sent'
 require 'rbbt/sources/pubmed'
 require 'rbbt/util/misc'
 require 'rbbt/util/open'
@@ -9,6 +10,7 @@ require 'stemmer'
 require 'open4'
 require 'progress-monitor'
 require 'yaml'
+require 'base64'
 
 # Produce Stem lists, used by the Web Service
 class String
@@ -41,6 +43,16 @@ class String
   reset_stem_list
 end
 
+def try3times
+  try = 0
+  begin
+    yield
+  rescue
+    try += 1
+    retry if try <= 3
+  end
+end
+
 
 module Sent
 
@@ -55,7 +67,8 @@ module Sent
     stdin.close
 
     Process.wait pid
-    raise ProcessAborted, "Error in R process" if $?.exitstatus != 0
+
+    raise Sent::ProcessAbortedError, "Error in R process: #{stdout.read + stderr.read}" if $?.exitstatus != 0
     result = stdout.read + stderr.read
     stdout.close
     stderr.close
@@ -64,71 +77,99 @@ module Sent
     result
   end
 
-  def self.metadocs(assocfile, output, low=0.001, hi=0.65, max=3000)
-    
-    associations = Open.to_hash(assocfile, :flatten => true, :sep => "\t|,")
+  def self.mentions(org)
+    ner   = Organism.ner(org, :rner)
+    norm  = Organism.norm(org)
+    pmids = Organism.literature(org)
+
+    chunks = pmids.chunk(100)
+
+    pmids = {}
+    Progress.monitor("Finding gene-article associations in text", :step => 1000)
+    chunks.each{|chunk|
+      try3times do
+        PubMed.get_article(chunk).each{|pmid, article|
+          text = article.text
+
+          mentions = ner.extract(text)
+
+          Progress.monitor("Resolving mentions", :step => 1000)
+          codes = mentions.collect{|mention| 
+            matches = norm.match(mention)
+            norm.select(matches,mention,text)
+          }.flatten.uniq.sort
+
+          codes.each{|code|
+            pmids[code] ||= []
+            pmids[code] << pmid
+          }
+        }
+      end
+    }
+    pmids
+  end
 
+  def self.dictionary(associations, options = {})
+    dict_options = {:low => 0.001, :hi => 0.65, :limit => 3000}.merge options
     dict = Dictionary::TF_IDF.new
 
     String.reset_stem_list
 
-    Progress.monitor("Building Dictionary for #{File.basename(output)}", 1000) 
-    associations.each{|gene, pmids|
-      text = PubMed.get_article(pmids).collect{|p| p[1].text}.join("\n")
-      dict.add(BagOfWords.count(text.bigrams))
-    }
-
-    # At least 3 genes must have a word to be chosen
-    hard_min = 3 * 100 / associations.keys.length
-    hi = hard_min if hi < hard_min
-    
-    d =  dict.weights(:low => low, :hi => hi, :limit => max)
-    Open.write(output + '.dict', d.sort.collect{|p| p.join("\t")}.join("\n"))
-    terms = d.keys.sort
+    associations.each do |gene, pmids|
+      try3times do
+        text = PubMed.get_article(pmids).collect{|pmid, article| article.text }.join("\n")
+        dict.add(BagOfWords.count(text.bigrams))
+      end
+    end
 
-    fout = File.open(output, 'w')
-    fout.puts("\t" + terms.join("\t"))
+    term_weigths = dict.weights(dict_options)
+    terms        = term_weigths.keys.sort
 
-    Progress.monitor("Building Metadoc for #{File.basename(output)}", 1000) 
-    associations.each{|gene, pmids|
-      text = PubMed.get_article(pmids).collect{|p| p[1].text}.join("\n")
-      fout.puts(([gene] + BagOfWords.features(text, terms)).join("\t"))
-    }
-    fout.close
+    stems        = String.stem_list(terms.collect{|p| p.split(/ /) }.flatten.uniq)
 
-    Open.write(output + '.stems', String.stem_list(terms.collect{|p| p.split(/ /)}.flatten.uniq).collect{|k,v| "#{ k }\t#{v.join("\t")}"}.join("\n"))
+    [term_weigths, stems]
+  end
+ 
+
+  def self.metadoc(associations, terms)
+    "%s\t%s\n" % ["Entity", terms.sort * "\t"] +
+    associations.collect do |gene, pmids|
+      try3times do
+        text = PubMed.get_article(pmids).collect{|pmid, article| article.text }.join("\n")
+        "%s\t%s" % [gene, BagOfWords.features(text, terms.sort).join("\t")]
+      end
+    end * "\n"
   end
 
-  def self.matrix(metadocs, output, list=nil)
-    list ||= []
-      
-    if list.empty?
-      FileUtils.cp metadocs, output
-    else
-      `head -n 1 #{ metadocs } > #{ output }`
-      `grep '^\\(#{list.join('\\|')}\\)[[:space:]]' #{ metadocs } >> #{output}`
-      raise Sent::NoGenesError, "No Genes Matched" if $? != 0
+  def self.matrix(metadoc_file, genes)
+
+    matrix = ""
+    File.open(metadoc_file) do |f|
+      matrix += f.gets
+      f.read.each_line do |line|
+        gene = line.match(/^(.*?)\t/)[1]
+        matrix += line if genes.include? gene
+      end
     end
 
-    dict = metadocs + '.dict'
-    run_R("SENT.prepare.matrix('#{ output }', '#{ output }', '#{metadocs + '.dict'}')")
+    matrix
   end
 
-
   @@bionmf_wsdl = "http://bionmf.dacya.ucm.es/WebService/BioNMFWS.wsdl"   
   def self.NMF(matrix, out, k, executions = 10)
     driver = SOAP::WSDLDriverFactory.new( @@bionmf_wsdl).create_rpc_driver
 
     # Upload matrix
     nmf_matrix = driver.upload_matrix(
-       File.open(matrix).read, # matrix
-       false, # binary 
-       true,  # column labels 
-       true,  # row labels
-       true,  # transpose
-       "No",  # positive
-       "No",  # normalization 
+       matrix,   # matrix
+       false,    # binary 
+       true,     # column labels 
+       true,     # row labels
+       true,     # transpose
+       "No",     # positive
+       "No",     # normalization 
        "matrix") # Suggested name
+
     # Send several executions in parallel
     while !driver.done(nmf_matrix)
       sleep(5)
@@ -149,14 +190,14 @@ module Sent
           job_id = driver.standardNMF(
             nmf_matrix, # Matrix job
             "Standard", # Algorithm
-            k, # Factor Start
-            k, # Factor End
-            1, # Runs
-            2000, # Iterations
-            40,   # Stop criteria
-            0,    # Not used (nsnmf smoothness)
-            false, # extra info
-            '')    # Suggested name
+            k,          # Factor Start
+            k,          # Factor End
+            1,          # Runs
+            2000,       # Iterations
+            40,         # Stop criteria
+            0,          # Not used (nsnmf smoothness)
+            false,      # extra info
+            '')         # Suggested name
 
           while !driver.done(job_id)
             sleep(5)
@@ -168,12 +209,15 @@ module Sent
           end
 
           results =  driver.results(job_id)
-          fw = File.open(out + ".matrix_w.#{num}",'w')
-          fw.write(driver.result(results[0]).sub(/\t(.*)\t$/,'\1'))
-          fw.close
-          fh = File.open(out + ".matrix_h.#{num}",'w')
-          fh.write(driver.result(results[1]).sub(/\t(.*)\t$/,'\1'))
-          fh.close
+          
+          File.open(out + ".matrix_w.#{num}",'w') do |f| 
+            f.write Base64.decode64 driver.result(results[0]) #.sub(/\t(.*)\t$/,'\1')
+          end
+
+          File.open(out + ".matrix_h.#{num}",'w') do |f|
+            f.write Base64.decode64 driver.result(results[1]) #.sub(/\t(.*)\t$/,'\1')
+          end
+
           driver.clean(job_id)
         rescue Sent::ProcessAbortedError
           puts "Process aborted for #{ num }"
@@ -228,7 +272,7 @@ module Sent
     if error
       raise Exception, "Error in NMF:\n" + error
     end
-
+    
     run_R("SENT.join.results('#{ out }')")
 
     FileUtils.rm Dir.glob(out + '.matrix_*.*')
@@ -275,7 +319,7 @@ module Sent
   def self.analyze(prefix, output, clusters = nil, num_words = 15)
 
     FileUtils.rm Dir.glob(output + '*.words') + Dir.glob(output + '*.genes')
-    run_R("SENT.analyze('#{ prefix }', '#{ output }', '#{clusters}', '#{num_words}')")
+    run_R("SENT.analyze('#{ prefix }', '#{ output }', #{clusters || 'NULL'}, #{num_words})")
     words = Dir.glob(output + '*.words').sort.collect{|f| Open.read(f).split(/\n/)}
     genes = Dir.glob(output + '*.genes').sort.collect{|f| Open.read(f).split(/\n/)}
 
@@ -284,7 +328,7 @@ module Sent
       groups << {:words => p[0], :genes => p[1]}
     }
 
-    Open.write(output + '.summary', groups.to_yaml)
+    groups
   end
 
   def self.literature_index(pmids, outfile)

metadata

@@ -1,7 +1,12 @@
 --- !ruby/object:Gem::Specification 
 name: rbbt-sent
 version: !ruby/object:Gem::Version 
-  version: 1.0.0
+  prerelease: false
+  segments: 
+  - 1
+  - 0
+  - 1
+  version: 1.0.1
 platform: ruby
 authors: 
 - Miguel Vazquez
@@ -9,79 +14,105 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-11-09 00:00:00 +01:00
+date: 2010-05-21 00:00:00 +02:00
 default_executable: sent_config
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: rake
-  type: :runtime
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        segments: 
+        - 0
         version: "0"
-    version: 
+  type: :runtime
+  version_requirements: *id001
 - !ruby/object:Gem::Dependency 
   name: simpleconsole
-  type: :runtime
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        segments: 
+        - 0
         version: "0"
-    version: 
+  type: :runtime
+  version_requirements: *id002
 - !ruby/object:Gem::Dependency 
   name: rbbt
-  type: :runtime
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        segments: 
+        - 0
         version: "0"
-    version: 
+  type: :runtime
+  version_requirements: *id003
 - !ruby/object:Gem::Dependency 
   name: ferret
-  type: :runtime
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        segments: 
+        - 0
         version: "0"
-    version: 
+  type: :runtime
+  version_requirements: *id004
 - !ruby/object:Gem::Dependency 
   name: stemmer
-  type: :runtime
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        segments: 
+        - 0
         version: "0"
-    version: 
+  type: :runtime
+  version_requirements: *id005
 - !ruby/object:Gem::Dependency 
   name: progress-monitor
-  type: :runtime
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
+  prerelease: false
+  requirement: &id006 !ruby/object:Gem::Requirement 
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        segments: 
+        - 0
         version: "0"
-    version: 
+  type: :runtime
+  version_requirements: *id006
 - !ruby/object:Gem::Dependency 
   name: open4
+  prerelease: false
+  requirement: &id007 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
   type: :runtime
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
+  version_requirements: *id007
+- !ruby/object:Gem::Dependency 
+  name: simplews
+  prerelease: false
+  requirement: &id008 !ruby/object:Gem::Requirement 
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        segments: 
+        - 0
         version: "0"
-    version: 
+  type: :runtime
+  version_requirements: *id008
 description: Use literature mining to find semantic features to describe clusers of genes
 email: miguel.vazquez@fdi.ucm.es
 executables: 
@@ -114,18 +145,20 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      segments: 
+      - 0
       version: "0"
-  version: 
 required_rubygems_version: !ruby/object:Gem::Requirement 
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      segments: 
+      - 0
       version: "0"
-  version: 
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.3.5
+rubygems_version: 1.3.6
 signing_key: 
 specification_version: 3
 summary: Semantic Features in Text