rbbt-sent 1.0.0

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Miguel Vazquez
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/R/matrix.R

@@ -0,0 +1,167 @@
+library('som')
+
+SENT.norm <- function(feats, feat.weights = NULL){
+    s = as.matrix(feats) %*% matrix(1,nrow=dim(feats)[2],ncol=1);
+    s = matrix(100/s,nrow=length(s),ncol=dim(feats)[2]);
+    feats.norm = feats * s;
+    rm(s);
+
+    feats.norm[is.na(feats.norm)] = 0
+
+    if (!is.null(feat.weights)){
+      feats.norm =  feats.norm  * matrix(abs(feat.weights),ncol=length(feat.weights),nrow=dim(feats.norm)[1],byrow=T)
+    }
+    
+    feats.norm;
+}
+
+SENT.prepare.matrix <- function(file.input, file.output, file.dict= NULL){
+    feats = read.table(file.input, sep="\t", header=T, row.names=1,check.names=FALSE);
+    
+    if (!is.null(file.dict)){
+        feats.weights = as.matrix(read.table(file=file.dict, sep="\t", row.names=1));
+    }else {
+        feats.weights = NULL;
+    }
+
+    good.words = apply(feats,2,sum) > 0
+    feats = feats[,good.words]
+    feats = SENT.norm(feats, feats.weights);
+
+    write.table(file=file.output, feats, sep="\t", quote=FALSE)
+}
+
+SENT.join.results <- function(prefix){
+    files.w <- Sys.glob(paste(prefix,'.matrix_w.*',sep=""))
+    files.h <- Sys.glob(paste(prefix,'.matrix_h.*',sep=""))
+    
+    data.w <- NULL
+    for (file in files.w){
+        data <- read.table(file, sep="\t", header=T, row.names=1, check.names=FALSE)
+        if (is.null(data.w)){
+            data.w = data
+        }else{
+            data.w <- cbind(data.w,data)
+        }
+    }
+
+    write.table(file=paste(prefix,'.features',sep=''),t(data.w), sep="\t", quote=FALSE, row.names = F)
+
+
+    data.h <- NULL
+    for (file in files.h){
+        data <- read.table(file, sep="\t", header=T, row.names=1, check.names=FALSE)
+        if (is.null(data.h)){
+            data.h = data
+        }else{
+            data.h <- rbind(data.h,data)
+        }
+    }
+
+    write.table(file=paste(prefix,'.profiles',sep=''),t(data.h), sep="\t", quote=FALSE,col.names=F)
+}
+
+
+SENT.analyze <- function(prefix, output, clusters = NULL, num.words = 15){
+    profiles     <- read.table(paste(prefix, '.profiles',sep=""),sep="\t", row.names=1, check.names=F);
+    features     <- read.table(paste(prefix, '.features',sep=""),sep="\t", header=T, check.names=F);
+
+    # Assume 10 repetitions
+    if (is.null(clusters)){
+        clusters = dim(features)[1] / 10 ;
+    }
+
+    # Form a clustering
+    fdist = dist(features)
+    hfeatures <- hclust(fdist, method="ward");
+    cfeatures <- cutree(hfeatures, k=clusters);
+
+    coph <- cor(fdist,cophenetic(hfeatures));
+    write(coph, file = paste(output, '.cophenetic',sep=""));
+
+
+
+    # Average between clusters
+    profiles.merged = vector();
+    features.merged = vector();
+    for (i in levels(factor(cfeatures))){
+        profiles.merged = cbind(profiles.merged, apply(as.matrix(profiles[,cfeatures==i]),1,mean, trim=0.1));
+        features.merged = rbind(features.merged, apply(as.matrix(features[cfeatures==i,]),2,mean, trim=0.1));
+    }
+
+
+    rownames(profiles.merged) <- rownames(profiles);
+    colnames(features.merged) <- colnames(features);
+
+    write.table(file=paste(output,'.merged.profiles',sep=''),profiles.merged, sep="\t", quote=FALSE,col.names=F)
+    write.table(file=paste(output,'.merged.features',sep=''),t(features.merged), sep="\t", quote=FALSE,col.names=F)
+
+    # Hard assign genes to features
+    profiles.bin = profiles.merged
+    for (i in 1:dim(profiles.bin)[1] ){
+        m                   = sort(profiles.bin[i,],index.return = T,decreasing = T)$ix[1];
+        profiles.bin[i,]       = 0;
+        profiles.bin[i,m]      = 1;
+    }
+
+    profiles.sorted = c();
+    profiles.bin.sorted = c();
+    glabels=c();
+
+
+
+    fgroups = cfeatures[unlist(dendrapply(as.dendrogram(hfeatures), function(e) attr(e, "label")))];
+    flabels = sapply(seq(1,dim(features)[1]-1), function(i){ if(fgroups[i] != fgroups[i+1]){ '___'}else{''}});
+    flabels = c(flabels,'');
+
+    flabels[unlist(dendrapply(as.dendrogram(hfeatures), function(e) attr(e, "label")))] = flabels;
+
+    order=unique(fgroups);
+    for (i in order){
+        profiles.sorted = rbind(profiles[profiles.bin[,i]==1,], profiles.sorted);
+        if (sum(profiles.bin[,i]==1) == 0) next
+        glabels = c(rep('',sum(profiles.bin[,i]==1)-1),glabels);
+        glabels = c('___',glabels);
+    }
+
+    # Produce heatmap image
+    bitmap(file=paste(output,'.jpg',sep=""),type='jpeg',res=75);
+    heatmap(as.matrix(profiles),Rowv=NA,Colv=as.dendrogram(hfeatures),xlab="Factors from 10 factorizations", ylab="Genes", labRow=glabels, labCol=flabels, margins=c(4,4));
+
+
+    # Produce heatmap image for hard assignment 
+    bitmap(file=paste(output,'.hard.jpg',sep=""),type='jpeg',res=75);
+    heatmap(as.matrix(profiles.sorted),Rowv=NA,Colv=as.dendrogram(hfeatures),xlab="Factors from 10 factorizations", ylab="Genes", labRow=glabels, labCol=flabels, margins=c(4,4));
+
+    dev.off();
+
+    features.merged.scores = apply(features.merged,2,function(x){ 
+                                   sapply(x,function(v){ 
+                                          (v - (sum(x) - v)/(length(x) - 1))
+                                          })
+                                   })
+
+#    a = 0.1
+#    features.merge.specificity = apply(features.merged, 2, function(x){ sapply(x, function(v){ v / mean(x)})})
+#    features.merge.importance  = apply(features.merged, 1, function(x){ sapply(x, function(v){ v / mean(x)})})
+#    features.scores = a * t(features.merge.importance) + (1-a) * features.merge.specificity
+
+
+
+
+
+# Save Group Genes and Words
+    g = 1
+    for (i in order){
+        genes = rownames(profiles)[profiles.bin[,i] == 1];
+        cat(file=paste(output,g,'genes',sep="."),genes,sep="\n");
+        words = names(sort(features.merged.scores[i,],decreasing=T))[1:num.words];
+##features.t.test = apply(features, 2, function(x){ t.test(x[cfeatures == i],x[cfeatures != i])})
+##words = names(sort(sapply(features.t.test,function(x) { x$p.value}),decreasing=F))[1:num.words]
+#words = names(sort(features.scores[i,],decreasing=T))[1:num.words]
+        cat(file=paste(output,g,'words',sep="."),words,sep="\n");
+        g = g + 1
+    }
+}
+
+

data/README.rdoc

@@ -0,0 +1,23 @@
+= rbbt-sent
+
+Description goes here.
+
+===
+
+Install R som package
+Install ghostscript
+
+== Note on Patches/Pull Requests
+ 
+* Fork the project.
+* Make your feature addition or bug fix.
+* Add tests for it. This is important so I don't break it in a
+  future version unintentionally.
+* Commit, do not mess with rakefile, version, or history.
+  (if you want to have your own version, that is fine but
+   bump version in a commit by itself I can ignore when I pull)
+* Send me a pull request. Bonus points for topic branches.
+
+== Copyright
+
+Copyright (c) 2009 Miguel Vazquez. See LICENSE for details.

data/bin/sent_config

@@ -0,0 +1,193 @@
+#!/usr/bin/ruby
+
+require 'simpleconsole'
+
+begin
+  require 'sent'
+rescue 
+  $noconfig = true
+end
+
+$USAGE =<<EOT
+#{__FILE__} <action> [<subaction>] [--force] [--organism <org>]
+  actions:
+    * configure:   Set paths for data, work, and tmp directories
+
+    * install:
+        * analysis: Install configuration to perform analysis
+    
+    * update:
+        * metadocs: Generate metadocs for all organisms
+    
+    * init:
+        * webservice: 
+        * www:
+
+
+
+EOT
+
+class Controller < SimpleConsole::Controller
+
+  params :bool => {:f => :force},
+    :string => {:o => :organism, :h => :host, :p => :port}
+
+  def init
+
+    @host = params[:host]
+    @port = params[:port]
+    render :action => params[:id]
+
+  end
+
+  def default
+    render :action => :usage
+  end
+
+  def help
+    render :action => :usage
+  end
+
+  def install
+    raise "Run #{__FILE__} configure first to configure sent" if $noconfig
+
+    case params[:id]
+    when "analysis"
+      @tasks = %w(analysis)
+    when nil
+      redirect_to :action => :help, :id => :install
+    else
+      @tasks = [params[:id]]
+    end
+
+    $force = true if params[:force]
+    $org = params[:organism] if params[:organism]
+
+  end
+
+  def update
+    raise "Run #{__FILE__} configure first to configure rbbt" if $noconfig
+
+    case params[:id]
+    when "metadocs"
+      @location = File.join(Sent.datadir,'analysis')
+    else
+      redirect_to :action => :help, :id => :update
+    end
+
+    $force = true if params[:force]
+    $org = params[:organism] if params[:organism]
+
+  end
+
+  def configure
+  end
+
+end
+
+class View < SimpleConsole::View
+  def usage
+    puts $USAGE
+  end
+
+  def install
+    require 'rake'
+    load File.join(Sent.rootdir, 'tasks/install.rake')
+
+    @tasks.each{|t|
+      puts "Invoking #{ t }"
+      Rake::Task[t].invoke
+    }
+  end
+
+  def update
+    require 'rake'
+
+    puts "Changing directory to #{@location}"
+    chdir @location
+
+    load "./Rakefile"
+
+    Rake::Task['update'].invoke
+  end
+
+
+  def configure
+
+    defaultdir = File.join(ENV['HOME'],'sent')
+
+    workdir   = File.join(defaultdir, 'work')
+    tmpdir   = File.join(defaultdir, 'tmp')
+    datadir   = File.join(defaultdir, 'data')
+
+    puts "Please indicate where you wish to place the data directories"
+    puts
+
+    puts
+    puts "* work Directory: This directory will hold downloads, from PubMed, 
+  Entrez and other, for local store. It might grow considerably."
+    print "[#{ workdir }]? "
+    input = STDIN.gets
+    workdir = input if input =~ /\w/
+
+      puts
+    puts "* Tmp Directory: Temporary files." 
+    print "[#{ tmpdir }]? "
+    input = STDIN.gets
+    tmpdir = input if input =~ /\w/
+
+    puts
+    puts "* Data Directory: Holds data from organisism, databases, third party software, etc."
+    print "[#{ datadir }]? "
+    input = STDIN.gets
+    datadir = input if input =~ /\w/
+
+
+
+    fout = File.open(File.join(ENV['HOME'], '.sent'),'w')
+    fout.puts "workdir: #{workdir}" 
+    fout.puts "tmpdir: #{tmpdir}" 
+    fout.puts "datadir: #{datadir}" 
+    fout.close
+
+  end
+
+  def webservice
+
+    FileUtils.cd File.join(Sent.rootdir, 'webservice/bin')
+    require 'sentWS'
+
+    host = @host || `hostname`.chomp.strip + '.' +  `hostname -d`.chomp.strip
+    port = @port || '8182'
+
+    puts "Starting Server in #{ host }:#{ port }"
+    server = SentWS.new("Sent", "Sent Web Server",host, port, Sent.workdir)
+
+    FileUtils.mkdir_p File.join(Sent.rootdir, '/webservice/wsdl/') unless File.exist? File.join(Sent.rootdir, '/webservice/wsdl/')
+    Open.write(File.join(Sent.rootdir, '/webservice/wsdl/SentWS.wsdl'), server.wsdl)
+
+    trap('INT') { server.abort_jobs; server.shutdown }
+    server.start
+
+  end
+
+  def www
+    FileUtils.cd File.join(Sent.rootdir, 'merb')
+
+    FileUtils.mkdir_p 'cache' unless File.exist? 'cache' 
+    FileUtils.mkdir_p 'public/tmp' unless File.exist? 'public/tmp' 
+
+    host = @host || `hostname`.chomp.strip + '.' +  `hostname -d`.chomp.strip
+    port = @port || '8181'
+
+    require 'merb-core'
+    Merb.start("-a mongrel -e production -p #{ port } -h #{ host }".split)
+
+
+  end
+
+end
+
+SimpleConsole::Application.run(ARGV, Controller, View)
+
+

data/install_scripts/analysis/Rakefile

@@ -0,0 +1,139 @@
+require 'sent'
+require 'sent/main'
+require 'rbbt/sources/organism'
+require 'progress-monitor'
+
+
+$list = ENV['list']
+$kstart = ENV['kstart'] || ENV['k']
+$kend = ENV['kend'] || $kstart
+
+rule (/summary\/(.*)/) => lambda {|n| n.sub(/summary/,'NMF') } do |t|
+  nmf = t.name.sub(/summary/,'NMF')
+
+  Sent.analyze(nmf, t.name)
+  FileUtils.touch t.name
+end
+
+rule (/NMF\/(.*)/) => lambda {|n| n.sub(/NMF/,'matrices') } do |t|
+  matrix = t.name.sub(/NMF/,'matrices')
+
+  k = $kstart
+  if $kstart < $kend
+    best = 0
+    ccc = Sent.CCC(matrix, $kstart, $kend)
+    ccc.each_with_index{|v,i|
+      if v.to_i > best
+        k = $kstart.to_i + i
+      end
+    }
+  end
+
+  Sent.NMF(matrix, t.name, k.to_i, 10)
+  FileUtils.touch t.name
+end
+
+rule (/matrices\/(.*)/) => lambda {|n| n.sub(/matrices/,'metadocs') } do |t|
+  metadocs = t.name.sub(/matrices/,'metadocs')
+  
+  list = nil
+  list = Open.read($list).collect{|l| l.chomp} if $list
+  
+  Sent.matrix(metadocs, t.name, list)
+end
+
+rule (/metadocs\/(.*)/) => lambda {|n| n.sub(/metadocs/,'associations') } do |t|
+  assocfile = t.name.sub(/metadocs/,'associations')
+
+  Sent.metadocs(assocfile, t.name)
+end
+
+rule(/associations\/(.*)_text/) do |t|
+  org = File.basename(t.name).sub(/_text/,'')
+
+  ner   = Organism.ner(org, :rner)
+  norm  = Organism.norm(org)
+  pmids = Organism.literature(org)
+
+  fout = File.open(t.name, 'w')
+  chunks = pmids.chunk(100)
+
+  Progress.monitor("Finding gene-article associations in text", 1000)
+  chunks.each{|chunk|
+    PubMed.get_article(chunk).each{|pmid, article|
+      text = article.text
+      
+      mentions = ner.extract(text)
+
+      Progress.monitor("Resolving mentions", 1000)
+      codes = mentions.collect{|mention| 
+        matches = norm.match(mention)
+        norm.select(matches,mention,text)
+      }.flatten.uniq.sort
+      
+      codes.each{|code|
+        fout.puts "#{ code }\t#{pmid}"
+      }
+
+    }
+  }
+  fout.close
+end
+
+rule (/associations\/(.*)/) => lambda{|n| n + '_text'} do |t|
+  org = File.basename(t.name)
+
+  fout = File.open(t.name, 'w')
+  fout.write Open.read(t.name + '_text')
+
+  associations = Organism.gene_literature(org)
+  associations.each{|gene, pmids|
+    pmids.each{|pmid|
+      fout.puts "#{ gene }\t#{pmid}"
+    }
+  }
+
+  associations = Organism.gene_literature_go(org)
+  associations.each{|gene, pmids|
+    pmids.each{|pmid|
+      fout.puts "#{ gene }\t#{pmid}"
+    }
+  }
+  fout.close
+
+  name = Organism.name(org)
+  supported_ids = Organism.supported_ids(org, :examples => true)
+  associations = Open.to_hash(t.name, :flatten => true)
+
+  description =<<-EOT
+Name: #{ name }
+Organism: #{ name }
+Description: #{associations.values.flatten.length} associations for #{associations.keys.length} genes and #{associations.values.flatten.uniq.length} articles
+ID Format: #{supported_ids.collect{|p| "#{ p[0] } (#{ p[1] })"}.join(", ")}
+  EOT
+
+  Open.write(t.name + '.description', description)
+end
+
+task 'clean' do
+  FileUtils.rm Dir.glob("associations/*")
+end
+
+task 'all' do
+  Organism.all.each{|org|
+    `rake metadocs/#{ org }`
+  }
+end
+
+task 'update' do
+  if $org
+    FileUtils.rm Dir.glob("**/#{$org}.*") if $force
+    Rake::Task["metadocs/#{$org}"].invoke
+  else
+    Rake::Task['clean'].invoke if $force
+    Rake::Task['all'].invoke
+  end
+end
+
+
+

data/lib/sent/main.rb

@@ -0,0 +1,328 @@
+require 'rbbt/sources/pubmed'
+require 'rbbt/util/misc'
+require 'rbbt/util/open'
+require 'rbbt/bow/bow'
+require 'rbbt/bow/dictionary'
+
+require 'soap/wsdlDriver'
+require 'stemmer'
+require 'open4'
+require 'progress-monitor'
+require 'yaml'
+
+# Produce Stem lists, used by the Web Service
+class String
+
+  alias old_stem stem
+
+  def self.reset_stem_list
+    @@stem_dictionary = Hash.new
+  end 
+
+  # Extends the stem functionality so that is generates a dictionary of
+  # stems. For each stem a list of words that reduce to it.
+  def stem
+    res = old_stem
+    @@stem_dictionary[res] ||= Hash.new
+    @@stem_dictionary[res][self] ||= 1
+    res
+  end
+
+  # Returns the dictionary of recorded stems.
+  def self.stem_list(dictionary = nil)
+    stem_list = Hash.new
+    @@stem_dictionary.each{|k,l|
+      next if dictionary && !dictionary.include?(k)
+      stem_list[k] = l.keys
+    }
+    stem_list
+  end
+
+  reset_stem_list
+end
+
+
+module Sent
+
+  class NoGenesError < StandardError; end
+  class ProcessAbortedError < StandardError; end
+  class WSError < StandardError; end
+
+  def self.run_R(command)
+    pid, stdin, stdout, stderr = Open4::popen4 "R --vanilla --slave"
+    stdin.write "source('#{File.join(Sent.rdir,'matrix.R')}');\n"
+    stdin.write "#{ command };\n"   
+    stdin.close
+
+    Process.wait pid
+    raise ProcessAborted, "Error in R process" if $?.exitstatus != 0
+    result = stdout.read + stderr.read
+    stdout.close
+    stderr.close
+
+    puts result if result != ""
+    result
+  end
+
+  def self.metadocs(assocfile, output, low=0.001, hi=0.65, max=3000)
+    
+    associations = Open.to_hash(assocfile, :flatten => true, :sep => "\t|,")
+
+    dict = Dictionary::TF_IDF.new
+
+    String.reset_stem_list
+
+    Progress.monitor("Building Dictionary for #{File.basename(output)}", 1000) 
+    associations.each{|gene, pmids|
+      text = PubMed.get_article(pmids).collect{|p| p[1].text}.join("\n")
+      dict.add(BagOfWords.count(text.bigrams))
+    }
+
+    # At least 3 genes must have a word to be chosen
+    hard_min = 3 * 100 / associations.keys.length
+    hi = hard_min if hi < hard_min
+    
+    d =  dict.weights(:low => low, :hi => hi, :limit => max)
+    Open.write(output + '.dict', d.sort.collect{|p| p.join("\t")}.join("\n"))
+    terms = d.keys.sort
+
+    fout = File.open(output, 'w')
+    fout.puts("\t" + terms.join("\t"))
+
+    Progress.monitor("Building Metadoc for #{File.basename(output)}", 1000) 
+    associations.each{|gene, pmids|
+      text = PubMed.get_article(pmids).collect{|p| p[1].text}.join("\n")
+      fout.puts(([gene] + BagOfWords.features(text, terms)).join("\t"))
+    }
+    fout.close
+
+    Open.write(output + '.stems', String.stem_list(terms.collect{|p| p.split(/ /)}.flatten.uniq).collect{|k,v| "#{ k }\t#{v.join("\t")}"}.join("\n"))
+  end
+
+  def self.matrix(metadocs, output, list=nil)
+    list ||= []
+      
+    if list.empty?
+      FileUtils.cp metadocs, output
+    else
+      `head -n 1 #{ metadocs } > #{ output }`
+      `grep '^\\(#{list.join('\\|')}\\)[[:space:]]' #{ metadocs } >> #{output}`
+      raise Sent::NoGenesError, "No Genes Matched" if $? != 0
+    end
+
+    dict = metadocs + '.dict'
+    run_R("SENT.prepare.matrix('#{ output }', '#{ output }', '#{metadocs + '.dict'}')")
+  end
+
+
+  @@bionmf_wsdl = "http://bionmf.dacya.ucm.es/WebService/BioNMFWS.wsdl"   
+  def self.NMF(matrix, out, k, executions = 10)
+    driver = SOAP::WSDLDriverFactory.new( @@bionmf_wsdl).create_rpc_driver
+
+    # Upload matrix
+    nmf_matrix = driver.upload_matrix(
+       File.open(matrix).read, # matrix
+       false, # binary 
+       true,  # column labels 
+       true,  # row labels
+       true,  # transpose
+       "No",  # positive
+       "No",  # normalization 
+       "matrix") # Suggested name
+    # Send several executions in parallel
+    while !driver.done(nmf_matrix)
+      sleep(5)
+    end
+
+    if driver.error(nmf_matrix) 
+      error = driver.messages(nmf_matrix).join("\n")
+      raise "Error pre-processing matrix!"  + driver.messages(nmf_matrix).join("\n")
+    end
+
+    threads = []
+    error = nil
+    executions.times{|i|
+      threads << Thread.new(i){ |num|
+        times = 3
+        begin
+          
+          job_id = driver.standardNMF(
+            nmf_matrix, # Matrix job
+            "Standard", # Algorithm
+            k, # Factor Start
+            k, # Factor End
+            1, # Runs
+            2000, # Iterations
+            40,   # Stop criteria
+            0,    # Not used (nsnmf smoothness)
+            false, # extra info
+            '')    # Suggested name
+
+          while !driver.done(job_id)
+            sleep(5)
+          end
+
+          if driver.error(job_id) 
+            error = driver.messages(job_id).join("\n")
+            raise "Error in NMF"  + driver.messages(job_id).join("\n")
+          end
+
+          results =  driver.results(job_id)
+          fw = File.open(out + ".matrix_w.#{num}",'w')
+          fw.write(driver.result(results[0]).sub(/\t(.*)\t$/,'\1'))
+          fw.close
+          fh = File.open(out + ".matrix_h.#{num}",'w')
+          fh.write(driver.result(results[1]).sub(/\t(.*)\t$/,'\1'))
+          fh.close
+          driver.clean(job_id)
+        rescue Sent::ProcessAbortedError
+          puts "Process aborted for #{ num }"
+          driver.abort(job_id)
+        rescue Timeout::Error
+          if times > 0
+            times -= 1
+            sleep 2
+            retry
+          else
+            raise Sent::ProcessAbortedError, "NMF Execution #{ num } timed out"
+          end
+        rescue Exception
+          puts $!.message
+          if times > 0
+            times -= 1
+            puts "Retrying thread #{ num }"
+            retry
+          else
+
+            puts "NMF Execution #{ num } Produced Exception"
+            puts $!.class
+            puts $!.message
+            puts $!.backtrace
+            raise Sent::ProcessAbortedError, "NMF Execution #{ num } Produced Exception"
+          end
+        ensure
+          Thread.exit
+        end
+      }
+      sleep 1
+
+    }
+
+    # Allow threads to be aborted
+    aborted = false
+    old_int = Signal.trap("INT") do
+      STDERR.puts "Killing threads"
+      threads.each{|t| t.raise Sent::ProcessAbortedError, "Process Aborted"}
+      aborted = true
+    end
+
+    threads.each { |aThread|  aThread.join }
+
+    Signal.trap("INT", old_int)
+    driver.clean(nmf_matrix)
+
+    if aborted
+      raise Sent::ProcessAbortedError, "Process Aborted"
+    end
+
+    if error
+      raise Exception, "Error in NMF:\n" + error
+    end
+
+    run_R("SENT.join.results('#{ out }')")
+
+    FileUtils.rm Dir.glob(out + '.matrix_*.*')
+  end
+
+  def self.CCC(matrix, kstart, kend)
+    raise "Error in range: #{ kstart } to #{ kend }" if kstart >= kend
+
+    driver = SOAP::WSDLDriverFactory.new( @@bionmf_wsdl).create_rpc_driver
+
+    # Prepare matrix for processing
+    nmf_matrix = driver.upload_matrix(File.open(matrix).read)
+    driver.preprocess(nmf_matrix,1,"No","No", true, true)    
+
+    job_id = driver.sample_classification(nmf_matrix,kstart.to_i,kend.to_i,10)
+
+    aborted = false
+    old_int = Signal.trap("INT") do
+      puts "Aborting bestK process"
+      driver.abort(job_id)
+      aborted = true
+    end
+
+    while (status = driver.status(job_id)) == 0
+      sleep(5)
+    end
+
+    driver.clean_matrix(nmf_matrix)
+    Signal.trap("INT", old_int)
+
+    if aborted
+      raise Sent::ProcessAbortedError, "Process Aborted"
+    end
+
+    if status == -1 
+      raise Sent::WSError, "Error processing matrix:\n" + driver.info(job_id)
+    end
+
+    results = driver.results(job_id)
+    text = driver.get_result(results[0])
+    text.split(/\n/s).last.split(/\t/)
+  end
+
+  def self.analyze(prefix, output, clusters = nil, num_words = 15)
+
+    FileUtils.rm Dir.glob(output + '*.words') + Dir.glob(output + '*.genes')
+    run_R("SENT.analyze('#{ prefix }', '#{ output }', '#{clusters}', '#{num_words}')")
+    words = Dir.glob(output + '*.words').sort.collect{|f| Open.read(f).split(/\n/)}
+    genes = Dir.glob(output + '*.genes').sort.collect{|f| Open.read(f).split(/\n/)}
+
+    groups = []
+    words.zip(genes).each{|p|
+      groups << {:words => p[0], :genes => p[1]}
+    }
+
+    Open.write(output + '.summary', groups.to_yaml)
+  end
+
+  def self.literature_index(pmids, outfile)
+
+    index = Ferret::Index::Index.new(:path => outfile)
+
+    index.field_infos.add_field(:title,  :index => :yes, :boost => 0.67)
+    index.field_infos.add_field(:abstract, :index => :yes, :boost => 0.33)
+
+    Progress.monitor("Building index for #{pmids.length} articles")
+    pmids.each{|pmid|
+      begin
+        article = PubMed.get_article(pmid)
+        abstract = article.abstract
+        title    = article.title
+
+        abstract_content = BagOfWords.terms(abstract).collect{|w,n| (1..n).collect{ w }}.flatten.join(" ")
+        title_content    = BagOfWords.terms(title).collect{|w,n| (1..n).collect{ w }}.flatten.join(" ")
+
+        index << {:id => pmid, :abstract => abstract_content, :name => title_content}
+      rescue Exception
+        puts $!.backtrace
+        puts $!.message
+      end
+
+    }
+    index.close
+  end
+  
+  def self.search_index(words, index)
+    index = Ferret::Index::Index.new(:path => index)
+
+    ranks = []
+    index.search_each("#{ words.collect{|w| w.stem}.join(" ") }", :limit => 8000) do |id,score|
+      next unless score > 0.0001
+      ranks << [index[id][:id],score]
+    end
+
+    ranks
+  end
+end

data/lib/sent.rb

@@ -0,0 +1,71 @@
+$:.unshift(File.dirname(__FILE__)) unless
+  $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
+
+require 'yaml'
+require 'fileutils'
+require 'stemmer'
+require 'ferret'
+
+module Sent
+
+  class NoConfigError < StandardError; end
+
+  @@rootdir = File.dirname(File.dirname(__FILE__))
+  @@datadir = @@workdir = @@tmpdir = nil
+
+  def self.load_config
+    if File.exist?File.join(@@rootdir, 'sent.config')
+      config = YAML.load_file(File.join(@@rotdir, 'sent.config'))
+      if config.is_a? Hash
+        @@datadir  = config['datadir'] if config['datadir'] 
+        @@workdir = config['workdir'] if config['workdir']
+        @@tmpdir   = config['tmpdir'] if config['tmpdir']
+      end
+    end
+
+
+
+    if File.exist?(File.join(ENV['HOME'], '.sent'))
+      config = YAML.load_file(File.join(ENV['HOME'], '.sent') )
+      if config.is_a? Hash
+        @@datadir  = config['datadir'] if config['datadir'] 
+        @@workdir = config['workdir'] if config['workdir']
+        @@tmpdir   = config['tmpdir'] if config['tmpdir']
+      end
+    end
+
+    if @@datadir.nil?  || @@workdir.nil? || @@tmpdir.nil?
+      raise Sent::NoConfig, "sent not configured. Edit #{File.join(@@rootdir, 'sent.config')} or $HOME/.sent"
+    end
+
+
+    FileUtils.mkdir_p @@datadir  unless File.exist? @@datadir
+    FileUtils.mkdir_p @@workdir unless File.exist? @@workdir
+    FileUtils.mkdir_p @@tmpdir   unless File.exist? @@tmpdir
+
+  end
+
+  def self.datadir
+    @@datadir
+  end 
+  def self.workdir
+    @@workdir
+  end 
+  def self.tmpdir
+    @@tmpdir
+  end 
+
+  def self.rootdir
+    @@rootdir
+  end
+
+
+  def self.rdir
+    File.join(@@rootdir, 'R')
+  end
+
+  self.load_config
+end
+
+
+

data/tasks/install.rake

@@ -0,0 +1,17 @@
+require 'sent'
+
+$datadir = Sent.datadir
+$scriptdir = File.join(Sent.rootdir, '/install_scripts')
+
+task 'analysis' do
+  directory = "#{$datadir}/analysis"
+  FileUtils.mkdir_p directory
+  %w(Rakefile).each{|f|
+    FileUtils.cp_r File.join($scriptdir, "analysis/#{ f }"), directory 
+  }
+  
+  %w(associations metadocs matrices NMF summary).each{|d|
+    FileUtils.mkdir_p File.join(directory, d)
+  }
+end
+

data/test/helper.rb

@@ -0,0 +1,9 @@
+require 'rubygems'
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'sent'
+
+class Test::Unit::TestCase
+end

metadata

@@ -0,0 +1,133 @@
+--- !ruby/object:Gem::Specification 
+name: rbbt-sent
+version: !ruby/object:Gem::Version 
+  version: 1.0.0
+platform: ruby
+authors: 
+- Miguel Vazquez
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-11-09 00:00:00 +01:00
+default_executable: sent_config
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rake
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: simpleconsole
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: rbbt
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: ferret
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: stemmer
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: progress-monitor
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: open4
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+description: Use literature mining to find semantic features to describe clusers of genes
+email: miguel.vazquez@fdi.ucm.es
+executables: 
+- sent_config
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE
+- README.rdoc
+files: 
+- R/matrix.R
+- bin/sent_config
+- install_scripts/analysis/Rakefile
+- lib/sent.rb
+- lib/sent/main.rb
+- tasks/install.rake
+- test/helper.rb
+- LICENSE
+- README.rdoc
+has_rdoc: true
+homepage: http://github.com/mikisvaz/rbbt-sent
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: Semantic Features in Text
+test_files: 
+- test/helper.rb