rbbt 1.1.7 → 2.0.0

data/install_scripts/classifier/R/classify.R

@@ -1,36 +0,0 @@
-library('e1071')
-
-BOW.norm <- function(x, weights = NULL){
-    x = 1 + log(x);
-    x[x==-Inf] = 0;
-    x.sum = as.matrix(x) %*% matrix(1,nrow=dim(x)[2],ncol=1);
-    x.sum = matrix(100/x.sum,nrow=length(x.sum),ncol=dim(x)[2]);
-    x.norm = x * x.sum;
-    rm(x.sum);
-    x.norm[is.na(x.norm)] = 0
-
-    if (!is.null(weights)){
-      x.norm =  x.norm  * matrix(abs(weights),ncol=length(weights),nrow=dim(x.norm)[1],byrow=T)
-    }
-
-    x.norm;
-}
-
-
-BOW.classification.model <- function(features, modelfile, dictfile = NULL){
-    feats = read.table(features, sep="\t", header=T, row.names=1);
-
-    if (!is.null(dictfile)){
-        svm.weights = read.table(file=dictfile, sep="\t")[2];
-    }else {
-        svm.weights = NULL;
-    }
-    feats[-1] = BOW.norm(feats[-1], svm.weights);
-    svm.model = svm(Class ~ ., data=feats, svm.weights);
-    save(svm.model,svm.weights, file=modelfile);
-}
-
-BOW.classification.classify <- function(modelfile, x, weights = NULL){
-    x = BOW.norm(x, weights);
-    predict(modelfile, x);
-}

data/install_scripts/classifier/Rakefile

@@ -1,145 +0,0 @@
-require 'rbbt'
-require 'rbbt/sources/organism'
-require 'rbbt/sources/pubmed'
-require 'rbbt/bow/bow'
-require 'rbbt/bow/dictionary'
-require 'rbbt/bow/classifier'
-require 'rbbt/util/misc'
-
-require 'progress-monitor'
-require 'rand'
-
-$hi      = ENV['hi']  || 0.8
-$low     = ENV['low'] || 0.01
-$max     = ENV['max'] || 3000
-$bigrams = ENV['bigrams'] == 'true' || false
-
-$ndocs   = ENV['ndocs'] || 5000
-
-desc "Bilds Dictionary and Features for an organism"
-rule(/data\/(.*)/) do |t|
-  org = File.basename(t.name)
-
-  go  = Organism.gene_literature_go(org).collect{|gene, pmids| pmids}.flatten.uniq
-  all = Organism.literature(org).flatten.uniq - go
-
-  ndocs = [go.length, all.length, $ndocs.to_i].min
-  puts "Using #{ ndocs } from each class\n\n"
-
-  go    = go.shuffle[0..ndocs - 1]
-  all   = all.shuffle[0..ndocs - 1]
-
-  dict = Dictionary::KL.new
-
- 
-
-  chunks = all.chunk(50)
-  Progress.monitor("Building Dictionary for #{ org }: -",1000)
-  chunks.each{|chunk|
-    PubMed.get_article(chunk).each{|pmid, article|
-      words = BagOfWords.terms(article.text,$bigrams)
-      dict.add(words, :-)
-    }
-  }
-
-  chunks = go.chunk(50)
-  Progress.monitor("Building Dictionary for #{ org }: +",1000)
-  chunks.each{|chunk|
-    PubMed.get_article(chunk).each{|pmid, article|
-      words = BagOfWords.terms(article.text,$bigrams)
-      dict.add(words, :+)
-    }
-  }
-
-  term_weigths = dict.weights(:low => $low.to_f, :hi => $hi.to_f, :limit => $max.to_i)
-  Open.write(t.name + '.dict', term_weigths.sort.collect{|p| p.join("\t")}.join("\n"))
-
-  terms = term_weigths.keys.sort
-
-  fout = File.open(t.name, 'w')
-  fout.puts((['Name','Class'] + terms).join("\t"))
-  
-  Progress.monitor("Building Features for #{ org }", 1000)
-  all.each{|pmid|
-    text = PubMed.get_article(pmid).text
-    fout.puts(([pmid, :-] + BagOfWords.features(text, terms)).join("\t"))
-  }
-  go.each{|pmid|
-    text = PubMed.get_article(pmid).text
-    fout.puts(([pmid, :+] + BagOfWords.features(text, terms)).join("\t"))
-  }
-
-
-  fout.close
-end
-
-rule (/model\/(.*)/) => lambda{|n| n.sub(/model/,'data')} do |t|
-  features = t.name.sub(/model/,'data')
-  Classifier.create_model(features, t.name, features + '.dict')
-end
-
-rule (/results\/(.*)/) => lambda{|n| n.sub(/results/,'model')} do |t|
-  model       = t.name.sub(/results/,'model')
-  features    = t.name.sub(/results/,'data')
-  org = File.basename(t.name)
-
-  ndocs    = 100
-
-  used = []
-  if "".respond_to? :collect
-    used = Open.read(features).collect{|l| l.chomp.split(/\t/).first}[1..-1]
-  else
-    used = Open.read(features).lines.collect{|l| l.chomp.split(/\t/).first}[1..-1]
-  end
-
-  classifier = Classifier.new(model)
-  go  = Organism.gene_literature_go(org).collect{|gene, pmids| pmids}.flatten.uniq - used
-  all = Organism.literature(org).flatten.uniq - go - used
-
-  go    = go.shuffle[0..ndocs - 1]
-  all   = all.shuffle[0..ndocs - 1]
-
-  ndocs = go.length + all.length
-
-  raise "Not enogh unused articles to evaluate" if  go.empty? || all.empty?
-
-  features_go = PubMed.get_article(go).collect{|pmid, article|
-    article = article.text
-  }
-  pos = classifier.classify(features_go).select{|v| v == '+'}.length
-
-  features_all = PubMed.get_article(all).collect{|pmid, article|
-    article = article.text
-  }
-  neg = classifier.classify(features_all).select{|v| v == '-'}.length
-
-  puts "#{ pos } #{ neg }"
-
-  precision = (pos + neg) / (ndocs).to_f
-  recall    = pos / go.length.to_f
-  f1        = ( 2 * precision * recall) / (precision + recall ).to_f
-
-  puts "Precision: #{ precision}, Recall: #{ recall }, F1: #{f1}"
-end
-
-task 'clean' do
-  FileUtils.rm Dir.glob("data/*")
-  FileUtils.rm Dir.glob("model/*")
-  FileUtils.rm Dir.glob("results/*")
-
-end
-task 'all' do
-  Organism.all.each{|org|
-    Rake::Task["model/#{ org }"].invoke
-  }
-end
-task 'update' do
-  if $org
-    FileUtils.rm Dir.glob("**/#{$org}.*") if $force
-    Rake::Task["model/#{$org}"].invoke
-  else
-    Rake::Task['clean'].invoke if $force
-    Rake::Task['all'].invoke
-  end
-end
-

data/install_scripts/get_abner.sh

@@ -1,2 +0,0 @@
-#!/bin/bash
-wget http://pages.cs.wisc.edu/~bsettles/abner/abner.jar

data/install_scripts/get_banner.sh

@@ -1,25 +0,0 @@
-#!/bin/bash
-
-wget "http://downloads.sourceforge.net/banner/BANNER_v02.zip?modtime=1196955449&big_mirror=0"
-wget "http://downloads.sourceforge.net/banner/gene_model_v02.bin?modtime=1196955509&big_mirror=0"
-mv BANNER_v02.zip BANNER.zip
-mv gene_model_v02.bin gene_model.bin
-unzip BANNER.zip
-cd BANNER
-libs=`find libs/ -name "*.jar"`
-mkdir classes
-javac -classpath `echo $libs|sed s/\ /:/g` -d classes `find src/ -name "*.java"`
-cd classes
-for f in ../libs/*.jar; do jar xf "$f";done
-jar cf banner.jar *
-mv banner.jar ../..
-cd ..
-cp -R nlpdata/ ../
-cd ..
-rm BANNER.zip
-rm -Rf BANNER
-
-
-
-
-

data/install_scripts/get_biocreative.sh

@@ -1,72 +0,0 @@
-#!/bin/bash
-
-mkdir src
-cd src
-wget "http://garr.dl.sourceforge.net/sourceforge/biocreative/bc2GNandGMgold_Subs.tar.gz"
-wget "http://switch.dl.sourceforge.net/sourceforge/biocreative/biocreative1task1a.tar.gz"
-wget "http://kent.dl.sourceforge.net/sourceforge/biocreative/biocreative1task1b.tar.gz"
-wget "http://mesh.dl.sourceforge.net/sourceforge/biocreative/biocreative1task2.tar.gz"
-wget "http://garr.dl.sourceforge.net/sourceforge/biocreative/bc2geneMention.tar.gz"
-wget "http://switch.dl.sourceforge.net/sourceforge/biocreative/bc2normal.1.4.tar.gz"
-wget "http://kent.dl.sourceforge.net/sourceforge/biocreative/bc2GNtest.zip"
-
-for f in *.gz; do tar xfz $f; done
-unzip bc2GNtest.zip
-
-cd ..
-
-mkdir BC2GM
-cp -R src/bc2geneMention/train/ BC2GM/
-cp -R src/sourceforgeDistrib-22-Sept-07/genemention/BC2GM/test/ BC2GM/
-mv BC2GM/train/alt_eval.perl BC2GM/
-
-mkdir BC2GN
-cp -R src/biocreative2normalization/* BC2GN/
-mv BC2GN/noisyTrainingData/ BC2GN/NoisyTrain
-mv BC2GN/trainingData/ BC2GN/Train
-cp -R src/bc2GNtest/bc2GNtestdocs/ BC2GN/Test
-mv BC2GN/NoisyTrain/noisytrain.genelist BC2GN/NoisyTrain/genelist
-mv BC2GN/Train/training.genelist BC2GN/Train/genelist
-cp src/sourceforgeDistrib-22-Sept-07/genenormalization/bc2test.genelist BC2GN/Test/genelist
-
-mkdir BC1GN
-cp -R src/biocreative1/bc1task1b/* BC1GN/
-mv BC1GN/fly/FlyDevTest/ BC1GN/fly/devtest
-mv BC1GN/fly/FlyEvaluation/ BC1GN/fly/test
-mv BC1GN/fly/FlyNoisyTraining/ BC1GN/fly/train
-mv BC1GN/fly/*.list  BC1GN/fly/synonyms.list
-mv BC1GN/fly/test/*gene_list  BC1GN/fly/test/genelist
-for f in BC1GN/fly/train/gene_list/*; do cat "$f" >> BC1GN/fly/train/genelist;done
-for f in BC1GN/fly/devtest/gene_lists/*; do cat "$f" >> BC1GN/fly/devtest/genelist;done
-mv BC1GN/mouse/MouseDevTest/ BC1GN/mouse/devtest
-mv BC1GN/mouse/MouseEvaluation/ BC1GN/mouse/test
-mv BC1GN/mouse/MouseNoisyTraining/ BC1GN/mouse/train
-mv BC1GN/mouse/*.list  BC1GN/mouse/synonyms.list
-mv BC1GN/mouse/test/*gene_list  BC1GN/mouse/test/genelist
-for f in BC1GN/mouse/train/gene_list/*; do cat "$f" >> BC1GN/mouse/train/genelist;done
-for f in BC1GN/mouse/devtest/gene_lists/*; do cat "$f" >> BC1GN/mouse/devtest/genelist;done
-mv BC1GN/yeast/YeastDevTest/ BC1GN/yeast/devtest
-mv BC1GN/yeast/YeastEvaluation/ BC1GN/yeast/test
-mv BC1GN/yeast/YeastNoisyTraining/ BC1GN/yeast/train
-mv BC1GN/yeast/*.list  BC1GN/yeast/synonyms.list
-mv BC1GN/yeast/test/*gene_list  BC1GN/yeast/test/genelist
-for f in BC1GN/yeast/train/gene_list/*; do cat "$f" >> BC1GN/yeast/train/genelist;done
-for f in BC1GN/yeast/devtest/gene_lists/*; do cat "$f" >> BC1GN/yeast/devtest/genelist;done
-# Fix a bug in the perl script! :-|
-cat BC1GN/task1Bscorer.pl |grep -v 'else {EVALFILE = STDIN;}' >foo; mv foo BC1GN/task1Bscorer.pl
-
-
-
-rm -Rf src
-
-
-
-
-
-
-
-
-
-
-
-

data/install_scripts/get_crf++.sh

@@ -1,26 +0,0 @@
-wget "http://downloads.sourceforge.net/crfpp/CRF%2B%2B-0.51.tar.gz?modtime=1215793886&big_mirror=0" -O crf++.tar.gz
-tar xvfz crf++.tar.gz
-rm crf++.tar.gz
-cd CRF*
-PREFIX=$(dirname $PWD)
-
-if [ `uname -m` == 'x86_64' ]; then
-  WITH_PIC='--with-pic';
-else
-  WITH_PIC=''
-fi
-
-./configure  --prefix=$PREFIX --exec-prefix=$PREFIX $WITH_PIC; 
-make install
-cd ruby
-
-ruby extconf.rb  --with-opt-lib=$PREFIX/lib/ --with-opt-include=$PREFIX/include/
-make
-cc -shared -o CRFPP.so CRFPP_wrap.o ../../lib/libcrfpp.a  -L. -L/usr/lib  -L.  -rdynamic -Wl,-export-dynamic    -lruby -lpthread  -lpthread -ldl -lcrypt -lm   -lc -lstdc++
-
-mkdir ../../ruby/
-cp CRFPP.so ../../ruby/
-cd ../../
-rm -Rf CRF* include
-
-

data/install_scripts/get_entrez.sh

@@ -1,4 +0,0 @@
-#!/bin/bash
-
-wget ftp://ftp.ncbi.nih.gov/gene/DATA/gene_info.gz; gunzip gene_info.gz
-wget ftp://ftp.ncbi.nih.gov/gene/DATA/gene2pubmed.gz; gunzip gene2pubmed.gz

data/install_scripts/get_go.sh

@@ -1,4 +0,0 @@
-#!/bin/bash
-
-wget ftp://ftp.geneontology.org/pub/go/ontology/gene_ontology.obo
-wget http://www.geneontology.org/GO_slims/goslim_generic.obo

data/install_scripts/get_polysearch.sh

@@ -1,8 +0,0 @@
-#!/bin/bash
-
-wget http://wishart.biology.ualberta.ca/polysearch/include/disease_IDlist.txt -O disease.txt
-wget http://wishart.biology.ualberta.ca/polysearch/include/organ_ID.txt -O organ.txt
-wget http://wishart.biology.ualberta.ca/polysearch/include/tissue_ID.txt -O tissue.txt
-wget http://wishart.biology.ualberta.ca/polysearch/include/subcellular_localization_ID.txt -O subcellular.txt
-wget http://wishart.biology.ualberta.ca/polysearch/include/drugnames.txt -O drug.txt
-wget http://wishart.biology.ualberta.ca/polysearch/include/HMDBnames.txt -O metabolite.txt

data/install_scripts/ner/Rakefile

@@ -1,206 +0,0 @@
-require 'rbbt/sources/organism'
-require 'rbbt/sources/biocreative'
-require 'rbbt/ner/rner'
-
-require 'progress-monitor'
-
-
-$type = ENV['type'] || 'rner'
-
-#{{{ FEATURES
-
-def BC2GM_features(dataset, outfile)
-  data = Biocreative.BC2GM(dataset)
-
-  fout = File.open(outfile,'w')
-  parser   = NERFeatures.new
-  
-  Progress.monitor("CRFPP Features BC2GM #{ dataset }")
-  data.each{|code, info|
-    text = info[:text]
-    mentions = info[:mentions]
-
-    features = parser.tagged_features(text,mentions)
-
-    features.each{|feat|
-      fout.puts feat.join(" ")
-    }
-    fout.puts
-  }
-  fout.close
-end
-
-def BC2GN_features(dataset, outfile)
-  data = {}
-  Dir.glob(File.join(Rbbt.datadir,'biocreative','BC2GN',dataset,'*.txt')).each{|f|
-    code = File.basename(f).sub(/.txt/,'')
-    data[code] = {}
-    data[code][:text] = Open.read(f)
-  }
-  Open.read(File.join(Rbbt.datadir,'biocreative','BC2GN',dataset,'genelist')).each_line{|l|
-   code, gene, mention = l.chomp.split(/\t/)
-   data[code][:mentions] ||= []
-   data[code][:mentions] << mention
-  }
-
-  fout = File.open(outfile,'w')
-  parser   = NERFeatures.new
-
-  Progress.monitor("CRFPP Features BC2GN #{ dataset }")
-  data.each{|code, info|
-    text = info[:text]
-    mentions = info[:mentions]
-    next if mentions.nil?
-
-    features = parser.tagged_features(text,mentions)
-
-    features.each{|feat|
-      fout.puts feat.join(" ")
-    }
-    fout.puts
-  }
-  fout.close
-end
-
-def org_features(org, outfile)
-  names = Organism.lexicon(org).collect{|code, names|
-    names
-  }.flatten
-
-  fout = File.open(outfile,'w')
-  parser   = NERFeatures.new
-
-  Progress.monitor("CRFPP Features #{ org }")
-  names.each{|name|
-    features = parser.text_features(name, true)
-    features.each{|feat|
-      fout.puts feat.join(" ")
-    }
-    fout.puts
-  }
-  fout.close
-
-
-end
-
-file "data/BC2GM_train.features" do |t|
-  BC2GM_features(:train, 'data/BC2GM_train.features')
-end
-
-file "data/BC2GM_test.features" do |t|
-  BC2GM_features(:test, 'data/BC2GM_test.features')
-end
-file "data/BC2GN_Train.features" do |t|
-  BC2GN_features('Train', 'data/BC2GN_Train.features')
-end
-
-file "data/BC2GN_Test.features" do |t|
-  BC2GN_features('Test', 'data/BC2GN_Test.features')
-end
-
-
-file "data/BC2GM.features" => ['data/BC2GM_train.features','data/BC2GM_test.features'] do |t|
-  Open.write('data/BC2GM.features',Open.read('data/BC2GM_train.features'))
-  Open.append('data/BC2GM.features',Open.read('data/BC2GM_test.features'))
-end
-
-file "data/BC2GN.features" => ['data/BC2GN_Train.features','data/BC2GN_Test.features'] do |t|
-  Open.write('data/BC2GN.features',Open.read('data/BC2GN_Train.features'))
-  Open.append('data/BC2GN.features',Open.read('data/BC2GN_Test.features'))
-end
-
-
-file "data/BC2.features" => ['data/BC2GN.features','data/BC2GM.features'] do |t|
-  Open.write('data/BC2.features',Open.read('data/BC2GM.features'))
-  Open.append('data/BC2.features',Open.read('data/BC2GN.features'))
-end
-
-file "data/train.features" => [
-  #'data/BC2GN.features',
-  'data/BC2GM_train.features'
-  ] do |t|
-  t.prerequisites.each_with_index{|f,i|
-    if i == 0
-      Open.write('data/train.features',Open.read(f))
-    else
-      Open.append('data/train.features',Open.read(f))
-    end
-  }
-end
-
-rule (/data\/(.*).features/) =>  ['data/BC2.features'] do |t|
-  org = File.basename(t.name).sub(/.features$/,'')
-  org_features(org, t.name)
-  Open.append(t.name, Open.read('data/BC2.features'))
-end
-
-
-
-#{{{ MODEL
-rule (/model\/(.*)/) => lambda {|t| t.sub(/model/,'data') + '.features'} do |t|
-  parser = NERFeatures.new 
-  parser.train( t.name.sub(/model/,'data') + '.features', t.name)
-end
-
-task 'clean' do
-  FileUtils.rm Dir.glob("data/*")
-  FileUtils.rm Dir.glob("model/*")
-  FileUtils.rm Dir.glob("results/*")
-
-end
-
-task 'all' do
-  Organism.all.each{|org|
-    Rake::Task["model/#{ org }"].invoke
-  }
-end
-
-task 'default' do
-  if $org
-    FileUtils.rm Dir.glob("**/#{$org}.*") if $force
-    Rake::Task["model/#{$org}"].invoke
-  else
-    Rake::Task['clean'].invoke if $force
-    Rake::Task['all'].invoke
-  end
-end
-
-#{{{ EVALUATE
-
-
-def find(model, type, outfile)
-  ner = Organism.ner(:human,type,:model => model)
- 
-  data = Biocreative.BC2GM(:test)
-
-  fout = File.open(outfile,'w')
-
-  Progress.monitor("Test")
-  data.each{|code,info|
-    text = info[:text]
-    mentions = ner.extract(text)
-
-    mentions.each{|mention|
-      positions = Biocreative.position(text,mention)
-      positions.each{|pos|
-        fout.puts "#{code}|#{pos[0]} #{pos[1]}|#{mention}"
-      }
-    }
-  }
-
-end
-
-
-
-rule (/results\/test$/)  do |t|
-  org = File.basename(t.name)
-
-  if $type == 'rner'
-    Rake::Task['model/train'].invoke
-  end
-  find('model/train',$type,t.name)
-end
-
-rule (/results\/test.eval$/) => ['results/test'] do |t|
-  Biocreative.BC2GM_eval('results/test',:test, 'results/test.eval')
-end