rbbt 1.2.5 → 2.0.0

data/install_scripts/norm/config/cue_default.rb

@@ -1,10 +0,0 @@
-equal    do |w| [w] end
-standard do |w| [w.downcase.split(/\s+/).sort.join("")] end
-cleaned  do |w| [w.downcase.sub(/,.*/,'').sub(/\(.*\)/,'').gsub(/s(?:=\W)/,'')] end
-special  do |w| s = w.split.select{|w| w.is_special?}.collect{|w| w.downcase.sub(/p$/,'')} end
-words    do |w| 
-  w.sub(/(.*)I$/,'\1I \1').
-    scan(/[a-z][a-z]+/i).
-    sort{|a,b| b.length <=> a.length}. 
-    collect{|n| n.downcase}
-end

data/install_scripts/norm/config/tokens_default.rb

@@ -1,86 +0,0 @@
-require 'rbbt/util/misc'
-
-
-plural = Proc.new do |t| t.sub(/s$/,'') end
-
-tokens do
-
-  # Some (possible) single letters first
-  receptor     /^(?:receptor|r)s?$/i 
-  protein      /^(?:protein|p)s?$/i 
-  roman        /^[IV]+$/ 
-  greek_letter do |w| $inverse_greek[w.downcase] != nil end
-  
-
-  # Some words for removal 
-  stopword     do |w|  $stopwords.include?( w.downcase_first)  end
-  gene         /genes?/i 
-  dna
-  cdna
-  rna
-  mrna
-  trna
-  cdna
-  component
-  exon
-  intron
-  domain
-  family
-
-
-  # Important words
-  number       /^(?:\d+[.,]?\d+|\d)$/ 
-  greek        do |w| $greek[w.downcase] != nil end
-  special      do |w| w.is_special? end 
-  promoter
-  similar      /^(homolog.*|like|related|associated)$/ 
-  ase          /ase$/ 
-  in_end       /in$/ 
-end
-
-comparisons do 
-
-  compare.number do |l1,l2|
-      v = 0
-      case
-      when l1.empty? && l2.empty?
-          v = 0
-      when l1.sort.uniq == l2.sort.uniq
-          v = 3
-      when l1.any? && l1[0] == l2[0] 
-          v = -3   
-      when l1.empty? && l2 == ['1'] 
-          v = -5   
-      else 
-          v = -10
-      end
-      v
-  end
-
-  diff.promoter   -10 
-  diff.receptor   -10 
-  diff.similar    -10 
-  diff.capital    -10 
-
-  same.unknown      1
-  miss.unknown      -2 
-  extr.unknown      -2 
-
-  same.greek      1
-  miss.greek      -2 
-  extr.greek      -2 
-
-  same.special    4
-  miss.special    -3 
-  extr.special    -3 
-
-  transform.receptor plural
-  transform.protein plural
-
-  transform.roman do |t| [t.arabic, :number] end
-  transform.greek_letter do |t| [$inverse_greek[t.downcase], :greek] end
-  transform.ase do |t| [t, :special] end
-  transform.in_end do |t| [t, :special] end
-  transform.unknown do |t| [t, (t.length < 4 ? :special : :unknown)] end
-end
-

data/install_scripts/norm/functions.sh

@@ -1,23 +0,0 @@
-#!/bin/bash
-function norm(){
-    organism=$1
-    shift
-    dataset=$1
-    shift
-    ner=$1
-    shift
-
-    CMD="rm results/${organism}_$dataset; rake results/${organism}_$dataset.eval ner=$ner $@ > ${organism}_$dataset.log_$ner; tail results/${organism}_$dataset.eval"
-    echo $CMD
-    $CMD
-}
-
-
-function norm_2(){
-    ner=$1
-    shift
-
-    CMD="rm results/bc2gn; rake results/bc2gn.eval ner=$ner $@ > bc2gn.log_$ner; tail results/bc2gn.eval"
-    echo $CMD
-    $CMD
-}

data/install_scripts/organisms/Ath.Rakefile

@@ -1,55 +0,0 @@
-require __FILE__.sub(/[^\/]*$/,'') + '../rake-include'
-
-$name = "Arabidopsis thaliana"
-
-
-$native_id = "TAIR Locus"
-
-$entrez2native = {
-  :tax =>3702,
-  :fix => proc{|code| code.sub(/^TAIR:/,'')},
-  :check => proc{|code| true },
-}
-
-$lexicon = {
-  :file => {
-    :url => "ftp://ftp.arabidopsis.org/home/tair/Genes/gene_aliases.20100413",
-    :native => 0,
-    :extra => [1,2],
-  },
-}
-
-$identifiers = {
-  :file => {
-    :url => "ftp://ftp.arabidopsis.org/home/tair/Microarrays/Affymetrix/affy_ATH1_array_elements-2009-7-29.txt",
-    :native => 4,
-    :extra => [0],
-    :fields => ["Affymetrix"],
-  },
-  :biomart => {
-    :database => 'athaliana_eg_gene',
-    :main => ['TAIR Locus', 'tair_locus'],
-    :extra => [
-      ['Associated Gene Name' , "external_gene_id"] ,
-      ['Gramene Gene ID' , "ensembl_gene_id"] ,
-      ['RefSeq peptide' , "refseq_peptide"] ,
-      ['Unigene' , "unigene"] ,
-      ['Interpro ID' , "interpro"] ,
-  
-  
-    ],
-    :filter => ['with_tair_locus'], # This is needed as the filter is not with_mgi_id as was expected
-  }
-
-}
-
-$go = {
-  :url =>  "ftp://ftp.arabidopsis.org/home/tair/Ontologies/Gene_Ontology/ATH_GO_GOSLIM.txt",
-  :code => 0,
-  :go   => 5,
-  :pmid => 12,
-}
-
-$query = '("arabidopsis"[MeSH Terms] OR Arabidopsis[Text Word]) AND ((("proteins"[TIAB] NOT Medline[SB]) OR "proteins"[MeSH Terms] OR protein[Text Word]) OR (("genes"[TIAB] NOT Medline[SB]) OR "genes"[MeSH Terms] OR gene[Text Word]))'
-
-

data/install_scripts/organisms/Cal.Rakefile

@@ -1,84 +0,0 @@
-require __FILE__.sub(/[^\/]*$/,'') + '../rake-include'
-
-$name = "Candida albicans"
-
-
-$native_id = "Systematic Name"
-
-$entrez2native = {
-  :tax => 237561,
-  :fix => proc{|code| code.sub(/^CaO/,'orf') },
-  :check => proc{|code| code.match(/^orf/)},
-  :native => 3
-}
-
-$lexicon = {
-  :file => {
-    :url => 'http://hypha.stanford.edu/download/chromosomal_feature_files/chromosomal_feature.tab',
-    :native => 0,
-    :extra => [8,1,2],
-    :exclude => proc{|l| l.match(/^!/) && !l.match(/^orf/)}
-  },
-}
-
-$identifiers = {
-  :file => {
-    :url => 'http://hypha.stanford.edu/download/chromosomal_feature_files/chromosomal_feature.tab',
-    :native => 0,
-    :extra => [8,1,2],
-    :exclude => proc{|l| l.match(/^!/)},
-    :fields => ["GCD ID", "Gene Name", "Gene Alias"]
-  },
-}
-
-$go = {
-  :url => "http://www.candidagenome.org/go/gene_association.cgd.gz",
-  :code => 10,
-  :go   => 4,
-  :pmid => 5,
-  :fix => proc{|l| v = l.split(/\t/); v[10] = (v[10] || "").split('|').first; v.join("\t")}
-}
-
-$query = '"candida albicans"[All Fields] AND ((("proteins"[TIAB] NOT Medline[SB]) OR "proteins"[MeSH Terms] OR protein[Text Word]) OR (("genes"[TIAB] NOT Medline[SB]) OR "genes"[MeSH Terms] OR gene[Text Word])) AND hasabstract[text] AND English[lang]'
-
-####
-
-#Rake::Task['identifiers'].clear
-#file 'identifiers' => ['lexicon'] do |t|
-#  identifiers = {}
-#  if $identifiers[:file]
-#    identifiers = Open.to_hash($identifiers[:file][:url], $identifiers[:file])
-#  end
-#
-#  orf2native = Open.to_hash('lexicon', :native => 1, :extra => 0, :single => true)
-#
-#  translations = {}
-#
-#  Entrez.entrez2native(*$entrez2native.values_at(:tax,:native,:fix,:check)).each{|entrez, orfs|
-#    orfs.each{|orf| 
-#      translations[orf] ||= []
-#      translations[orf] << entrez
-#    }
-#  }
-#
-#  orf2native.each{|orf, native|
-#    next unless identifiers[native]
-#    identifiers[native] << [orf]
-#    if translations[orf]
-#      identifiers[native] << translations[orf]
-#    else
-#      identifiers[native] << []
-#    end
-#
-#  }
-#
-#  header = "#" + [$native_id, 'Gene Name', 'Orf',  "Entrez Gene ID"].uniq.join("\t") + "\n" 
-#  Open.write('identifiers', 
-#             header + 
-#             identifiers.collect{|code, name_lists|
-#               "#{ code }\t" + name_lists.collect{ |names| names.join("|") }.join("\t")
-#             }.join("\n")
-#            )
-#end
-#
-#

data/install_scripts/organisms/Cel.Rakefile

@@ -1,109 +0,0 @@
-require __FILE__.sub(/[^\/]*$/,'') + '../rake-include'
-
-$name = "Caenorhabditis elegans"
-
-
-$native_id = "WormBase ID"
-
-$entrez2native = {
-  :tax => 6239,
-  :fix => proc{|code| code.sub(/^WormBase:/,'')},
-  :check => proc{|code| code.match(/^WBGene/)},
-}
-
-$lexicon = {
-
-  :file =>{
-
-    :url => "ftp://ftp.wormbase.org/pub/wormbase/genomes/elegans/annotations/GO/current.txt.gz",
-    :native => 0,
-    :extra   => [1,2], 
-
-#    :url => "ftp://ftp.wormbase.org/pub/wormbase/genomes/elegans/annotations/gene_ids/current.gz",
-#    :native => 0,
-#    :extra => [2,3,4,5],
-  
-  },
-}
-
-
-$identifiers = {
-
-  :file =>{
-
-    :url => "ftp://ftp.wormbase.org/pub/wormbase/genomes/elegans/annotations/GO/current.txt.gz",
-    :native => 0,
-    :extra   => [1,2], 
-
-#    :url => "ftp://ftp.wormbase.org/pub/wormbase/genomes/elegans/annotations/gene_ids/current.gz",
-#    :native => 0,
-#    :extra => [2,3,4,5],
-  
-  },
-
-  :biomart => {
-    :database => 'celegans_gene_ensembl',
-    :main => ['Entrez Gene ID' , "entrezgene"], 
-    :extra => [ 
-      ['WormBase gene', "wormbase_gene"  ],
-      ['Associated Gene Name ', "external_gene_id"  ],
-      ['WormPep id', "wormpep_id"  ],
-      [ 'Ensembl Gene ID', "ensembl_gene_id"  ],
-      [ 'Ensembl Protein ID', "ensembl_peptide_id"  ],
-      [ 'Protein ID ', "protein_id"  ],
-      [ 'RefSeq Protein ID ', "refseq_peptide"  ],
-      [ 'Unigene ID ', "unigene"  ],
-      [ 'UniProt/SwissProt ID', "uniprot_swissprot"  ],
-      [ 'UniProt/SwissProt Accession', "uniprot_swissprot_accession"  ],
-      ['EMBL (Genbank) ID' , "embl"] , 
-    ],
-    :filter => [],
-  }
-}
-
-$go = {
-  :url => "ftp://ftp.wormbase.org/pub/wormbase/genomes/elegans/annotations/GO/current.txt.gz",
-  :code => 0,
-  :go   => 3,
-  :pmid => 3,
-}
-
-$query = '"caenorhabditis elegans"[MeSH Terms] OR Caenorhabditis elegans[Text Word]'
-##########################
-
-
-module Open
-
-  class << self
-    alias_method :old_read, :read
-
-    def read(url, options = {})
-      content = old_read(url, options)
-
-      if url =~ /GO/
-        return content.gsub(/.*:.*\((GO:\d+)\)/,'\1').gsub(/\nGO/,"|GO").
-                collect{|l|
-                  l = l.sub(/\|/,"\t")
-                  names, gos = l.chomp.split(/\t/)
-
-                  id, name, extra = names.split(/ /)
-                  extra = extra.gsub(/[()]/,'') if extra
-
-                  if gos
-                    gos.split(/\|/).collect{|go|
-                      [id, name, extra, go].join("\t")
-                    }.join("\n")
-                  else
-                    [id, name, extra].join("\t") + "\n"
-                  end
-                }.join("\n")
-      elsif url =~ /gene_ids/
-        return content.gsub(/,/,"\t")
-      else
-        return content
-      end
-
-    end
-  end
-end
-

data/install_scripts/organisms/Hsa.Rakefile

@@ -1,140 +0,0 @@
-require __FILE__.sub(/[^\/]*$/,'') + '../rake-include'
-
-$name = "Homo sapiens"
-
-
-$native_id = "Entrez Gene ID"
-
-$entrez2native = {
-  :tax => 9606,
-  :fix => nil,
-  :check => proc{|code| false},
-}
-
-$lexicon = {
-  :biomart => {
-    :database => 'hsapiens_gene_ensembl',
-    :main => ['Entrez Gene ID' , "entrezgene"], 
-    :extra => [ 
-      [ 'Associated Gene Name' , "external_gene_id"], 
-      [ 'HGNC symbol', "hgnc_symbol"  ],
-      [ 'HGNC automatic gene name', "hgnc_automatic_gene_name"  ],
-      [ 'HGNC curated gene name ', "hgnc_curated_gene_name"  ],
-    ],
-  }
-
-}
-
-$identifiers = {
-  :biomart => {
-    :database => 'hsapiens_gene_ensembl',
-    :main => ['Entrez Gene ID' , "entrezgene"], 
-    :extra => [ 
-      [ 'Ensembl Gene ID', "ensembl_gene_id"  ],
-      [ 'Ensembl Protein ID', "ensembl_peptide_id"  ],
-      [ 'Associated Gene Name', "external_gene_id"  ],
-      [ 'CCDS ID', "ccds"  ],
-      [ 'Protein ID', "protein_id"  ],
-      [ 'RefSeq Protein ID', "refseq_peptide"  ],
-      [ 'Unigene ID', "unigene"  ],
-      [ 'UniProt/SwissProt Accession', "uniprot_swissprot_accession"  ],
-      [ 'HGNC ID', "hgnc_id", 'HGNC'],
-      ['EMBL (Genbank) ID' , "embl"] , 
-
-      # Affymetrix
-      [ 'AFFY HC G110', 'affy_hc_g110' ],
-      [ 'AFFY HG FOCUS', 'affy_hg_focus' ],
-      [ 'AFFY HG U133-PLUS-2', 'affy_hg_u133_plus_2' ],
-      [ 'AFFY HG U133A_2', 'affy_hg_u133a_2' ],
-      [ 'AFFY HG U133A', 'affy_hg_u133a' ],
-      [ 'AFFY HG U133B', 'affy_hg_u133b' ],
-      [ 'AFFY HG U95AV2', 'affy_hg_u95av2' ],
-      [ 'AFFY HG U95B', 'affy_hg_u95b' ],
-      [ 'AFFY HG U95C', 'affy_hg_u95c' ],
-      [ 'AFFY HG U95D', 'affy_hg_u95d' ],
-      [ 'AFFY HG U95E', 'affy_hg_u95e' ],
-      [ 'AFFY HG U95A', 'affy_hg_u95a' ],
-      [ 'AFFY HUGENEFL', 'affy_hugenefl' ],
-      [ 'AFFY HuEx', 'affy_huex_1_0_st_v2' ],
-      [ 'AFFY HuGene', 'affy_hugene_1_0_st_v1' ],
-      [ 'AFFY U133 X3P', 'affy_u133_x3p' ],
-      [ 'Agilent WholeGenome',"agilent_wholegenome" ],
-      [ 'Agilent CGH 44b', 'agilent_cgh_44b' ],
-      [ 'Codelink ID', 'codelink' ],
-      [ 'Illumina HumanWG 6 v2', 'illumina_humanwg_6_v2' ],
-      [ 'Illumina HumanWG 6 v3', 'illumina_humanwg_6_v3' ],
-
-    ],
-    :filter => [],
-  }
-}
-
-$go = {
- :url => "http://cvsweb.geneontology.org/cgi-bin/cvsweb.cgi/go/gene-associations/gene_association.goa_human.gz?rev=HEAD",
- :code => 2,
- :go   => 4,
- :pmid => 5,
-}
-
-$query = '"humans"[MeSH Terms] AND ((("genes"[TIAB] NOT Medline[SB]) OR "genes"[MeSH Terms] OR gene[Text Word]) OR (("proteins"[TIAB] NOT Medline[SB]) OR "proteins"[MeSH Terms] OR protein[Text Word])) AND (hasabstract[text] AND "humans"[MeSH Terms] AND English[lang])'
-##########################
-
-require 'rbbt/util/index'
-
-Rake::Task['gene.go'].clear
-file 'gene.go' => ['identifiers'] do 
-  if File.exists? 'identifiers'
-    require 'rbbt/sources/organism'
-    index = Organism.id_index('Hsa', :other => ['Associated Gene Name'])
-    data = Open.to_hash($go[:url], :native => $go[:code], :extra => $go[:go], :exclude => $go[:exclude])
-
-    data = data.collect{|code, value_lists|
-      code = index[code]
-      [code, value_lists.flatten.select{|ref| ref =~ /GO:\d+/}.collect{|ref| ref.match(/(GO:\d+)/)[1]}]
-    }.select{|p| p[0] && p[1].any?}
-
-    Open.write('gene.go', 
-               data.collect{|p| 
-                 "#{p[0]}\t#{p[1].uniq.join("|")}"
-               }.join("\n")
-              )
-  end
-end
-
-Rake::Task['gene_go.pmid'].clear
-file 'gene_go.pmid' => ['identifiers'] do
-  if File.exists? 'identifiers'
-    index = Index.index('identifiers')
-    data = Open.to_hash($go[:url], :native => $go[:code], :extra => $go[:pmid], :exclude => $go[:exclude])
-
-    data = data.collect{|code, value_lists|
-      code = index[code]
-      [code, value_lists.flatten.select{|ref| ref =~ /PMID:\d+/}.collect{|ref| ref.match(/PMID:(\d+)/)[1]}]
-    }.select{|p| p[0] && p[1].any?}
-
-    Open.write('gene_go.pmid', 
-               data.collect{|p| 
-                 "#{p[0]}\t#{p[1].uniq.join("|")}"
-               }.join("\n")
-              )
-  end
-end
-
-
-Rake::Task['lexicon'].clear
-file 'lexicon' => ['identifiers'] do
-  if File.exists? 'identifiers'
-    require 'rbbt/sources/organism'
-    HGNC_URL = 'http://www.genenames.org/cgi-bin/hgnc_downloads.cgi?title=HGNC+output+data&hgnc_dbtag=on&col=gd_hgnc_id&col=gd_app_sym&col=gd_app_name&col=gd_prev_sym&col=gd_prev_name&col=gd_aliases&col=gd_name_aliases&col=gd_pub_acc_ids&status=Approved&status_opt=2&level=pri&=on&where=&order_by=gd_app_sym_sort&limit=&format=text&submit=submit&.cgifields=&.cgifields=level&.cgifields=chr&.cgifields=status&.cgifields=hgnc_dbtag'
-    names = Open.to_hash(HGNC_URL, :exclude => proc{|l| l.match(/^HGNC ID/)}, :flatten => true)
-    translations = Organism.id_index('Hsa', :native => 'Entrez Gene ID', :other => ['HGNC ID'])
-
-    Open.write('lexicon',
-               names.collect{|code, names|
-                 next unless translations[code]
-                 ([translations[code]] + names).join("\t")
-               }.compact.join("\n")
-               )
-  end
-
-end