bio-exominer 0.0.3

data/Rakefile

@@ -0,0 +1,58 @@
+# encoding: utf-8
+
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
+  gem.name = "bio-exominer"
+  gem.homepage = "http://github.com/pjotrp/bioruby-exominer"
+  gem.license = "MIT"
+  gem.summary = %Q{Mine publications for gene names}
+  gem.description = %Q{Parse publications for gene names in a fuzzy fashion}
+  gem.email = "pjotr.public01@thebird.nl"
+  gem.authors = ["Pjotr Prins"]
+  # dependencies defined in Gemfile
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+# require 'rspec/core'
+# require 'rspec/core/rake_task'
+# RSpec::Core::RakeTask.new(:spec) do |spec|
+#   spec.pattern = FileList['spec/**/*_spec.rb']
+# end
+
+# RSpec::Core::RakeTask.new(:rcov) do |spec|
+#   spec.pattern = 'spec/**/*_spec.rb'
+#   spec.rcov = true
+# end
+
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.pattern = "spec/*_spec.rb"
+end
+
+require 'cucumber/rake/task'
+Cucumber::Rake::Task.new(:features)
+
+task :default => :test
+
+require 'rdoc/task'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "bio-exominer #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.0.3

data/bin/exominer

@@ -0,0 +1,250 @@
+#!/usr/bin/env ruby
+#
+# BioRuby exominer Plugin BioExominer
+# Author:: Pjotr Prins
+#
+# Copyright (C) 2013,2014 Cuppen Group & Pjotr Prins <pjotr.prins@thebird.nl>
+
+USAGE = "exominer takes a symbol file and parses the piped data for gene symbols"
+
+gempath = File.dirname(File.dirname(__FILE__))
+$: << File.join(gempath,'lib')
+
+VERSION_FILENAME=File.join(gempath,'VERSION')
+version = File.new(VERSION_FILENAME).read.chomp
+
+# print banner
+$stderr.print "exominer #{version} (biogem Ruby #{RUBY_VERSION}) by Pjotr Prins 2014\n"
+
+if ARGV.size == 0
+  print USAGE
+end
+
+require 'bio-exominer'
+require 'optparse'
+
+# Uncomment when using the bio-logger 
+# require 'bio-logger'
+# log = Bio::Log::LoggerPlus.new 'exominer'
+# log.outputters = Bio::Log::Outputter.stderr 
+# Bio::Log::CLI.logger('stderr')
+# Bio::Log::CLI.trace('info')
+
+options = { show_help: false, symbols: []}
+opts = OptionParser.new do |o|
+  o.banner = "Usage: #{File.basename($0)} [options] filename\ne.g.  #{File.basename($0)} -s ncbi_symbols.tab --rdf < test.txt"
+
+  o.on("--rdf", "Generate RDF") do |b|
+    options[:rdf] = true
+  end
+  o.on("--name name", 'Set name of result set') do |name|
+    options[:name] = name
+  end
+  o.on("-s","--symbols fn", 'Symbol file') do |fn|
+    options[:symbols] += [fn]
+  end
+  o.on("--hugo [fn]", 'Hugo symbol file') do |fn|
+    if fn
+      options[:hugo] = fn
+    else
+      options[:hugo] = gempath + '/test/data/input/hugo_symbols'
+    end
+  end
+  o.on("-i","--ignore fn", 'Ignore symbols in fn (NYI)') do |fn|
+    options[:ignore] = fn
+  end
+  o.on("--context [TYPE]",[:off,:line], 'Context parser mode (off,line)') do |context|
+    options[:context] = context
+  end
+  o.on("--doi doi", 'DOI') do |doi|
+    options[:doi] = doi
+  end
+  o.on("--tag string", 'Tag string') do |tag|
+    tags = {}
+    ts = tag.split(/\s?;\s?/)
+    ts.each do |field|
+      key,value = field.strip.split(/\s?=\s?/)
+      tags[key.to_sym] = value
+    end
+    options[:tags] = tags
+  end
+  
+  # Uncomment the following when using the bio-logger 
+  # o.separator ""
+  # o.on("--logger filename",String,"Log to file (default stderr)") do | name |
+  #   Bio::Log::CLI.logger(name)
+  # end
+  #
+  # o.on("--trace options",String,"Set log level (default INFO, see bio-logger)") do | s |
+  #   Bio::Log::CLI.trace(s)
+  # end
+  # 
+  # o.on("-q", "--quiet", "Run quietly") do |q|
+  #   Bio::Log::CLI.trace('error')
+  # end
+  # 
+  # o.on("-v", "--verbose", "Run verbosely") do |v|
+  #   Bio::Log::CLI.trace('info')
+  # end
+  # 
+  # o.on("--debug", "Show debug messages") do |v|
+  #   Bio::Log::CLI.trace('debug')
+  # end
+
+  o.separator ""
+  o.on_tail('-h', '--help', 'display this help and exit') do
+    options[:show_help] = true
+  end
+end
+
+require 'bio-exominer/textparser'
+require 'bio-exominer/symbols'
+require 'yaml'
+
+include BioExominer
+
+begin
+  opts.parse!(ARGV)
+
+  if options[:show_help]
+    print opts 
+    exit 1
+  end
+
+  $stderr.print options
+
+  # Create unique name for the resultset
+  doi  = options[:doi]
+  name = options[:name]
+  name = options[:tags][:name].to_sym if not name and options[:tags] and options[:tags][:name]
+  name = doi if not name and doi
+  name = rand(36**8).to_s(36) if not name # finally a random ID, if nothing else works
+  give_context = options[:context] != :off
+
+  # context may be override by tags
+  options[:context]=options[:tags][:context] if options[:tags][:context]
+
+  # Uncomment when using the bio-logger
+  # Bio::Log::CLI.configure('exominer')
+  # logger = Bio::Log::LoggerPlus['exominer']
+  # Log parsed options and remaining arguments in ARGV
+  # logger.info [options, ARGV]
+
+  $stderr.print "\nLoading text..."
+  buf = ARGF.read
+  $stderr.print "\nTokenizing..."
+  tokens,context =
+  TextParser::tokenize_with_context(buf,options[:context])
+
+  symbol_count = 0
+  alias_count = 0
+  hugo_count = 0
+  hugo_matches = {}
+  symbol_matches = {}   # match symbols
+  alias_matches = {}    # match aliases
+  info = {}             # the main symbol match tracker
+  hugo = {}             # HUGO tracker
+
+  $stderr.print "\nParse symbol files..."
+  parse_symbols = lambda { |symbolfn,is_hugo=false|  
+    # ---- for every symbol file
+    $stderr.print "\nParse symbol file #{symbolfn}..."
+    Symbols::each(symbolfn) do | symbol,aliases,descr |
+      # ---- for every symbol and aliases
+      # $stderr.print "\nHUGO-"+symbol if symbol =~ /L3MBTL/
+      hugo[symbol] = true if is_hugo
+      # alias_count += aliases.size if aliases
+      # ---- If the symbol has a match, and it is not in the list, add it
+      if tokens[symbol] and not info[symbol]
+        symbol_matches[symbol] = tokens[symbol]
+        hugo_matches[symbol] ||= true if is_hugo
+        info[symbol] = { :symbol => symbol, :is_hugo=>is_hugo, :aliases => aliases, :descr => descr, :symbolfn => symbolfn }
+      end  
+      # ---- If an alias has a match and is not in the list, add it
+      if aliases
+        aliases.each do | word |
+          # $stderr.print "\n!!"+word+':'+symbol if word == "L3MBTL"
+          # full = word + ' (' + symbol + ')'
+          if tokens[word] and not info[word]
+            # $stderr.print "\n=="+word+':'+symbol if word == "L3MBTL"
+            hugo_matches[word] ||= true if hugo[word]
+            alias_matches[word] = tokens[word]
+            info[word] = { :symbol => symbol, :is_hugo=>hugo[word], :aliases => aliases, :descr => descr, :symbolfn => symbolfn }
+          end
+        end
+      end
+    end
+  }
+
+  parse_symbols.call(options[:hugo],is_hugo = true) if options[:hugo]
+  options[:symbols].each { |fn| parse_symbols.call(fn) }
+
+  if options[:rdf]
+    # Write RDF!
+    print <<HEADER
+
+# RDF output by bio-exominer https://github.com/pjotrp/bioruby-exominer
+@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
+@prefix dc: <http://purl.org/dc/elements/1.1/> .
+@prefix hgnc: <http://identifiers.org/hgnc.symbol/> .
+@prefix doi: <http://dx.doi.org/> .
+@prefix bibo: <http://purl.org/ontology/bibo/> .
+@prefix ncbigene: <https://www.google.nl/search?q=ncbi+gene+alias+> .
+@prefix : <http://biobeat.org/rdf/exominer/ns#>  .
+
+HEADER
+
+    # Fix DOI with http://www.doi.org/doi_handbook/2_Numbering.html#2.6.2
+    print ":#{name} :doi \"doi:#{doi}\" . \n" if doi
+    print ":#{name} rdf:label \"#{name}\" . \n" 
+    print ":#{name} a \"text resource\" . \n" 
+    print ":#{name} dc:type \"#{options[:type]}\" . \n" if options[:type]
+    if options[:tags]
+      options[:tags].each do | k,v |
+        print ":#{name} :#{k.to_sym} \"#{v}\" .\n"
+      end
+    end
+    print "\n"
+  
+    write_symbol_rdf = lambda { |symbol,freq,is_alias=false| 
+      match_info = info[symbol]
+      match_context = context[symbol].join("; ")
+      symbol1 = match_info[:symbol] 
+      symboluri = Symbols::uri(symbol1,hugo)
+      symbolidentifier = RDF::make_identifier(symbol1)
+      symbol1 = symbolidentifier if not hugo[symbol1]
+      symbolref = name + '_' + symbol1
+      print ":#{symbolref} a #{symboluri} .\n" 
+      # print ":#{symbolref} :alias #{Symbols::uri(symbol,hugo)} .\n" if symbol1 != symbol
+      print ":#{symbolref} :textmatch \"#{symbol}\" .\n" 
+      print ":#{symbolref} dc:partOf :#{name} .\n"
+      print ":#{symbolref} :frequency #{freq} .\n"
+      print ":#{symbolref} :context \"#{match_context.encode("UTF-8").encode(:xml => :text)}\" .\n" if give_context
+      print "#{symboluri} rdf:label \"#{symboluri}\" .\n"
+      print "#{symboluri} rdf:comment \"#{info[symbol][:descr]}\" .\n"
+      print "\n"
+    }
+
+    alias_matches.each do | symbol, freq |
+      write_symbol_rdf.call(symbol,freq,is_alias=true) if not symbol_matches[symbol]
+    end
+    symbol_matches.each do | symbol, freq |
+      write_symbol_rdf.call(symbol,freq,is_alias=false)
+    end
+  else
+    print "\nMatching symbol aliases:\n"
+    alias_matches.sort_by{|k,v| v}.each do | k,v |
+      print v,"\t",(hugo[info[k][:symbol]]?" HUGO":""),"\t",k,"\t",info[k][:symbol],"\t",info[k][:descr],"\n" if not symbol_matches[k]
+    end
+    print "\nMatching symbols:\n"
+    symbol_matches.sort_by{|k,v| v}.each do | k,v |
+      print v,"\t",(hugo[k]?" HUGO":""),"\t",k,"\t",info[k][:descr],"\n"
+    end
+  end
+  # $stderr.print "\nA total of #{symbol_count} symbols and #{alias_count} aliases scanned."
+  $stderr.print "\nThere were #{hugo_matches.size} HUGO matches out of #{hugo.size} symbols."
+  $stderr.print "\nDone!\n"
+rescue OptionParser::InvalidOption => e
+  options[:invalid_argument] = e.message
+end
+

data/bin/hugo_exominer_symbols

@@ -0,0 +1,74 @@
+#! /usr/bin/env ruby
+#
+# Convert NCBI gene_info to symbol file and calculate used letter
+# frequencies. Note: all symbols that are numbers are removed.
+#
+# Copyright (C) 2013 Pjotr Prins <pjotr.prins@thebird.nl> 
+#
+
+require 'yaml'
+
+# Returns true or false
+def valid_symbol s
+  s = s.strip
+  return false if s =~ /^\d+$/  # drop all digit id's
+  $stderr.print "Symbol contains a space! <"+s+">\n" if s =~ /\s/
+  true
+end
+
+module Freq
+  def Freq::to_s freq
+    buf = freq.to_yaml + "\n"
+    freq.keys.sort.each do |c|
+      buf += c
+    end
+    buf
+  end
+end
+
+freq = {}
+
+counter = 0
+
+ARGF.each_line do | line |
+  counter += 1
+  $stderr.print "." if counter % 10_000 == 0
+
+  next if line =~ /^HGNC ID/
+  a = line.strip.split(/\t/)
+  symbol = a[1]
+  next if not valid_symbol(symbol)
+  name = a[2]
+  oldnames = nil
+  oldnames = a[4].strip.split(/\s?,\s?/) if a.size > 4
+  aliases = nil
+  aliases = a[6].strip.split(/\s?,\s?/) if a.size > 6
+  # p [a[4],a[6]]
+  as = []
+  as = aliases if aliases
+  as += oldnames if oldnames
+  as.reject! { |c| c.empty? }
+  aliases = if as.size == 0
+              'NA'
+            else
+              as.uniq.join('|')
+            end
+  descr = a[2]
+  descr = '' if descr == '-'
+  print symbol,"\t",aliases,"\t",descr,"\n"
+  # Add stats
+  cs = symbol.scan(/./)
+  if aliases != 'NA'
+    cs += aliases.scan(/./) - ['|']
+  end
+  cs.each do |c|
+    freq[c] = 0 if not freq[c]
+    freq[c] += 1
+  end
+end
+
+File.open('hugo_exominer_symbols.freq','w') do |f|
+  f.print(Freq::to_s(freq))
+end
+
+

data/bin/ncbi_exominer_symbols

@@ -0,0 +1,79 @@
+#! /usr/bin/env ruby
+#
+# Convert HUGO gene_info to symbol file and calculate used letter
+# frequencies. Note: all symbols that are numbers are removed.
+#
+# Copyright (C) 2013 Pjotr Prins <pjotr.prins@thebird.nl> 
+#
+
+require 'yaml'
+
+# Returns true or false
+def valid_symbol s
+  s = s.strip
+  return false if s =~ /^\d+$/  # drop all digit id's
+  $stderr.print "Symbol contains a space! <"+s+">\n" if s =~ /\s/
+  true
+end
+
+module Freq
+  def Freq::to_s freq
+    buf = freq.to_yaml + "\n"
+    freq.keys.sort.each do |c|
+      buf += c
+    end
+    buf
+  end
+end
+
+freq = {}
+
+counter = 0
+
+ARGF.each_line do | line |
+  counter += 1
+  $stderr.print "." if counter % 100_000 == 0
+
+  next if line =~ /^#/
+  a = line.strip.split(/\t/)
+  symbol = a[2]
+  next if symbol == 'NEWENTRY' or symbol == '-'
+  # Skip gene names that are numbers only
+  next if not valid_symbol(symbol)
+  name = a[3]
+  aliases = a[4]
+  if name != '-' and name != symbol and valid_symbol(name)
+    if aliases == '-'
+      aliases = name
+    else
+      aliases += '|'+name
+    end
+  end
+  aliases =
+    if aliases == '-'
+      'NA'
+    else
+      as1 = aliases.split(/\|/)
+      # Skip gene names that are numbers only
+      as2 = as1.delete_if { |a| not valid_symbol(a) }
+      as2.uniq.join('|')
+    end
+  descr = a[8]
+  descr = '' if descr == '-'
+  print symbol,"\t",aliases,"\t",descr,"\n"
+  # Add stats
+  cs = symbol.scan(/./)
+  if aliases != 'NA'
+    cs += aliases.scan(/./) - ['|']
+  end
+  cs.each do |c|
+    freq[c] = 0 if not freq[c]
+    freq[c] += 1
+  end
+end
+
+File.open('ncbi_exominer_symbols.freq','w') do |f|
+  f.print(Freq::to_s(freq))
+end
+
+