unipept 0.4.2 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: eadba0bac0ca2f5d8dbbbceffe168f26c621cd86
-  data.tar.gz: 5115330bf9a71bfabfa9ac78c852769c9299a51d
+  metadata.gz: 6f20b5baa54b98b05f96b2ed54cdcc4bc6772021
+  data.tar.gz: 0fdb2fc4944c5bfbe8ad76395531b5414d054f22
 SHA512:
-  metadata.gz: c03d75460bbb446335ce708e3602b5edb327434f4e427deb6097231be446e9ecb004bf0f31bcc7034026e685690eb900128ba5b7a6148cc269ea6407397d2ce2
-  data.tar.gz: 8d83ec62c72179ec629777c54dee5f8b41db8ecedcdc3ddd9b5ec82b419c7e99592bc5ab54cd83b85f0a307d8156c7f5a85f4d12168b4036b200dd7aaadbdc75
+  metadata.gz: 182a1babb95902e0cb74d08494748940d02fb77f0c502086eec19809ed2c4b2b8af5a9f190041efb8ac7e30614230971bd5e2e53aec6770587888543b128eafc
+  data.tar.gz: fde9797bcdf7be09258b8010a39dd91b686732c2bb647b1488f51bac96a9c66efddf1061ab1919b35c90f3eb76334f67b84e8a707c26a7dcc630c3e1020053a0

data/VERSION

@@ -1 +1 @@
-0.4.2
+0.5.0

data/bin/peptfilter

@@ -16,6 +16,11 @@ root_cmd = Cri::Command.new_basic_root.modify do
     lacks = opts.fetch(:lacks, "").chars.to_a
     contains = opts.fetch(:contains, "").chars.to_a
     $stdin.each_line do |pept|
+      # FASTA headers
+      if pept.start_with? '>'
+        puts pept
+        next
+      end
       pept = pept.chomp
       length_ok = pept.length >= minlen && pept.length <= maxlen
       lacks_ok = (pept.chars.to_a & lacks).size == 0

data/bin/prot2pept

@@ -8,8 +8,29 @@ root_cmd = Cri::Command.new_basic_root.modify do
   required :p, :pattern, "cleavage pattern to split input protein (default: ([KR])([^P]))"
   run do |opts, args, cmd|
     pattern = opts.fetch(:pattern, "([KR])([^P])")
-    $stdin.each_line do |prot|
-      puts prot.gsub(/#{pattern}/,"\\1\n\\2").gsub(/#{pattern}/, "\\1\n\\2").split("\n").reject(&:empty?)
+    # decide if we have FASTA input
+    fasta_header = gets
+    if fasta_header.start_with? '>'
+      # fasta input, need to join lines
+      while !$stdin.eof?
+        prot = ""
+        # Sometimes you just got to accept this weird and ugly code
+        until $stdin.eof? || (line = gets).start_with?('>')
+          prot += line.chomp
+        end
+        puts fasta_header
+        puts prot.gsub(/#{pattern}/,"\\1\n\\2").gsub(/#{pattern}/, "\\1\n\\2").split("\n").reject(&:empty?)
+
+        fasta_header = line
+      end
+    else
+      # handle our already read line
+      puts fasta_header.gsub(/#{pattern}/,"\\1\n\\2").gsub(/#{pattern}/, "\\1\n\\2").split("\n").reject(&:empty?)
+
+      # we no longer have to join lines as input is now more sane
+      $stdin.each_line do |prot|
+        puts prot.gsub(/#{pattern}/,"\\1\n\\2").gsub(/#{pattern}/, "\\1\n\\2").split("\n").reject(&:empty?)
+      end
     end
   end
 end

data/bin/unipept

@@ -10,213 +10,51 @@ require_relative '../lib/unipept'
 
 Signal.trap("PIPE", "EXIT")
 Signal.trap("INT", "EXIT")
-class ApiRunner < Cri::CommandRunner
-
-  def initialize(args, opts, cmd)
-    super
-    @configuration = Unipept::Configuration.new
-    host = @configuration['host']
-    if host.nil? || host.empty?
-      puts "WARNING: no host has been set, you can set the host with `unipept config host http://localhost:3000/`"
-      exit 1
-    end
-    if !host.start_with? "http://"
-      host = "http://#{host}"
-    end
 
-    @url = "#{host}/api/v1/#{mapping[cmd.name]}.json"
-    @message_url = "#{host}/api/v1/messages.json"
-  end
+module Unipept
+  class Taxa2lca < ApiRunner
 
-  def mapping
-    {'pept2taxa' => 'pept2taxa', 'pept2lca' => 'pept2lca'}
-  end
-
-  def input_iterator
-    if options[:input]
-       File.readlines(options[:input]).each
-    else
-      STDIN.each_line
+    def mapping
+      {"taxa2lca" => "taxa2lca"}
     end
-  end
 
-  def batch_size
-    100
-  end
+    def peptide_iterator(peptides, &block)
+      block.call(peptides.to_a, 0)
+    end
 
-  def url_options(sub_part)
-    filter = options[:select] ? options[:select] : []
-    if filter.empty?
-      names = true
-    else
-      names = filter.any? {|f| /.*name.*/.match f}
+    def batch_size
+      raise "NOT NEEDED FOR TAXA2LCA"
     end
-    {:input => sub_part,
-     :equate_il => options[:equate],
-     :extra => options[:extra],
-     :names => names,
-    }
+
   end
 
-  def get_server_message
-    return if options[:quiet]
-    return unless STDOUT.tty?
-    last_fetched = @configuration['last_fetch_date']
-    if last_fetched.nil? || (last_fetched + 60 * 60 * 24) < Time.now
-      version = File.read(File.join(File.dirname(__FILE__), "..", "VERSION"))
-      puts Typhoeus.get(@message_url, params: {version: version}).body
+  class Pept2prot < ApiRunner
 
-      @configuration['last_fetch_date'] = Time.now
-      @configuration.save
+    def mapping
+      {"pept2prot" => "pept2prot"}
     end
-  end
 
-  def run
-    get_server_message
-
-    formatter = Unipept::Formatter.new_for_format(options[:format])
-    peptides = input_iterator
-
-    filter_list = options[:select] ? options[:select] : []
-    filter_list = filter_list.map {|f| glob_to_regex(f) }
-    output = STDOUT.tty? ? STDOUT : STDERR
-
-    batch_order = Unipept::BatchOrder.new
-
-    printed_header = false
-    result = []
-
-    hydra = Typhoeus::Hydra.new(max_concurrency: 20)
-    num_req = 0
-
-    peptide_iterator(peptides) do |sub_division, i|
-      request = Typhoeus::Request.new(
-        @url,
-        method: :post,
-        body: url_options(sub_division),
-        accept_encoding: "gzip"
-      )
-      request.on_complete do |resp|
-        if resp.timed_out?
-          $stderr.puts "request timed out, continuing anyway, but results might be incomplete"
-        else
-          if resp.success?
-            # if JSON parsing goes wrong
-            sub_result = JSON[resp.response_body] rescue []
-            sub_result = [sub_result] if not sub_result.kind_of? Array
-
-            sub_result.map! {|r| r.select! {|k,v| filter_list.any? {|f| f.match k } } } if ! filter_list.empty?
-
-            if options[:xml]
-              result << sub_result
-            end
-
-            # wait till it's our turn to write
-            batch_order.wait(i) do
-              if ! sub_result.empty?
-                if ! printed_header
-                  write_to_output formatter.header(sub_result)
-                  printed_header = true
-                end
-                write_to_output formatter.format(sub_result)
-              end
-            end
-          else
-            path = File.expand_path(File.join(Dir.home, "unipept.log"))
-            File.open(path, "w") do |f|
-              f.write resp.response_body
-            end
-            $stderr.puts "API request failed! log can be found in #{path}"
+    def download_xml(result)
+      if options[:xml]
+        FileUtils.mkdir_p(options[:xml])
+        result.first.each do |prot|
+          File.open(options[:xml] + "/#{prot['uniprot_id']}.xml", "wb") do |f|
+            f.write Typhoeus.get("http://www.uniprot.org/uniprot/#{prot['uniprot_id']}.xml").response_body
           end
         end
       end
-      hydra.queue request
-
-      num_req += 1
-      if num_req % 200 == 0
-        hydra.run
-      end
-
     end
 
-    hydra.run
-
-    begin
-      download_xml(result)
-    rescue
-      STDERR.puts "Something went wrong while downloading xml information! please check the output"
+    def batch_size
+      10
     end
-
-  end
-
-  def write_to_output(string)
-    if options[:output]
-      File.open(options[:output], 'a') do |f|
-        f.write string
-      end
-    else
-      puts string
-    end
-  end
-
-
-  def download_xml(result)
-    if options[:xml]
-      File.open(options[:xml] + ".xml", "wb") do |f|
-        f.write Typhoeus.get("http://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=taxonomy&id=#{result.first.map{|h| h['taxon_id'] }.join(",")}&retmode=xml").response_body
-      end
-    end
-  end
-
-  def peptide_iterator(peptides, &block)
-    peptides.each_slice(batch_size).with_index(&block)
-  end
-
-  private
-
-  def glob_to_regex(glob_string)
-    # only implement * -> . for now
-    Regexp.new glob_string.gsub("*", ".*")
-  end
-end
-
-class Taxa2lca < ApiRunner
-
-  def mapping
-    {"taxa2lca" => "taxa2lca"}
-  end
-
-  def peptide_iterator(peptides, &block)
-    block.call(peptides.to_a, 0)
-  end
-
-  def batch_size
-    raise "NOT NEEDED FOR TAXA2LCA"
   end
 
-end
-
-class Pept2prot < ApiRunner
-
-  def mapping
-    {"pept2prot" => "pept2prot"}
-  end
-
-  def download_xml(result)
-    if options[:xml]
-      FileUtils.mkdir_p(options[:xml])
-      result.first.each do |prot|
-        File.open(options[:xml] + "/#{prot['uniprot_id']}.xml", "wb") do |f|
-          f.write Typhoeus.get("http://www.uniprot.org/uniprot/#{prot['uniprot_id']}.xml").response_body
-        end
-      end
+  class Taxonomy < ApiRunner
+    def mapping
+      {"taxonomy" => "taxonomy"}
     end
   end
-
-  def batch_size
-    10
-  end
-
 end
 
 root_cmd = Cri::Command.new_basic_root.modify do
@@ -227,6 +65,9 @@ root_cmd = Cri::Command.new_basic_root.modify do
   option :o, :output, "output file", :argument => :required
   option :f, :format, "output format (available: #{Unipept::Formatter.available.join "," }) (default: #{Unipept::Formatter.default})", :argument => :required
 
+  # Configuration options
+  option nil, "config-host", "Override host setting", argument: :required
+
   run do |opts, args, cmd|
     if opts[:version]
       puts File.read(File.join(File.dirname(__FILE__), "..", "VERSION"))
@@ -260,7 +101,7 @@ root_cmd.define_command('pept2taxa') do
   option :a, :extra, "Show full lineage"
   option :x, :xml, "Download taxonomy from NCBI as xml (specify output filename)", :argument => :required
 
-  runner ApiRunner
+  runner Unipept::ApiRunner
 end
 
 root_cmd.define_command('pept2lca') do
@@ -273,7 +114,7 @@ root_cmd.define_command('pept2lca') do
   option :s, :select, "select the attributes", :argument => :required, :multiple => true
   option :a, :extra, "Show full lineage"
 
-  runner ApiRunner
+  runner Unipept::ApiRunner
 end
 
 root_cmd.define_command('taxa2lca') do
@@ -285,7 +126,7 @@ root_cmd.define_command('taxa2lca') do
   option :s, :select, "select the attributes", :argument => :required, :multiple => true
   option :a, :extra, "Show full lineage"
 
-  runner Taxa2lca
+  runner Unipept::Taxa2lca
 end
 
 root_cmd.define_command('pept2prot') do
@@ -299,7 +140,19 @@ root_cmd.define_command('pept2prot') do
   option :x, :xml, "download uniprot record in specified directory", :argument => :required
   flag :a, :extra, "include all information. WARNING: will take much longer!"
 
-  runner Pept2prot
+  runner Unipept::Pept2prot
+end
+
+root_cmd.define_command('taxonomy') do
+  usage       'taxonomy [options]'
+  aliases     :tax
+  summary     'Give NCBI taxonomy information on given input taxon ids'
+  description 'Returns information for each input taxon id'
+
+  option :s, :select, "select the attributes", :argument => :required, :multiple => true
+  flag :a, :extra, "include all information. WARNING: will take much longer!"
+
+  runner Unipept::Taxonomy
 end
 
 root_cmd.run(ARGV)

data/lib/unipept/api_runner.rb

@@ -0,0 +1,217 @@
+module Unipept
+  class ApiRunner < Cri::CommandRunner
+
+    def initialize(args, opts, cmd)
+      super
+      @configuration = Unipept::Configuration.new
+
+      set_configuration
+
+      @url = "#{@host}/api/v1/#{mapping[cmd.name]}.json"
+      @message_url = "#{@host}/api/v1/messages.json"
+    end
+
+    def set_configuration
+      # find host in opts first
+      if options[:'config-host']
+        host = options[:'config-host']
+      else
+        host = @configuration['host']
+      end
+
+      # No host has been set?
+      if host.nil? || host.empty?
+        puts "WARNING: no host has been set, you can set the host with `unipept config host http://localhost:3000/`"
+        exit 1
+      end
+      if !host.start_with? "http://"
+        host = "http://#{host}"
+      end
+
+      @host = host
+    end
+
+    def mapping
+      {'pept2taxa' => 'pept2taxa', 'pept2lca' => 'pept2lca'}
+    end
+
+    def input_iterator
+      # Argument over file input over stdin
+      if !arguments.empty?
+        arguments.each
+      else
+        if options[:input]
+          IO.foreach(options[:input])
+        else
+          STDIN.each_line
+        end
+      end
+    end
+
+    def batch_size
+      100
+    end
+
+    def url_options(sub_part)
+      filter = options[:select] ? options[:select] : []
+      if filter.empty?
+        names = true
+      else
+        names = filter.any? {|f| /.*name.*/.match f}
+      end
+      {:input => sub_part,
+       :equate_il => options[:equate],
+       :extra => options[:extra],
+       :names => names,
+      }
+    end
+
+    def get_server_message
+      return if options[:quiet]
+      return unless STDOUT.tty?
+      last_fetched = @configuration['last_fetch_date']
+      if last_fetched.nil? || (last_fetched + 60 * 60 * 24) < Time.now
+        version = File.read(File.join(File.dirname(__FILE__), "..", "VERSION"))
+        puts Typhoeus.get(@message_url, params: {version: version}).body
+
+        @configuration['last_fetch_date'] = Time.now
+        @configuration.save
+      end
+    end
+
+    def run
+      get_server_message
+
+      formatter = Unipept::Formatter.new_for_format(options[:format])
+      peptides = input_iterator
+
+      filter_list = options[:select] ? options[:select] : []
+      filter_list = filter_list.map {|f| glob_to_regex(f) }
+      output = STDOUT.tty? ? STDOUT : STDERR
+
+      batch_order = Unipept::BatchOrder.new
+
+      printed_header = false
+      result = []
+
+      hydra = Typhoeus::Hydra.new(max_concurrency: 10)
+      num_req = 0
+
+      peptide_iterator(peptides) do |sub_division, i, fasta_mapper|
+        request = Typhoeus::Request.new(
+          @url,
+          method: :post,
+          body: url_options(sub_division),
+          accept_encoding: "gzip"
+        )
+        request.on_complete do |resp|
+          if resp.timed_out?
+            $stderr.puts "request timed out, continuing anyway, but results might be incomplete"
+          else
+            if resp.success?
+              # if JSON parsing goes wrong
+              sub_result = JSON[resp.response_body] rescue []
+              sub_result = [sub_result] if not sub_result.kind_of? Array
+
+              sub_result.map! {|r| r.select! {|k,v| filter_list.any? {|f| f.match k } } } if ! filter_list.empty?
+
+              if options[:xml]
+                result << sub_result
+              end
+
+              # wait till it's our turn to write
+              batch_order.wait(i) do
+                if ! sub_result.empty?
+                  if ! printed_header
+                    write_to_output formatter.header(sub_result, fasta_mapper)
+                    printed_header = true
+                  end
+                  write_to_output formatter.format(sub_result, fasta_mapper)
+                end
+              end
+            else
+              save_error(resp.response_body)
+            end
+          end
+        end
+        hydra.queue request
+
+        num_req += 1
+        if num_req % 200 == 0
+          hydra.run
+        end
+
+      end
+
+      hydra.run
+
+      begin
+        download_xml(result)
+      rescue
+        STDERR.puts "Something went wrong while downloading xml information! please check the output"
+      end
+
+    end
+
+    def save_error(message)
+      path = File.expand_path(File.join(Dir.home, ".unipept", "unipept-#{Time.now.strftime("%F-%T")}.log"))
+      FileUtils.mkdir_p File.dirname(path)
+      File.open(path, "w") do |f|
+        f.write message
+      end
+      $stderr.puts "API request failed! log can be found in #{path}"
+    end
+
+    def write_to_output(string)
+      if options[:output]
+        File.open(options[:output], 'a') do |f|
+          f.write string
+        end
+      else
+        puts string
+      end
+    end
+
+
+    def download_xml(result)
+      if options[:xml]
+        File.open(options[:xml] + ".xml", "wb") do |f|
+          f.write Typhoeus.get("http://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=taxonomy&id=#{result.first.map{|h| h['taxon_id'] }.join(",")}&retmode=xml").response_body
+        end
+      end
+    end
+
+    def peptide_iterator(peptides, &block)
+      first = peptides.first
+      if first.start_with? '>'
+        # FASTA MODE ENGAGED
+        fasta_header = first
+        peptides.each_slice(batch_size).with_index do |sub,i|
+          fasta_mapper = {}
+          sub.map! {|s| s.chomp}
+          j = 0
+          while j < sub.size
+            if sub[j].start_with? '>'
+              fasta_header = sub[j]
+            else
+              fasta_mapper[sub[j]] = fasta_header
+            end
+            j += 1
+          end
+          sub -= fasta_mapper.values.uniq
+          block.call(sub, i, fasta_mapper)
+        end
+
+      else
+        peptides.each_slice(batch_size).with_index(&block)
+      end
+    end
+
+    private
+
+    def glob_to_regex(glob_string)
+      # only implement * -> . for now
+      Regexp.new glob_string.gsub("*", ".*")
+    end
+  end
+end

data/lib/unipept/formatters.rb

@@ -30,7 +30,7 @@ module Unipept
     end
 
     # JSON formatted data goes in, something other comes out
-    def format(data)
+    def format(data, fasta_mapper = nil)
       data
     end
   end
@@ -49,23 +49,40 @@ module Unipept
 
     register :csv
 
-    def header(data)
+    def header(data, fasta_mapper = nil)
       CSV.generate do |csv|
         first = data.first
         if first.kind_of? Array
           first = first.first
         end
-        csv << first.keys.map(&:to_s) if first
+        if fasta_mapper
+          csv << (['fasta_header'] + first.keys).map(&:to_s) if first
+        else
+          csv << first.keys.map(&:to_s) if first
+        end
+
       end
     end
 
-    def format(data)
+    def format(data, fasta_mapper = nil)
       CSV.generate do |csv|
         data.each do |o|
           if o.kind_of? Array
-            o.each {|h| csv << h.values.map { |v| v == ""  ? nil : v }}
+            o.each do |h|
+              if fasta_mapper
+                extra_key = [fasta_mapper[h.values.first]]
+                csv << (extra_key + h.values).map { |v| v == ""  ? nil : v }
+              else
+                csv << h.values.map { |v| v == ""  ? nil : v }
+              end
+            end
           else
-            csv << o.values.map { |v| v == "" ? nil : v }
+            if fasta_mapper
+              extra_key = [fasta_mapper[o.values.first]]
+              csv << (extra_key + o.values).map { |v| v == ""  ? nil : v }
+            else
+              csv << o.values.map { |v| v == ""  ? nil : v }
+            end
           end
         end
       end

data/lib/unipept.rb

@@ -1,6 +1,7 @@
 require_relative 'unipept/formatters'
 require_relative 'unipept/configuration'
 require_relative 'unipept/batch_order'
+require_relative 'unipept/api_runner'
 
 module Unipept
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: unipept
 version: !ruby/object:Gem::Version
-  version: 0.4.2
+  version: 0.5.0
 platform: ruby
 authors:
 - Toon Willems
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-14 00:00:00.000000000 Z
+date: 2014-05-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: shoulda
@@ -131,6 +131,7 @@ files:
 - bin/unipept
 - bin/uniprot
 - lib/unipept.rb
+- lib/unipept/api_runner.rb
 - lib/unipept/batch_order.rb
 - lib/unipept/configuration.rb
 - lib/unipept/formatters.rb