unipept 0.7.1 → 0.8.0

data/lib/batch_iterator.rb

@@ -0,0 +1,73 @@
+require 'set'
+
+module Unipept
+  class BatchIterator
+    attr_reader :batch_size
+
+    def initialize(batch_size)
+      @batch_size = batch_size
+    end
+
+    # Splits the input lines into slices, based on the batch_size of the current
+    # command. Executes the given block for each of the batches.
+    #
+    # Supports both normal input and input in the fasta format.
+    #
+    # @input [Iterator] lines An iterator containing the input lines
+    #
+    # @input [lambda] block The code to execute on the slices
+    def iterate(lines, &block)
+      first_line = lines.next rescue return
+      if fasta? first_line
+        fasta_iterator(first_line, lines, &block)
+      else
+        normal_iterator(first_line, lines, &block)
+      end
+    end
+
+    # Checks if the geven line is a fasta header.
+    #
+    # @param [String] line The input line
+    #
+    # @return [Boolean] Whether te input is a fasta header
+    def fasta?(line)
+      line.start_with? '>'
+    end
+
+    private
+
+    # Splits the input lines in fasta format into slices, based on the
+    # batch_size of the current command. Executes the given block for each of
+    # the batches.
+    def fasta_iterator(first_line, next_lines, &block)
+      current_fasta_header = first_line.chomp
+      next_lines.each_slice(batch_size).with_index do |slice, i|
+        fasta_mapper = []
+        input_set = Set.new
+
+        slice.each do |line|
+          line.chomp!
+          if fasta? line
+            current_fasta_header = line
+          else
+            fasta_mapper << [current_fasta_header, line]
+            input_set << line
+          end
+        end
+
+        block.call(input_set.to_a, i, fasta_mapper)
+      end
+    end
+
+    # Splits the input lines into slices, based on the batch_size of the current
+    # command. Executes the given block for each of the batches.
+    def normal_iterator(first_line, next_lines, &block)
+      Enumerator.new do |y|
+        y << first_line
+        loop do
+          y << next_lines.next
+        end
+      end.each_slice(batch_size).with_index(&block)
+    end
+  end
+end

data/lib/batch_order.rb

@@ -0,0 +1,20 @@
+module Unipept
+  class BatchOrder
+    attr_reader :order
+
+    def initialize
+      @order = {}
+      @current = 0
+    end
+
+    # Executes block if it's its turn, queues the block in the other case.
+    def wait(i, &block)
+      @order[i] = block
+      return unless i == @current
+      while order[@current]
+        order.delete(@current).call
+        @current += 1
+      end
+    end
+  end
+end

data/lib/commands/peptfilter.rb

@@ -0,0 +1,118 @@
+module Unipept::Commands
+  class Peptfilter
+    attr_reader :root_command
+
+    @root_command = Cri::Command.define do
+      name 'peptfilter'
+      summary 'Filter peptides based on specific criteria.'
+      usage 'peptfilter [options]'
+      description <<-EOS
+      The peptfilter command filters a list of peptides according to specific criteria. The command expects a list of peptides that are passed to standard input.
+
+      The input should have one peptide per line. FASTA headers are preserved in the output, so that peptides remain bundled.
+      EOS
+      # flag :u, :unique, "filter duplicate peptides."
+      required nil, :minlen, 'only retain tryptic peptides that have at least min (default: 5) amino acids.'
+      required nil, :maxlen, 'only retain tryptic peptides that have at most max (default: 50) amino acids.'
+      required :l, :lacks, 'only retain tryptic peptides that lack all amino acids from the string of residues.'
+      required :c, :contains, 'only retain tryptic peptides that contain all amino acids from the string of residues.'
+      flag :h, :help, 'show help for this command' do |_value, cmd|
+        puts cmd.help
+        exit 0
+      end
+      run do |opts, _args, _cmd|
+        minlen = opts.fetch(:minlen, '5').to_i
+        maxlen = opts.fetch(:maxlen, '50').to_i
+        lacks = opts.fetch(:lacks, '').chars.to_a
+        contains = opts.fetch(:contains, '').chars.to_a
+        $stdin.each_line do |pept|
+          # FASTA headers
+          if pept.start_with? '>'
+            puts pept
+            next
+          end
+
+          pept = pept.chomp
+          puts pept if Peptfilter.filter(pept, minlen, maxlen, lacks, contains)
+        end
+      end
+    end
+
+    @root_command.add_command(Cri::Command.new_basic_help)
+
+    # Invokes the peptfilter command-line tool with the given arguments.
+    #
+    # @param [Array<String>] args An array of command-line arguments
+    #
+    # @return [void]
+    def self.run(args)
+      @root_command.run(args)
+    end
+
+    # Checks if a peptide satisfies the min length, max length, lacks and contains requirements.
+    # Returns true if
+    # - the peptide length is equal or higher than min
+    # - the peptide length is equal or lower than max
+    # - the peptide doesn't contain any of the amino acids in lacks
+    # - the peptide contains all of the amino acids in contains
+    #
+    # @param [String] peptide The peptide to check
+    #
+    # @param [Integer] min The minimal length requirement
+    #
+    # @param [Integer] max The maximal length requirement
+    #
+    # @param [Array<String>] lacks The forbidden amino acids
+    #
+    # @param [Array<String>] contains The required amino acids
+    #
+    # @return [Boolean] true if the peptide satisfies all requirements
+    def self.filter(peptide, min, max, lacks, contains)
+      filter_length(peptide, min, max) &&
+        filter_lacks(peptide, lacks) &&
+        filter_contains(peptide, contains)
+    end
+
+    # Checks if a peptide satisfies the min length and max length requirements.
+    # Returns true if
+    # - the peptide length is equal or higher than min
+    # - the peptide length is equal or lower than max
+    #
+    # @param [String] peptide The peptide to check
+    #
+    # @param [Integer] min The minimal length requirement
+    #
+    # @param [Integer] max The maximal length requirement
+    #
+    # @return [Boolean] true if the peptide satisfies all requirements
+    def self.filter_length(peptide, min, max)
+      peptide.length >= min && peptide.length <= max
+    end
+
+    # Checks if a peptide satisfies lacks requirement.
+    # Returns true if
+    # - the peptide doesn't contain any of the amino acids in lacks
+    #
+    # @param [String] peptide The peptide to check
+    #
+    # @param [Array<String>] lacks The forbidden amino acids
+    #
+    # @return [Boolean] true if the peptide satisfies all requirements
+    def self.filter_lacks(peptide, lacks)
+      (peptide.chars.to_a & lacks).size == 0
+    end
+
+    # Checks if a peptide satisfies the contains requirement.
+    # Returns true if
+    # - the peptide contains all of the amino acids in contains
+    #
+    # @param [String] peptide The peptide to check
+    #
+    # @param [Array<String>] contains The required amino acids
+    #
+    # @return [Boolean] true if the peptide satisfies all requirements
+    def self.filter_contains(peptide, contains)
+      (peptide.chars.to_a & contains).size == contains.size
+    end
+  end
+end

data/lib/commands/prot2pept.rb

@@ -0,0 +1,61 @@
+module Unipept::Commands
+  class Prot2pept
+    attr_reader :root_command
+    attr_reader :valid_formats
+
+    @root_command = Cri::Command.define do
+      name 'prot2pept'
+      summary 'Split protein sequences into peptides.'
+      usage 'prot2pept [options]'
+      description <<-EOS
+      The prot2pept command splits each protein sequence into a list of peptides according to a given cleavage-pattern. The command expects a list of protein sequences that are passed to standard input.
+
+      The input should have either one protein sequence per line or contain a FASTA formatted list of protein sequences. FASTA headers are preserved in the output, so that peptides can be bundled per protein sequence.
+
+      EOS
+      required :p, :pattern, 'specify cleavage-pattern (regex) as the pattern after which the next peptide will be cleaved (default: ([KR])([^P]) for tryptic peptides).'
+      flag :h, :help, 'show help for this command' do |_value, cmd|
+        puts cmd.help
+        exit 0
+      end
+      run do |opts, _args, _cmd|
+        pattern = opts.fetch(:pattern, '([KR])([^P])')
+
+        # decide if we have FASTA input
+        first_char = $stdin.getc
+        $stdin.ungetc(first_char)
+        if first_char == '>'
+          # fasta mode!
+          protein = ''
+          while (line = $stdin.gets)
+            if line.start_with? '>'
+              puts Prot2pept.split(protein, pattern)
+              protein = ''
+              puts line
+            else
+              protein += line.chomp
+            end
+          end
+          puts Prot2pept.split(protein, pattern)
+        else
+          $stdin.each_line do |prot|
+            puts Prot2pept.split(prot, pattern)
+          end
+        end
+      end
+    end
+
+    def self.split(protein, pattern)
+      protein.gsub(/#{pattern}/, "\\1\n\\2").gsub(/#{pattern}/, "\\1\n\\2").split("\n").reject(&:empty?)
+    end
+
+    # Invokes the uniprot command-line tool with the given arguments.
+    #
+    # @param [Array<String>] args An array of command-line arguments
+    #
+    # @return [void]
+    def self.run(args)
+      @root_command.run(args)
+    end
+  end
+end

data/lib/commands/unipept/api_runner.rb

@@ -0,0 +1,199 @@
+module Unipept
+  class Commands::ApiRunner < Cri::CommandRunner
+    attr_reader :configuration
+
+    attr_reader :url
+
+    attr_reader :message_url
+
+    attr_reader :user_agent
+
+    def initialize(args, opts, cmd)
+      super
+      @configuration = Unipept::Configuration.new
+      set_configuration
+
+      @url = "#{@host}/api/v1/#{cmd.name}.json"
+      @message_url = "#{@host}/api/v1/messages.json"
+    end
+
+    # Sets the configurable options of the command line app:
+    # - the host
+    # - the user agent
+    def set_configuration
+      @host = host
+      @user_agent = 'Unipept CLI - unipept ' + Unipept::VERSION
+    end
+
+    # Returns the host. If a value is defined by both an option and the config
+    # file, the value of the option is used.
+    def host
+      # find host in opts first
+      host = options[:host] ? options[:host] : @configuration['host']
+
+      # No host has been set?
+      if host.nil? || host.empty?
+        abort 'WARNING: no host has been set, you can set the host with `unipept config host http://api.unipept.ugent.be/`'
+      end
+
+      # add http:// if needed
+      if host.start_with?('http://') || host.start_with?('https://')
+        host
+      else
+        "http://#{host}"
+      end
+    end
+
+    # Returns an input iterator to use for the request.
+    # - if arguments are given, uses arguments
+    # - if the input file option is given, uses file input
+    # - if none of the previous are given, uses stdin
+    def input_iterator
+      return arguments.each unless arguments.empty?
+      return IO.foreach(options[:input]) if options[:input]
+      $stdin.each_line
+    end
+
+    # Returns the default batch_size of a command.
+    def batch_size
+      100
+    end
+
+    # Constructs a request body (a Hash) for set of input strings, using the
+    # options supplied by the user.
+    def construct_request_body(input)
+      names = selected_fields.empty? || selected_fields.any? { |f| f.to_s.include? 'name' }
+      { input: input,
+        equate_il: options[:equate] == true,
+        extra: options[:all] == true,
+        names: options[:all] == true && names
+      }
+    end
+
+    # Returns an array of regular expressions containing all the selected fields
+    def selected_fields
+      @selected_fields ||= [*options[:select]].map { |f| f.split(',') }.flatten.map { |f| glob_to_regex(f) }
+    end
+
+    # Returns a formatter, based on the format specified in the options
+    def formatter
+      @formatter ||= Unipept::Formatter.new_for_format(options[:format])
+    end
+
+    # Checks if the server has a message and prints it if not empty.
+    # We will only check this once a day and won't print anything if the quiet
+    # option is set or if we output to a file.
+    def print_server_message
+      return if options[:quiet]
+      return unless $stdout.tty?
+      return if recently_fetched?
+      @configuration['last_fetch_date'] = Time.now
+      @configuration.save
+      resp = fetch_server_message
+      puts resp unless resp.empty?
+    end
+
+    # Fetches a message from the server and returns it
+    def fetch_server_message
+      Typhoeus.get(@message_url, params: { version: Unipept::VERSION }).body.chomp
+    end
+
+    # Returns true if the last check for a server message was less than a day
+    # ago.
+    def recently_fetched?
+      last_fetched = @configuration['last_fetch_date']
+      !last_fetched.nil? && (last_fetched + 60 * 60 * 24) > Time.now
+    end
+
+    # Returns a new batch_iterator based on the batch_size
+    def batch_iterator
+      Unipept::BatchIterator.new(batch_size)
+    end
+
+    # Runs the command
+    def run
+      print_server_message
+      hydra = Typhoeus::Hydra.new(max_concurrency: 10)
+      batch_order = Unipept::BatchOrder.new
+
+      batch_iterator.iterate(input_iterator) do |input_slice, batch_id, fasta_mapper|
+        request = Typhoeus::Request.new(
+          @url,
+          method: :post,
+          body: construct_request_body(input_slice),
+          accept_encoding: 'gzip',
+          headers: { 'User-Agent' => @user_agent }
+        )
+
+        request.on_complete do |resp|
+          block = handle_response(resp, batch_id, fasta_mapper)
+          batch_order.wait(batch_id, &block)
+        end
+
+        hydra.queue request
+        hydra.run if batch_id % 200 == 0
+      end
+
+      hydra.run
+    end
+
+    # Saves an error to a new file in the .unipept directory in the users home
+    # directory.
+    def save_error(message)
+      path = error_file_path
+      FileUtils.mkdir_p File.dirname(path)
+      File.open(path, 'w') { |f| f.write message }
+      $stderr.puts "API request failed! log can be found in #{path}"
+    end
+
+    # Write a string to the output defined by the command. If a file is given,
+    # write it to the file. If not, write to stdout
+    def write_to_output(string)
+      if options[:output]
+        File.open(options[:output], 'a') { |f| f.write string }
+      else
+        puts string
+      end
+    end
+
+    private
+
+    def error_file_path
+      File.expand_path(File.join(Dir.home, '.unipept', "unipept-#{Time.now.strftime('%F-%T')}.log"))
+    end
+
+    # Handles the response of an API request.
+    # Returns a block to execute.
+    def handle_response(response, batch_id, fasta_mapper)
+      if response.success?
+        result = filter_result(response.response_body)
+
+        lambda do
+          unless result.empty?
+            write_to_output formatter.header(result, fasta_mapper) if batch_id == 0
+            write_to_output formatter.format(result, fasta_mapper)
+          end
+        end
+      elsif response.timed_out?
+        -> { save_error('request timed out, continuing anyway, but results might be incomplete') }
+      elsif response.code == 0
+        -> { save_error('could not get an http response, continuing anyway, but results might be incomplete' + response.return_message) }
+      else
+        -> { save_error("Got #{response.code}: #{response.response_body}\nRequest headers: #{response.request.options}\nRequest body:\n#{response.request.encoded_body}\n\n") }
+      end
+    end
+
+    # Parses the json_response, wraps it in an array if needed and filters the
+    # fields based on the selected_fields
+    def filter_result(json_response)
+      result = JSON[json_response] rescue []
+      result = [result] unless result.is_a? Array
+      result.map! { |r| r.select! { |k, _v| selected_fields.any? { |f| f.match k } } } unless selected_fields.empty?
+      result
+    end
+
+    def glob_to_regex(string)
+      /^#{string.gsub('*', '.*')}$/
+    end
+  end
+end

data/lib/commands/unipept/config.rb

@@ -0,0 +1,29 @@
+module Unipept
+  class Commands::Config < Cri::CommandRunner
+    def run
+      abort command.help if arguments.size == 0 || arguments.size > 2
+
+      key, value = *arguments
+
+      if arguments.size == 2
+        set_config(key, value)
+        puts key +  ' was set to ' + value
+      else
+        puts get_config(key)
+      end
+    end
+
+    def config
+      @config ||= Unipept::Configuration.new
+    end
+
+    def set_config(key, value)
+      config[key] = value
+      config.save
+    end
+
+    def get_config(key)
+      config[key]
+    end
+  end
+end

data/lib/commands/unipept/pept2lca.rb

@@ -0,0 +1,12 @@
+require_relative 'api_runner'
+module Unipept::Commands
+  class Pept2lca < ApiRunner
+    def batch_size
+      if options[:all]
+        100
+      else
+        1000
+      end
+    end
+  end
+end

data/lib/commands/unipept/pept2prot.rb

@@ -0,0 +1,13 @@
+require_relative 'api_runner'
+
+module Unipept::Commands
+  class Pept2prot < ApiRunner
+    def batch_size
+      if options[:all]
+        5
+      else
+        10
+      end
+    end
+  end
+end

data/lib/{unipept/commands → commands/unipept}/pept2taxa.rb

@@ -1,5 +1,12 @@
 require_relative 'api_runner'
 module Unipept::Commands
   class Pept2taxa < ApiRunner
+    def batch_size
+      if options[:all]
+        5
+      else
+        10
+      end
+    end
   end
 end

data/lib/commands/unipept/taxa2lca.rb

@@ -0,0 +1,18 @@
+require_relative 'api_runner'
+module Unipept::Commands
+  class Taxa2lca < ApiRunner
+    def batch_iterator
+      SimpleBatchIterator.new
+    end
+
+    def batch_size
+      fail 'NOT NEEDED FOR TAXA2LCA'
+    end
+  end
+
+  class SimpleBatchIterator
+    def iterate(input, &block)
+      block.call(input.to_a, 0)
+    end
+  end
+end

data/lib/{unipept/commands → commands/unipept}/taxonomy.rb

@@ -1,5 +1,8 @@
 require_relative 'api_runner'
 module Unipept::Commands
   class Taxonomy < ApiRunner
+    def batch_size
+      100
+    end
   end
 end