unipept 0.8.0 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2079ea0c1b40f957d73ba24d167f5ec468f71c5e
-  data.tar.gz: c9a508bdc4a5293d5801e8bfba74dc8c52d641a8
+  metadata.gz: 4b610ddbb209024a30cd8d1c8a6846a26953869f
+  data.tar.gz: d0d6b1577ca3e5ddc30d334ef0f5cc1500a1b482
 SHA512:
-  metadata.gz: eff7e936edc50c79d0f9d33486b5e9666da22ba937d3989ad19e70a4a3b1b36aee106ef15e7441c6cd422fa8779980dbb7be4f5695bfbf4511566004c425c6a1
-  data.tar.gz: c411d49dc862f28bbde1edafe74f7b2275c0b9735cea484d3fb75f43ecaf4f3d2e51141a409db7ea35b8d9864178f233986731fa1b2e64817be104aefee1afb2
+  metadata.gz: 5c364689109c0f29928078371edc4fd4ba7a9b60999b14537563d63784b7289322af0b2b94d8d6710bfd19508cad49bedde989a1f662ddf94ebc62b81cdc86f6
+  data.tar.gz: 244d132a18e970ba7d13811c0429ece4cc97bc39719336d460bac3664fe7c59dc0d361d8fadbd5cd8b5ed57cad41b62492b019b7250ec070c77e3bad4111bdd3

data/.travis.yml

@@ -5,3 +5,5 @@ rvm:
   - 2.0.0
   - 1.9.3
   - jruby-19mode
+notifications:
+  slack: zeuswpi:yBidVPnaP2JuP2zfa36Ew0E1

data/README.md

@@ -14,7 +14,7 @@ To use the Unipept CLI, Ruby version 1.9.3 or higher needs to be installed. You
 
 ```
 $ ruby -v
-ruby 2.1.1p76 (2014-02-24 revision 45161) [x86_64-darwin12.0]
+ruby 2.1.6p336 (2015-04-13 revision 50298) [x86_64-darwin14.0]
 ```
 
 More information on installing Ruby can be found at https://www.ruby-lang.org/en/installation/
@@ -23,8 +23,8 @@ The Unipept CLI is available as a *gem*. This means it can easily be installed w
 
 ```bash
 $ gem install unipept
-Successfully installed unipept-0.8.0
-Parsing documentation for unipept-0.8.0
+Successfully installed unipept-0.9.0
+Parsing documentation for unipept-0.9.0
 Done installing documentation for unipept after 0 seconds
 1 gem installed
 ```
@@ -33,7 +33,7 @@ After successful installation, the unipept command should be available:
 
 ```bash
 $ unipept -v
-0.8.0
+0.9.0
 ```
 
 The help can be accessed by running `unipept -h`.

data/VERSION

@@ -1 +1 @@
-0.8.0
+0.9.0

data/lib/commands/unipept/api_runner.rb

@@ -4,25 +4,15 @@ module Unipept
 
     attr_reader :url
 
-    attr_reader :message_url
-
     attr_reader :user_agent
 
     def initialize(args, opts, cmd)
       super
       @configuration = Unipept::Configuration.new
-      set_configuration
-
-      @url = "#{@host}/api/v1/#{cmd.name}.json"
-      @message_url = "#{@host}/api/v1/messages.json"
-    end
 
-    # Sets the configurable options of the command line app:
-    # - the host
-    # - the user agent
-    def set_configuration
       @host = host
       @user_agent = 'Unipept CLI - unipept ' + Unipept::VERSION
+      @url = "#{@host}/api/v1/#{cmd.name}.json"
     end
 
     # Returns the host. If a value is defined by both an option and the config
@@ -54,20 +44,39 @@ module Unipept
       $stdin.each_line
     end
 
-    # Returns the default batch_size of a command.
+    def output_writer
+      @output_writer ||= OutputWriter.new(options[:output])
+    end
+
+    # Returns the default default_batch_size of a command.
+    def default_batch_size
+      fail NotImplementedError, 'This must be implemented in a subclass.'
+    end
+
+    # returns the effective batch_size of a command
     def batch_size
-      100
+      if options[:batch]
+        options[:batch].to_i
+      else
+        default_batch_size
+      end
     end
 
-    # Constructs a request body (a Hash) for set of input strings, using the
-    # options supplied by the user.
-    def construct_request_body(input)
-      names = selected_fields.empty? || selected_fields.any? { |f| f.to_s.include? 'name' }
-      { input: input,
-        equate_il: options[:equate] == true,
-        extra: options[:all] == true,
-        names: options[:all] == true && names
-      }
+    # Returns a new batch_iterator based on the batch_size
+    def batch_iterator
+      Unipept::BatchIterator.new(batch_size)
+    end
+
+    def concurrent_requests
+      if options[:parallel]
+        options[:parallel].to_i
+      else
+        10
+      end
+    end
+
+    def queue_size
+      concurrent_requests * 20
     end
 
     # Returns an array of regular expressions containing all the selected fields
@@ -80,43 +89,27 @@ module Unipept
       @formatter ||= Unipept::Formatter.new_for_format(options[:format])
     end
 
-    # Checks if the server has a message and prints it if not empty.
-    # We will only check this once a day and won't print anything if the quiet
-    # option is set or if we output to a file.
-    def print_server_message
-      return if options[:quiet]
-      return unless $stdout.tty?
-      return if recently_fetched?
-      @configuration['last_fetch_date'] = Time.now
-      @configuration.save
-      resp = fetch_server_message
-      puts resp unless resp.empty?
-    end
-
-    # Fetches a message from the server and returns it
-    def fetch_server_message
-      Typhoeus.get(@message_url, params: { version: Unipept::VERSION }).body.chomp
-    end
-
-    # Returns true if the last check for a server message was less than a day
-    # ago.
-    def recently_fetched?
-      last_fetched = @configuration['last_fetch_date']
-      !last_fetched.nil? && (last_fetched + 60 * 60 * 24) > Time.now
-    end
-
-    # Returns a new batch_iterator based on the batch_size
-    def batch_iterator
-      Unipept::BatchIterator.new(batch_size)
+    # Constructs a request body (a Hash) for set of input strings, using the
+    # options supplied by the user.
+    def construct_request_body(input)
+      names = selected_fields.empty? || selected_fields.any? { |f| f.to_s.include? 'name' }
+      { input: input,
+        equate_il: options[:equate] == true,
+        extra: options[:all] == true,
+        names: options[:all] == true && names
+      }
     end
 
     # Runs the command
     def run
-      print_server_message
-      hydra = Typhoeus::Hydra.new(max_concurrency: 10)
+      ServerMessage.new(@host).print unless options[:quiet]
+      hydra = Typhoeus::Hydra.new(max_concurrency: concurrent_requests)
       batch_order = Unipept::BatchOrder.new
+      last_id = 0
 
       batch_iterator.iterate(input_iterator) do |input_slice, batch_id, fasta_mapper|
+        last_id =  batch_id
+
         request = Typhoeus::Request.new(
           @url,
           method: :post,
@@ -131,10 +124,11 @@ module Unipept
         end
 
         hydra.queue request
-        hydra.run if batch_id % 200 == 0
+        hydra.run if batch_id % queue_size == 0
       end
 
       hydra.run
+      batch_order.wait(last_id + 1) { output_writer.write_line formatter.footer }
     end
 
     # Saves an error to a new file in the .unipept directory in the users home
@@ -146,16 +140,6 @@ module Unipept
       $stderr.puts "API request failed! log can be found in #{path}"
     end
 
-    # Write a string to the output defined by the command. If a file is given,
-    # write it to the file. If not, write to stdout
-    def write_to_output(string)
-      if options[:output]
-        File.open(options[:output], 'a') { |f| f.write string }
-      else
-        puts string
-      end
-    end
-
     private
 
     def error_file_path
@@ -166,15 +150,25 @@ module Unipept
     # Returns a block to execute.
     def handle_response(response, batch_id, fasta_mapper)
       if response.success?
-        result = filter_result(response.response_body)
+        handle_success_response(response, batch_id, fasta_mapper)
+      else
+        handle_failed_response(response)
+      end
+    end
+
+    def handle_success_response(response, batch_id, fasta_mapper)
+      result = filter_result(response.response_body)
 
-        lambda do
-          unless result.empty?
-            write_to_output formatter.header(result, fasta_mapper) if batch_id == 0
-            write_to_output formatter.format(result, fasta_mapper)
-          end
+      lambda do
+        unless result.empty?
+          output_writer.write_line formatter.header(result, fasta_mapper) if batch_id == 0
+          output_writer.write_line formatter.format(result, fasta_mapper, batch_id == 0)
         end
-      elsif response.timed_out?
+      end
+    end
+
+    def handle_failed_response(response)
+      if response.timed_out?
         -> { save_error('request timed out, continuing anyway, but results might be incomplete') }
       elsif response.code == 0
         -> { save_error('could not get an http response, continuing anyway, but results might be incomplete' + response.return_message) }

data/lib/commands/unipept/pept2lca.rb

@@ -1,7 +1,7 @@
 require_relative 'api_runner'
 module Unipept::Commands
   class Pept2lca < ApiRunner
-    def batch_size
+    def default_batch_size
       if options[:all]
         100
       else

data/lib/commands/unipept/pept2prot.rb

@@ -2,7 +2,7 @@ require_relative 'api_runner'
 
 module Unipept::Commands
   class Pept2prot < ApiRunner
-    def batch_size
+    def default_batch_size
       if options[:all]
         5
       else

data/lib/commands/unipept/pept2taxa.rb

@@ -1,7 +1,7 @@
 require_relative 'api_runner'
 module Unipept::Commands
   class Pept2taxa < ApiRunner
-    def batch_size
+    def default_batch_size
       if options[:all]
         5
       else

data/lib/commands/unipept/taxa2lca.rb

@@ -5,7 +5,7 @@ module Unipept::Commands
       SimpleBatchIterator.new
     end
 
-    def batch_size
+    def default_batch_size
       fail 'NOT NEEDED FOR TAXA2LCA'
     end
   end

data/lib/commands/unipept/taxonomy.rb

@@ -1,7 +1,7 @@
 require_relative 'api_runner'
 module Unipept::Commands
   class Taxonomy < ApiRunner
-    def batch_size
+    def default_batch_size
       100
     end
   end

data/lib/commands/unipept.rb

@@ -1,9 +1,11 @@
 require 'typhoeus'
 
-require_relative '../formatters'
-require_relative '../configuration'
 require_relative '../batch_order'
 require_relative '../batch_iterator'
+require_relative '../configuration'
+require_relative '../formatters'
+require_relative '../output_writer'
+require_relative '../server_message'
 require_relative '../version'
 
 require_relative 'unipept/config'
@@ -50,6 +52,8 @@ module Unipept
         flag :v, :version, 'displays the version'
         flag :q, :quiet, 'disable service messages'
         option :i, :input, 'read input from file', argument: :required
+        option nil, :batch, 'specify the batch size', argument: :required, hidden: true
+        option nil, :parallel, 'specify the number of parallel requests', argument: :required, hidden: true
         option :o, :output, 'write output to file', argument: :required
         option :f, :format, "define the output format (available: #{Unipept::Formatter.available.join ', ' }) (default: #{Unipept::Formatter.default})", argument: :required
 

data/lib/formatters.rb

@@ -42,7 +42,7 @@ module Unipept
 
     # @return [String] The type of the current formatter
     def type
-      ''
+      fail NotImplementedError, 'This must be implemented in a subclass.'
     end
 
     # Returns the header row for the given sample_data and fasta_mapper. This
@@ -58,7 +58,15 @@ module Unipept
     #
     # @return [String] The header row
     def header(_sample_data, _fasta_mapper = nil)
-      ''
+      fail NotImplementedError, 'This must be implemented in a subclass.'
+    end
+
+    # Returns the footer row. This row is output only once at the end of the
+    # output
+    #
+    # @return [String] The footer row
+    def footer
+      fail NotImplementedError, 'This must be implemented in a subclass.'
     end
 
     # Converts the given input data and corresponding fasta headers to another
@@ -66,14 +74,55 @@ module Unipept
     #
     # @param [Array] data The data we wish to convert
     #
-    # @param [Array<Array<String>>] _fasta_mapper Optional mapping between input
+    # @param [Array<Array<String>>] fasta_mapper Optional mapping between input
     # data and corresponding fasta header. The data is represented as a list
     # containing tuples where the first element is the fasta header and second
     # element is the input data
     #
+    # @param [Boolean] Is this the first output batch?
+    #
     # @return [String] The converted input data
-    def format(data, _fasta_mapper = nil)
-      data
+    def format(data, fasta_mapper = nil, first)
+      data = integrate_fasta_headers(data, fasta_mapper) if fasta_mapper
+      convert(data, first)
+    end
+
+    # Converts the given input data to another format.
+    #
+    # @param [Array] data The data we wish to convert
+    #
+    # @param [Boolean] Is this the first output batch?
+    #
+    # @return [String] The converted input data
+    def convert(_data, _first)
+      fail NotImplementedError, 'This must be implemented in a subclass.'
+    end
+
+    # Integrates the fasta headers into the data object
+    def integrate_fasta_headers(data, fasta_mapper)
+      data_dict = group_by_first_key(data)
+      data = fasta_mapper.map do |header, key|
+        result = data_dict[key]
+        unless result.nil?
+          result = result.map do |row|
+            copy = { fasta_header: header }
+            copy.merge(row)
+          end
+        end
+        result
+      end
+      data.compact.flatten(1)
+    end
+
+    # Groups the data by the first key of each element, for example
+    # [{key1: v1, key2: v2},{key1: v1, key2: v3},{key1: v4, key2: v2}]
+    # to {v1 => [{key1: v1, key2: v2},{key1: v1, key2: v3}], v4 => [{key1: v4, key2: v2}]}
+    #
+    # @param [Array<Hash>] data The data we wish to group
+    #
+    # @return [Hash] The input data grouped by the first key
+    def group_by_first_key(data)
+      data.group_by { |el| el.values.first.to_s }
     end
   end
 
@@ -86,20 +135,24 @@ module Unipept
       'json'
     end
 
-    # Converts the given input data and corresponding fasta headers to JSON.
-    # Currently ignores the fasta_mapper.
+    def header(_data, _fasta_mapper = nil)
+      '['
+    end
+
+    def footer
+      "]\n"
+    end
+
+    # Converts the given input data to the JSON format.
     #
     # @param [Array] data The data we wish to convert
     #
-    # @param [Array<Array<String>>] _fasta_mapper Optional mapping between input
-    # data and corresponding fasta header. The data is represented as a list
-    # containing tuples where the first element is the fasta header and second
-    # element is the input data
+    # @param [Boolean] Is this the first output batch?
     #
-    # @return [String] The input data converted to the JSON format.
-    def format(data, _fasta_mapper = nil)
-      # TODO: add fasta header based on fasta_mapper information
-      data.to_json
+    # @return [String] The converted input data in the JSON format
+    def convert(data, first)
+      output = data.map(&:to_json).join(',')
+      first ? output : ',' + output
     end
   end
 
@@ -133,75 +186,24 @@ module Unipept
       end
     end
 
-    # Converts the given input data and corresponding fasta headers to the csv
-    # format
-    #
-    # @param [Array] data The data we wish to convert
-    #
-    # @param [Array<Array<String>>] fasta_mapper Optional mapping between input
-    # data and corresponding fasta header. The data is represented as a list
-    # containing tuples where the first element is the fasta header and second
-    # element is the input data
-    #
-    # @return [String] The converted input data into the csv format
-    def format(data, fasta_mapper = nil)
-      CSV.generate do |csv|
-        if fasta_mapper
-          format_fasta(csv, data, fasta_mapper)
-        else
-          format_normal(csv, data)
-        end
-      end
-    end
-
-    # Converts the given input data and corresponding fasta headers to the csv
-    # format
-    #
-    # @param [CSV] csv object we write the csv output to
-    #
-    # @param [Array] data The data we wish to convert
-    #
-    # @return [String] The converted input data into the csv format
-    def format_normal(csv, data)
-      data.each do |o|
-        csv << o.values.map { |v| v == ''  ? nil : v }
-      end
+    def footer
+      ''
     end
 
-    # Converts the given input data and corresponding fasta headers to the csv
-    # format
-    #
-    # @param [CSV] csv object we write the csv output to
+    # Converts the given input data to the CSV format.
     #
     # @param [Array] data The data we wish to convert
     #
-    # @param [Array<Array<String>>] fasta_mapper Optional mapping between input
-    # data and corresponding fasta header. The data is represented as a list
-    # containing tuples where the first element is the fasta header and second
-    # element is the input data
+    # @param [Boolean] Is this the first output batch?
     #
-    # @return [String] The converted input data into the csv format
-    def format_fasta(csv, data, fasta_mapper)
-      data_dict = group_by_first_key(data)
-      fasta_mapper.each do |fasta_header, key|
-        next if data_dict[key].nil?
-
-        data_dict[key].each do |r|
-          csv << ([fasta_header] + r.values).map { |v| v == '' ? nil : v }
+    # @return [String] The converted input data in the CSV format
+    def convert(data, _first)
+      CSV.generate do |csv|
+        data.each do |o|
+          csv << o.values.map { |v| v == ''  ? nil : v }
         end
       end
     end
-
-    # Groups the data by the first key of each element, for example
-    # [{key1: v1, key2: v2},{key1: v1, key2: v3},{key1: v4, key2: v2}]
-    # to {v1 => [{key1: v1, key2: v2},{key1: v1, key2: v3}], v4 => [{key1: v4, key2: v2}]]
-    #
-    # @param [Array<Hash>] data The data we wish to Groups
-    #
-    # @return [Hash] The input data grouped by the first key
-    def group_by_first_key(data)
-      data.group_by { |el| el.values.first.to_s }
-    end
   end
 
   class XMLFormatter < Formatter
@@ -215,7 +217,7 @@ module Unipept
 
     class ::Array
       def to_xml(array_name = :array, _item_name = :item)
-        %(<#{array_name} size="#{size}">) + map { |n|n.to_xml(:item) }.join + "</#{array_name}>"
+        %(<#{array_name}>) + map { |n|n.to_xml(:item) }.join + "</#{array_name}>"
       end
     end
 
@@ -233,20 +235,23 @@ module Unipept
       'xml'
     end
 
-    # Converts the given input data and corresponding fasta headers to XML.
-    # Currently ignores the fasta_mapper.
+    def header(_data, _fasta_mapper = nil)
+      '<results>'
+    end
+
+    def footer
+      "</results>\n"
+    end
+
+    # Converts the given input data to the XML format.
     #
     # @param [Array] data The data we wish to convert
     #
-    # @param [Array<Array<String>>] _fasta_mapper Optional mapping between input
-    # data and corresponding fasta header. The data is represented as a list
-    # containing tuples where the first element is the fasta header and second
-    # element is the input data
+    # @param [Boolean] Is this the first output batch?
     #
-    # @return [String] The input data converted to the XML format.
-    def format(data, _fasta_mapper = nil)
-      # TODO: add fasta header based on fasta_mapper information
-      data.to_xml
+    # @return [String] The converted input data in the XML format
+    def convert(data, _first)
+      data.map { |row| '<result>' + row.to_xml + '</result>' }.join('')
     end
   end
 end

data/lib/output_writer.rb

@@ -0,0 +1,13 @@
+module Unipept
+  class OutputWriter
+    attr_reader :output
+
+    def initialize(file)
+      @output = file ? File.open(file, 'a') : $stdout
+    end
+
+    def write_line(line)
+      @output.write line
+    end
+  end
+end

data/lib/server_message.rb

@@ -0,0 +1,45 @@
+require 'typhoeus'
+
+require_relative 'configuration'
+
+module Unipept
+  class ServerMessage
+    attr_reader :message_url
+
+    attr_reader :configuration
+
+    def initialize(host)
+      @message_url = "#{host}/api/v1/messages.json"
+      @configuration = Unipept::Configuration.new
+    end
+
+    # Checks if the server has a message and prints it if not empty.
+    # We will only check this once a day and won't print anything if the quiet
+    # option is set or if we output to a file.
+    def print
+      return unless $stdout.tty?
+      return if recently_fetched?
+      resp = fetch_server_message
+      update_fetched
+      puts resp unless resp.empty?
+    end
+
+    # Fetches a message from the server and returns it
+    def fetch_server_message
+      Typhoeus.get(@message_url, params: { version: Unipept::VERSION }).body.chomp
+    end
+
+    # Returns true if the last check for a server message was less than a day
+    # ago.
+    def recently_fetched?
+      last_fetched = @configuration['last_fetch_date']
+      !last_fetched.nil? && (last_fetched + 60 * 60 * 24) > Time.now
+    end
+
+    # Updates the last checked timestamp
+    def update_fetched
+      @configuration['last_fetch_date'] = Time.now
+      @configuration.save
+    end
+  end
+end