aspera-cli 4.27.2 → 4.27.3

data/lib/aspera/agent/connect.rb

@@ -6,6 +6,8 @@ require 'aspera/products/other'
 require 'aspera/rest'
 require 'aspera/environment'
 require 'securerandom'
+require 'aspera/rainbow'
+using Rainbow
 
 module Aspera
   module Agent
@@ -21,9 +23,9 @@ module Aspera
         # agent_params must contain 'app_id'; the Connect URL is auto-discovered.
         # @return [Hash] normalized status hash
         def transfer_status(transfer_id, agent_params)
-          connect_api = Rest.new(
+          connect_api = Rest::Client.new(
             base_url: "#{connect_api_url}/v5/connect",
-            headers:  {'Origin' => RestParameters.instance.user_agent}
+            headers:  {'Origin' => Rest::Parameters.instance.user_agent}
           )
           tr_info = connect_api.create("transfers/info/#{transfer_id}", {'aspera_connect_settings' => {'app_id' => agent_params['app_id']}})
           transfer = tr_info['transfer_info']
@@ -72,9 +74,9 @@ module Aspera
           # raise exception if connect not started and file does not exist
           connect_url = self.class.connect_api_url
           Log.log.debug { "found: #{connect_url}" }
-          @connect_api = Rest.new(
+          @connect_api = Rest::Client.new(
             base_url: "#{connect_url}/v5/connect", # could use v6 also now
-            headers: {'Origin' => RestParameters.instance.user_agent}
+            headers: {'Origin' => Rest::Parameters.instance.user_agent}
           )
           connect_info = @connect_api.read('info/version')
           Log.log.debug('Connect was reached') if method_index > 0

data/lib/aspera/agent/desktop.rb

@@ -24,7 +24,7 @@ module Aspera
         # @return [Hash] normalized status hash
         def transfer_status(transfer_id, agent_params)
           app_id = agent_params['application_id']
-          client = Aspera::JsonRpc::Client.new(Aspera::Rest.new(base_url: desktop_api_url))
+          client = Aspera::JsonRpc::Client.new(Aspera::Rest::Client.new(base_url: desktop_api_url))
           raw = client.get_transfer(app_id: app_id, transfer_id: transfer_id)
           normalize_status(raw['status'], bytes: raw['bytes_written'].to_i, error: raw['error_desc'])
         end
@@ -70,7 +70,7 @@ module Aspera
         begin
           # curl 'http://127.0.0.1:33024/' -X POST -H 'content-type: application/json' --data-raw '{"jsonrpc":"2.0","params":[],"id":999999,"method":"rpc.discover"}'
           # https://playground.open-rpc.org/?schemaUrl=http://127.0.0.1:33024
-          @client_app_api = Aspera::JsonRpc::Client.new(Aspera::Rest.new(base_url: aspera_client_api_url))
+          @client_app_api = Aspera::JsonRpc::Client.new(Aspera::Rest::Client.new(base_url: aspera_client_api_url))
           client_info = @client_app_api.get_info
           Log.dump(:client_version, client_info)
           Log.log.debug('Client was reached') if method_index > 0

data/lib/aspera/agent/direct.rb

@@ -14,6 +14,8 @@ require 'socket'
 require 'securerandom'
 require 'shellwords'
 require 'English'
+require 'aspera/rainbow'
+using Rainbow
 
 module Aspera
   module Agent
@@ -258,7 +260,7 @@ module Aspera
           @resume_policy.execute_with_resume do
             start_and_monitor_process(session: session, exec_spec: session[:exec_spec])
           end
-          Log.log.debug('transfer ok'.bg_green)
+          Log.log.debug('transfer ok'.bg(:green))
         rescue => e
           session[:error] = e
           raise if Log.log.debug? || !e.is_a?(Transfer::Error)

data/lib/aspera/agent/node.rb

@@ -24,7 +24,7 @@ module Aspera
           else
             rest_params[:auth] = {type: :basic, username: agent_params['username'], password: agent_params['password']}
           end
-          node_api = Rest.new(**rest_params)
+          node_api = Rest::Client.new(**rest_params)
           data = node_api.read("ops/transfers/#{transfer_id}") || {'status' => 'unknown'}
           normalize_status(data)
         end
@@ -71,7 +71,7 @@ module Aspera
             password: password
           }
         end
-        @node_api = Rest.new(**rest_params)
+        @node_api = Rest::Client.new(**rest_params)
         # TODO: currently only supports one transfer. This is bad shortcut. but ok for CLI.
         @transfer_id = nil
       end
@@ -149,7 +149,7 @@ module Aspera
 
       # used internally to ensure node api is set before using.
       def node_api_
-        Aspera.assert(!@node_api.nil?, 'Before using this object, set the node_api attribute to a Aspera::Rest object')
+        Aspera.assert(!@node_api.nil?, 'Before using this object, set the node_api attribute to a Aspera::Rest::Client object')
         return @node_api
       end
     end

data/lib/aspera/api/alee.rb

@@ -4,7 +4,7 @@ require 'aspera/api/aoc'
 module Aspera
   module Api
     # Aspera License Entitlement Engine API client
-    class Alee < Aspera::Rest
+    class Alee < Aspera::Rest::Client
       def initialize(entitlement_id, customer_id, api_domain: AoC::SAAS_DOMAIN_PROD, version: 'v1')
         super(
           base_url: "https://api.#{api_domain}/metering/#{version}",

data/lib/aspera/api/aoc.rb

@@ -7,16 +7,18 @@ require 'aspera/hash_ext'
 require 'aspera/data_repository'
 require 'aspera/transfer/spec'
 require 'aspera/api/node'
-require 'aspera/rest_list'
+require 'aspera/rest/list'
 require 'base64'
+require 'aspera/rainbow'
+using Rainbow
 
 module Aspera
   module Api
     # information relative to Files or Packages App
 
     # Aspera on Cloud API client
-    class AoC < Rest
-      include RestList
+    class AoC < Rest::Client
+      include Rest::List
 
       module AppType
         FILES = 'files'
@@ -140,7 +142,7 @@ module Aspera
         # @param url [String] URL of AoC public link
         # @return [Hash{Symbol => String, Hash}] Information about public link, or nil if not a public link
         def link_info(url)
-          final_uri = Rest.new(base_url: url, redirect_max: MAX_AOC_URL_REDIRECT).call(operation: 'GET', ret: :resp).uri
+          final_uri = Rest::Client.new(base_url: url, redirect_max: MAX_AOC_URL_REDIRECT).call(operation: 'GET', ret: :resp).uri
           Log.dump(:final_uri, final_uri, level: :trace1)
           org_domain = split_org_domain(final_uri)
           if (m = final_uri.path.match(%r{/oauth2/([^/]+)/login$}))
@@ -202,8 +204,8 @@ module Aspera
           Aspera.assert(block_given?, 'block required for call_paging')
           # set default large page if user does not specify own parameters. AoC Caps to 1000 anyway
           query['per_page'] = 1000 unless query.key?('per_page')
-          max_items = query.delete(RestList::MAX_ITEMS)
-          max_pages = query.delete(RestList::MAX_PAGES)
+          max_items = query.delete(Rest::List::MAX_ITEMS)
+          max_pages = query.delete(Rest::List::MAX_PAGES)
           item_list = []
           total_count = nil
           current_page = query['page']
@@ -225,9 +227,9 @@ module Aspera
             break if !max_items.nil? && item_list.count >= max_items
             break if !max_pages.nil? && page_count >= max_pages
             break if total_count&.<=(item_list.count)
-            RestParameters.instance.spinner_cb.call("#{item_list.count} / #{total_count}") unless total_count.eql?(item_list.count.to_s)
+            Rest::Parameters.instance.spinner_cb.call("#{item_list.count} / #{total_count}") unless total_count.eql?(item_list.count.to_s)
           end
-          RestParameters.instance.spinner_cb.call(action: :success)
+          Rest::Parameters.instance.spinner_cb.call(action: :success)
           item_list = item_list[0..max_items - 1] if !max_items.nil? && item_list.count > max_items
           return {items: item_list, total: total_count}
         end
@@ -394,7 +396,7 @@ module Aspera
         @cache_user_info =
           begin
             read('self')
-          rescue Aspera::RestCallError => e
+          rescue Aspera::Rest::CallError => e
             raise if exception || e.message.include?('invalid_grant')
             Log.log.debug { "Ignoring error: (#{e.class}) #{e}" }
             {}
@@ -496,9 +498,9 @@ module Aspera
           }
         else
           # OAuth bearer token
-          node_params[:auth] = auth_params.clone
-          node_params[:auth][:params] ||= {}
-          node_params[:auth][:params][:scope] = Node.token_scope(node_info['access_key'], scope)
+          # copy nested params, so that auth params of this API are not modified
+          node_params[:auth] = auth_params.merge(params: auth_params[:params]&.dup || {})
+          node_params[:auth][:params][:scope] = Node::Scope.join(access_key: node_info['access_key'], scope: scope)
           node_params[:auth][:params][:owner_access] = true if scope.eql?(Node::Scope::ADMIN)
           # Special header required for bearer token only
           node_params[:headers] = {Node::HEADER_X_ASPERA_ACCESS_KEY => node_info['access_key']}

data/lib/aspera/api/ats.rb

@@ -6,7 +6,7 @@ require 'aspera/rest'
 module Aspera
   module Api
     # Aspera Transfer Service API client without authentication
-    class Ats < Aspera::Rest
+    class Ats < Aspera::Rest::Client
       SERVICE_BASE_URL = 'https://ats.aspera.io'
       # currently supported clouds
       # Note to Aspera: shall be an API call

data/lib/aspera/api/cos_node.rb

@@ -22,7 +22,7 @@ module Aspera
             Aspera.assert(service_credentials.key?(field)) { "service_credentials must have a field: #{field}" }
           end
           # read endpoints from service provided in service credentials
-          endpoints = Aspera::Rest.new(base_url: service_credentials['endpoints']).read('')
+          endpoints = Aspera::Rest::Client.new(base_url: service_credentials['endpoints']).read('')
           Log.dump(:endpoints, endpoints)
           endpoint = endpoints.dig('service-endpoints', 'regional', bucket_region, 'public', bucket_region)
           Aspera.assert(!endpoint.nil?) { "no such region: #{bucket_region}" }
@@ -40,7 +40,7 @@ module Aspera
         endpoint = "https://#{endpoint}" unless endpoint.start_with?('http')
         @auth_url = auth_url
         @api_key = api_key
-        s3_api = Aspera::Rest.new(
+        s3_api = Aspera::Rest::Client.new(
           base_url:       endpoint,
           not_auth_codes: %w[401 403], # error codes when not authorized
           headers:        {'ibm-service-instance-id' => instance_id},

data/lib/aspera/api/faspex.rb

@@ -3,7 +3,7 @@
 require 'aspera/assert'
 require 'aspera/rest'
 require 'aspera/oauth/base'
-require 'aspera/rest_list'
+require 'aspera/rest/list'
 require 'digest'
 
 module Aspera
@@ -15,6 +15,7 @@ module Aspera
     # @param context        [String] The `context` query parameter in public link
     # @param redirect_uri   [String] URI of web UI login
     # @param path_authorize [String] Path to provide passcode
+    # @param base_params    [Hash] Parameters for base class
     def initialize(
       context:,
       redirect_uri:,
@@ -48,18 +49,18 @@ module Aspera
       raise Error, info['action_message'] if info['action_message']
       Aspera.assert(info['code'], 'Missing code in answer')
       # Exchange code for token
-      return create_token_call(base_params.merge(
+      create_token_base(
         grant_type:   'authorization_code',
         code:         info['code'],
         redirect_uri: @redirect_uri
-      ))
+      )
     end
   end
   OAuth::Factory.instance.register_token_creator(FaspexPubLink)
   module Api
     # Aspera Faspex 5 API Client
-    class Faspex < Aspera::Rest
-      include RestList
+    class Faspex < Aspera::Rest::Client
+      include Rest::List
 
       # endpoint for authentication API
       PATH_AUTH = 'auth'
@@ -141,6 +142,7 @@ module Aspera
 
       # @param url            [String] Faspex URL, can be a public link
       # @param auth           [Symbol] Authentication method: :boot (token in header), :web (open browser), :jwt (client_id + private key), :public_link (context in URL)
+      # @param root           [String] Root path for API
       # @param password       [String, nil] For :boot auth, the token copied directly from browser in developer mode
       # @param client_id      [String, nil] For :web and :jwt auth, the client_id of web UI application
       # @param client_secret  [String, nil] For :web auth, the client_secret of web UI application (not needed for :jwt)
@@ -166,7 +168,7 @@ module Aspera
           case auth
           when :public_link
             # Get URL of final redirect of provided public link
-            final_url = Rest.new(base_url: url, redirect_max: 3).call(operation: 'GET', ret: :resp).uri.to_s
+            final_url = Rest::Client.new(base_url: url, redirect_max: 3).call(operation: 'GET', ret: :resp).uri.to_s
             Log.dump(:final_url, final_url, level: :trace1)
             # Get context from query
             encoded_context = Rest.query_to_h(URI.parse(final_url).query)['context']
@@ -178,7 +180,7 @@ module Aspera
             base_url = final_url.gsub(%r{/public/.*}, '').gsub(/\?.*/, '')
             # Get web UI client_id and redirect_uri
             # TODO: change this for something more reliable
-            config = JSON.parse(Rest.new(base_url: "#{base_url}/config.js", redirect_max: 3).call(operation: 'GET').sub(/^[^=]+=/, '').gsub(/([a-z_]+):/, '"\1":').delete("\n ").tr("'", '"')).symbolize_keys
+            config = JSON.parse(Rest::Client.new(base_url: "#{base_url}/config.js", redirect_max: 3).call(operation: 'GET').sub(/^[^=]+=/, '').gsub(/([a-z_]+):/, '"\1":').delete("\n ").tr("'", '"')).symbolize_keys
             Log.dump(:config_js, config)
             {
               base_url: "#{base_url}/#{root}",

data/lib/aspera/api/httpgw.rb

@@ -9,8 +9,10 @@ require 'securerandom'
 require 'websocket'
 require 'base64'
 require 'json'
+require 'aspera/rainbow'
+using Rainbow
 
-# throw exception on error, instead of error code
+# Throw exception on error, instead of error code
 WebSocket.should_raise = true
 
 module Aspera
@@ -29,7 +31,7 @@ module Aspera
     #   1     JSON.slice_upload   File start          "end_slice_upload"  sent_v2_delimiter
     #   2..   Binary              File binary chunks  "end upload"        sent_general
     #   last  JSON.slice_upload   File end            "end_slice_upload"  sent_v2_delimiter
-    class Httpgw < Aspera::Rest
+    class Httpgw < Aspera::Rest::Client
       DEFAULT_BASE_PATH = '/aspera/http-gwy'
       INFO_ENDPOINT = 'info'
       MSG_SEND_TRANSFER_SPEC = 'transfer_spec'
@@ -40,27 +42,24 @@ module Aspera
       API_V1 = 'v1'
       API_V2 = 'v2'
       THR_RECV = 'recv'
-      LOG_WS_SEND = 'ws: send: '.red
-      LOG_WS_RECV = "ws: #{THR_RECV}: ".green
       private_constant :MSG_RECV_DATA_RECEIVED_SIGNAL, :MSG_RECV_SLICE_UPLOAD_SIGNAL
-      # send message on http gw web socket
+
+      # Send message on http gw web socket
+      # @param msg_type [String] type of message to send
+      # @param payload [Hash] Data to send
       def ws_snd_json(msg_type, payload)
         if msg_type.eql?(MSG_SEND_SLICE_UPLOAD) && @upload_version.eql?(API_V2)
           @shared_info[:count][:sent_v2_delimiter] += 1
         else
           @shared_info[:count][:sent_general] += 1
         end
-        Log.log.trace1 do
-          log_data = payload.dup
-          log_data[:data] = "[data #{log_data[:data].length} bytes]" if log_data.key?(:data)
-          "#{LOG_WS_SEND}json: #{msg_type}: #{JSON.generate(log_data)}"
-        end
+        Log.dump("#{log_ws_send} #{msg_type}", level: :trace1) { payload.dup.tap { |d| d[:data] = "[data #{d[:data].length} bytes]" if d.key?(:data) } }
         ws_send(ws_type: :text, data: JSON.generate({msg_type => payload}))
       end
 
       # send data on http gw web socket
       def ws_send(ws_type:, data:)
-        Log.log.trace1 { "#{LOG_WS_SEND}sending: #{ws_type} (#{data&.length || 0} bytes)" }
+        Log.log.trace1 { "#{log_ws_send}sending: #{ws_type} (#{data&.length || 0} bytes)" }
         @shared_info[:count][:sent_general] += 1 if ws_type.eql?(:binary)
         frame_generator = ::WebSocket::Frame::Outgoing::Client.new(data: data, type: ws_type, version: @ws_handshake.version)
         @ws_io.write(frame_generator.to_s)
@@ -72,12 +71,12 @@ module Aspera
                 @shared_info[:read_exception].nil? &&
                 (((@shared_info[:count][:sent_general] - @shared_info[:count][:received_general]) > 1) ||
                   ((@shared_info[:count][:received_v2_delimiter] - @shared_info[:count][:sent_v2_delimiter]) > 1))
-              Log.log.trace1 { "#{LOG_WS_SEND}#{'timeout'.blue}: #{@shared_info[:count]}" } if !@shared_info[:cond_var].wait(@shared_info[:mutex], 2.0)
+              Log.log.trace1 { "#{log_ws_send}#{'timeout'.blue}: #{@shared_info[:count]}" } if !@shared_info[:cond_var].wait(@shared_info[:mutex], 2.0)
             end
           end
         end
         raise @shared_info[:read_exception] unless @shared_info[:read_exception].nil?
-        Log.log.trace2 { "#{LOG_WS_SEND}counts: #{@shared_info[:count]}" }
+        Log.log.trace2 { "#{log_ws_send}counts: #{@shared_info[:count]}" }
       end
 
       # Check header ourself and give precise error message, as websocket will only throw error without details
@@ -85,13 +84,13 @@ module Aspera
       # @return [String] Response Header
       def validated_ws_response_header(header)
         first_line = header.split("\r\n").first
-        raise RestCallError.new({messages: ["Unexpected: #{first_line}", 'Expected: 101 Switching Protocols']}) unless first_line.split(/\s+/, 3)[1].eql?('101')
+        raise Rest::CallError.new({messages: ["Unexpected: #{first_line}", 'Expected: 101 Switching Protocols']}) unless first_line.split(/\s+/, 3)[1].eql?('101')
         header
       end
 
       # message processing for read thread
       def process_received_message(message)
-        Log.log.debug { "#{LOG_WS_RECV}message: [#{message}] (#{message.class})" }
+        Log.log.debug { "#{log_ws_recv}message: [#{message}] (#{message.class})" }
         if message.eql?(MSG_RECV_DATA_RECEIVED_SIGNAL)
           @shared_info[:mutex].synchronize do
             @shared_info[:count][:received_general] += 1
@@ -117,23 +116,23 @@ module Aspera
         end
       end
 
-      # main function of read thread
+      # Main function of read thread
       def process_read_thread
-        Log.log.debug { "#{LOG_WS_RECV}read thread started" }
+        Log.log.debug { "#{log_ws_recv}read thread started" }
         frame_parser = ::WebSocket::Frame::Incoming::Client.new(version: @ws_handshake.version)
         until @ws_io.eof?
           begin
             # ready byte by byte until frame is ready
             # blocking read
             byte = @ws_io.read(1)
-            Log.log.trace2 { "#{LOG_WS_RECV}read: #{byte} (#{byte.class}) eof=#{@ws_io.eof?}" }
+            Log.log.trace2 { "#{log_ws_recv}read: #{byte} (#{byte.class}) eof=#{@ws_io.eof?}" }
             frame_parser << byte
             frame_ok = frame_parser.next
             next if frame_ok.nil?
             process_received_message(frame_ok.data.to_s)
-            Log.log.trace2 { "#{LOG_WS_RECV}counts: #{@shared_info[:count]}" }
+            Log.log.trace2 { "#{log_ws_recv}counts: #{@shared_info[:count]}" }
           rescue => e
-            Log.log.debug { "#{LOG_WS_RECV}Exception: #{e}" }
+            Log.log.debug { "#{log_ws_recv}Exception: #{e}" }
             @shared_info[:mutex].synchronize do
               @shared_info[:read_exception] = e
               @shared_info[:cond_var].signal
@@ -142,9 +141,9 @@ module Aspera
           end
         end
         Log.log.debug do
-          "#{LOG_WS_RECV}exception: #{@shared_info[:read_exception]},cls=#{@shared_info[:read_exception].class})"
+          "#{log_ws_recv}exception: #{@shared_info[:read_exception]},cls=#{@shared_info[:read_exception].class})"
         end unless @shared_info[:read_exception].nil?
-        Log.log.debug { "#{LOG_WS_RECV}read thread stopped (ws eof=#{@ws_io.eof?})" }
+        Log.log.debug { "#{log_ws_recv}read thread stopped (ws eof=#{@ws_io.eof?})" }
       end
 
       def upload(transfer_spec)
@@ -170,7 +169,7 @@ module Aspera
         # Get whole HTTP response header, Check and process
         # no need to check `finished?` or result of `<<` (true), as we give the whole header at once
         @ws_handshake << validated_ws_response_header(@ws_io.readuntil("\r\n\r\n"))
-        Log.log.debug { "#{LOG_WS_SEND}handshake success" }
+        Log.log.debug { "#{log_ws_send}handshake success" }
         # data shared between main thread and read thread
         @shared_info = {
           read_exception: nil, # error message if any in callback
@@ -224,7 +223,7 @@ module Aspera
                   # send once, before data, at beginning
                   ws_snd_json(MSG_SEND_SLICE_UPLOAD, slice_info) if slice_info[:slice].eql?(0)
                   ws_send(ws_type: :binary, data: slice_bin_data)
-                  Log.log.trace1 { "#{LOG_WS_SEND}buffer: file: #{file_index}, slice: #{slice_info[:slice]}/#{last_slice}" }
+                  Log.log.trace1 { "#{log_ws_send}buffer: file: #{file_index}, slice: #{slice_info[:slice]}/#{last_slice}" }
                   # send once, after data, at end
                   ws_snd_json(MSG_SEND_SLICE_UPLOAD, slice_info) if slice_info[:slice].eql?(last_slice)
                 end
@@ -283,17 +282,20 @@ module Aspera
             # it is a plain file if we don't require zip and there is only one file
             File.basename(default_file_name)
           end
-        file_path = File.join(transfer_spec['destination_root'], file_name)
-        call(operation: 'GET', subpath: "download/#{transfer_uuid}", save_to: file_path)
+        call(
+          operation: 'GET',
+          subpath: "download/#{transfer_uuid}",
+          save_to: File.join(transfer_spec['destination_root'], file_name)
+        )
       end
 
       def info
-        return @api_info
+        @api_info
       end
 
       # @return [String] the base url of the gateway
       def base_url
-        return @gw_root_url
+        @gw_root_url
       end
 
       # @param url [String] URL of the HTTP Gateway, without version
@@ -306,10 +308,10 @@ module Aspera
         **opts
       )
         Log.dump(:gw_url, url)
-        # add scheme if missing
+        # Add scheme if missing
         url = "https://#{url}" unless url.match?(%r{^[a-z]{1,6}://})
         Aspera.assert(url.start_with?('https://'), type: Error) { 'GW URL shall be with scheme https' }
-        # remove trailing slash and version (o=only once) if present
+        # Remove trailing slash and version (o=only once) if present
         # TODO: issue warning ?
         url = url.chomp('/').gsub(%r{/#{API_V1}$}o, '')
         # assume GW is always under specific path (TODO: remove this ?)
@@ -339,6 +341,10 @@ module Aspera
 
       private
 
+      # Log prefixes (evaluated on each call, as colors can be enabled or disabled by option)
+      def log_ws_send = 'ws: send: '.red
+      def log_ws_recv = "ws: #{THR_RECV}: ".green
+
       # compute total size of files to upload (for progress)
       # modify transfer spec to be suitable for HTTPGW
       # @param transfer_spec [Hash] transfer specification
@@ -372,8 +378,7 @@ module Aspera
             until folders_to_process.empty?
               folder = folders_to_process.shift
               # read all entries
-              Dir.entries(folder).each do |entry|
-                next if entry.eql?('.') || entry.eql?('..')
+              Dir.each_child(folder) do |entry|
                 entry_path = File.join(folder, entry)
                 if File.directory?(entry_path)
                   folders_to_process.push(entry_path)
@@ -396,7 +401,6 @@ module Aspera
         end
         transfer_spec['paths'] = files_to_send.map { |i| {'source' => i[:name]} }
         files_to_send.push(total_bytes_to_transfer)
-        return files_to_send
       end
     end
   end