aspera-cli 4.17.0 → 4.18.0

data/lib/aspera/api/httpgw.rb

@@ -0,0 +1,339 @@
+# frozen_string_literal: true
+
+require 'aspera/log'
+require 'aspera/rest'
+require 'aspera/transfer/faux_file'
+require 'aspera/assert'
+require 'securerandom'
+require 'websocket'
+require 'base64'
+require 'json'
+
+module Aspera
+  module Api
+    # Start a transfer using Aspera HTTP Gateway, using web socket secure for uploads
+    # ref: https://api.ibm.com/explorer/catalog/aspera/product/ibm-aspera/api/http-gateway-api/doc/guides-toc
+    # https://developer.ibm.com/apis/catalog?search=%22aspera%20http%22
+    # HTTP GW Upload protocol:
+    #   #     type                Contents            Ack                 Counter
+    # v1
+    #   0     JSON.transfer_spec  Transfer Spec       "end upload"        sent_general
+    #   1..   JSON.slice_upload   File base64 chunks  "end upload"        sent_general
+    # v2
+    #   0     JSON.transfer_spec  Transfer Spec       "end upload"        sent_general
+    #   1     JSON.slice_upload   File start          "end_slice_upload"  sent_v2_delimiter
+    #   2..   Binary              File binary chunks  "end upload"        sent_general
+    #   last  JSON.slice_upload   File end            "end_slice_upload"  sent_v2_delimiter
+    class Httpgw < Aspera::Rest
+      DEFAULT_BASE_PATH = '/aspera/http-gwy'
+      INFO_ENDPOINT = 'info'
+      MSG_SEND_TRANSFER_SPEC = 'transfer_spec'
+      MSG_SEND_SLICE_UPLOAD = 'slice_upload'
+      MSG_RECV_DATA_RECEIVED_SIGNAL = 'end upload'
+      MSG_RECV_SLICE_UPLOAD_SIGNAL = 'end_slice_upload'
+      # upload API versions
+      API_V1 = 'v1'
+      API_V2 = 'v2'
+      THR_RECV = 'recv'
+      LOG_WS_SEND = 'ws: send: '.red
+      LOG_WS_RECV = "ws: #{THR_RECV}: ".green
+      private_constant :MSG_RECV_DATA_RECEIVED_SIGNAL, :MSG_RECV_SLICE_UPLOAD_SIGNAL
+      # send message on http gw web socket
+      def ws_snd_json(msg_type, payload)
+        if msg_type.eql?(MSG_SEND_SLICE_UPLOAD) && @upload_version.eql?(API_V2)
+          @shared_info[:count][:sent_v2_delimiter] += 1
+        else
+          @shared_info[:count][:sent_general] += 1
+        end
+        Log.log.debug do
+          log_data = payload.dup
+          log_data[:data] = "[data #{log_data[:data].length} bytes]" if log_data.key?(:data)
+          "#{LOG_WS_SEND}json: #{msg_type}: #{JSON.generate(log_data)}"
+        end
+        ws_send(ws_type: :text, data: JSON.generate({msg_type => payload}))
+      end
+
+      # send data on http gw web socket
+      def ws_send(ws_type:, data:)
+        Log.log.debug{"#{LOG_WS_SEND}sending: #{ws_type} (#{data&.length || 0} bytes)"}
+        @shared_info[:count][:sent_general] += 1 if ws_type.eql?(:binary)
+        frame_generator = ::WebSocket::Frame::Outgoing::Client.new(data: data, type: ws_type, version: @ws_handshake.version)
+        @ws_io.write(frame_generator.to_s)
+        if @synchronous
+          @shared_info[:mutex].synchronize do
+            # if read thread exited, there will be no more updates
+            # we allow for 1 of difference else it stays blocked
+            while @ws_read_thread.alive? &&
+                @shared_info[:read_exception].nil? &&
+                (((@shared_info[:count][:sent_general] - @shared_info[:count][:received_general]) > 1) ||
+                  ((@shared_info[:count][:received_v2_delimiter] - @shared_info[:count][:sent_v2_delimiter]) > 1))
+              if !@shared_info[:cond_var].wait(@shared_info[:mutex], 2.0)
+                Log.log.debug{"#{LOG_WS_SEND}#{'timeout'.blue}: #{@shared_info[:count]}"}
+              end
+            end
+          end
+        end
+        raise @shared_info[:read_exception] unless @shared_info[:read_exception].nil?
+        Log.log.debug{"#{LOG_WS_SEND}counts: #{@shared_info[:count]}"}
+      end
+
+      # message processing for read thread
+      def process_received_message(message)
+        Log.log.debug{"#{LOG_WS_RECV}message: [#{message}] (#{message.class})"}
+        if message.eql?(MSG_RECV_DATA_RECEIVED_SIGNAL)
+          @shared_info[:mutex].synchronize do
+            @shared_info[:count][:received_general] += 1
+            @shared_info[:cond_var].signal
+          end
+        elsif message.eql?(MSG_RECV_SLICE_UPLOAD_SIGNAL)
+          @shared_info[:mutex].synchronize do
+            @shared_info[:count][:received_v2_delimiter] += 1
+            @shared_info[:cond_var].signal
+          end
+        else
+          message.chomp!
+          error_message =
+            if message.start_with?('"') && message.end_with?('"')
+              # remove double quotes : 1..-2
+              JSON.parse(Base64.strict_decode64(message.chomp[1..-2]))['message']
+            elsif message.start_with?('{') && message.end_with?('}')
+              JSON.parse(message)['message']
+            else
+              "unknown message from gateway: [#{message}]"
+            end
+          raise error_message
+        end
+      end
+
+      # main function of read thread
+      def process_read_thread
+        Log.log.debug{"#{LOG_WS_RECV}read thread started"}
+        frame_parser = ::WebSocket::Frame::Incoming::Client.new(version: @ws_handshake.version)
+        until @ws_io.eof?
+          begin # rubocop:disable Style/RedundantBegin
+            # ready byte by byte until frame is ready
+            # blocking read
+            byte = @ws_io.read(1)
+            Log.log.trace1{"#{LOG_WS_RECV}read: #{byte} (#{byte.class}) eof=#{@ws_io.eof?}"}
+            frame_parser << byte
+            frame_ok = frame_parser.next
+            next if frame_ok.nil?
+            process_received_message(frame_ok.data.to_s)
+            Log.log.debug{"#{LOG_WS_RECV}counts: #{@shared_info[:count]}"}
+          rescue => e
+            Log.log.debug{"#{LOG_WS_RECV}Exception: #{e}"}
+            @shared_info[:mutex].synchronize do
+              @shared_info[:read_exception] = e
+              @shared_info[:cond_var].signal
+            end
+            break
+          end
+        end
+        Log.log.debug do
+          "#{LOG_WS_RECV}exception: #{@shared_info[:read_exception]},cls=#{@shared_info[:read_exception].class})"
+        end unless @shared_info[:read_exception].nil?
+        Log.log.debug{"#{LOG_WS_RECV}read thread stopped (ws eof=#{@ws_io.eof?})"}
+      end
+
+      def upload(transfer_spec)
+        # identify this session uniquely
+        session_id = SecureRandom.uuid
+        @notify_cb&.call(session_id: nil, type: :pre_start, info: 'starting')
+        # total size of all files
+        total_bytes_to_transfer = 0
+        # we need to keep track of actual file path because transfer spec is modified to be sent in web socket
+        files_to_read = []
+        # get source root or nil
+        source_root = transfer_spec.key?('source_root') && !transfer_spec['source_root'].empty? ? transfer_spec['source_root'] : nil
+        # source root is ignored by GW, used only here
+        transfer_spec.delete('source_root')
+        # compute total size of files to upload (for progress)
+        # modify transfer spec to be suitable for GW
+        transfer_spec['paths'].each do |item|
+          # save actual file location to be able read contents later
+          file_to_add = Transfer::FauxFile.open(item['source'])
+          if file_to_add
+            item['source'] = file_to_add.path
+            item['file_size'] = file_to_add.size
+          else
+            file_to_add = item['source']
+            # add source root if needed
+            file_to_add = File.join(source_root, file_to_add) unless source_root.nil?
+            # GW expects a simple file name in 'source' but if user wants to change the name, we take it
+            item['source'] = File.basename(item['destination'].nil? ? item['source'] : item['destination'])
+            item['file_size'] = File.size(file_to_add)
+          end
+          # save so that we can actually read the file later
+          files_to_read.push(file_to_add)
+          total_bytes_to_transfer += item['file_size']
+        end
+        # TODO: check that this is available in endpoints: @api_info['endpoints']
+        upload_url = File.join(@gw_root_url, @upload_version, 'upload')
+        @notify_cb&.call(session_id: nil, type: :pre_start, info: 'connecting wss')
+        # open web socket to end point (equivalent to Net::HTTP.start)
+        http_session = Rest.start_http_session(upload_url)
+        # get the underlying socket i/o
+        @ws_io = Rest.io_http_session(http_session)
+        @ws_handshake = ::WebSocket::Handshake::Client.new(url: upload_url, headers: {})
+        @ws_io.write(@ws_handshake.to_s)
+        sleep(0.1)
+        @ws_handshake << @ws_io.readuntil("\r\n\r\n")
+        Aspera.assert(@ws_handshake.finished?){'Error in websocket handshake'}
+        Log.log.debug{"#{LOG_WS_SEND}handshake success"}
+        # start read thread after handshake
+        @ws_read_thread = Thread.new {process_read_thread}
+        @notify_cb&.call(session_id: session_id, type: :session_start)
+        @notify_cb&.call(session_id: session_id, type: :session_size, info: total_bytes_to_transfer)
+        sleep(1)
+        # data shared between main thread and read thread
+        @shared_info = {
+          read_exception: nil, # error message if any in callback
+          count:          {
+            sent_general:          0,
+            received_general:      0,
+            sent_v2_delimiter:     0,
+            received_v2_delimiter: 0
+          },
+          mutex:          Mutex.new,
+          cond_var:       ConditionVariable.new
+        }
+        # notify progress bar
+        @notify_cb&.call(type: :session_size, session_id: session_id, info: total_bytes_to_transfer)
+        # first step send transfer spec
+        Log.log.debug{Log.dump(:ws_spec, transfer_spec)}
+        ws_snd_json(MSG_SEND_TRANSFER_SPEC, transfer_spec)
+        # current file index
+        file_index = 0
+        # aggregate size sent
+        session_sent_bytes = 0
+        # process each file
+        transfer_spec['paths'].each do |item|
+          slice_info = {
+            name:       nil,
+            # TODO: get mime type?
+            type:       'application/octet-stream',
+            size:       item['file_size'],
+            slice:      0, # current slice index
+            # index of last slice (i.e number of slices - 1)
+            last_slice: (item['file_size'] - 1) / @upload_chunk_size,
+            fileIndex:  file_index
+          }
+          file = files_to_read[file_index]
+          if file.is_a?(Transfer::FauxFile)
+            slice_info[:name] = file.path
+          else
+            file = File.open(file)
+            slice_info[:name] = File.basename(item[item['destination'].nil? ? 'source' : 'destination'])
+          end
+          begin
+            until file.eof?
+              slice_bin_data = file.read(@upload_chunk_size)
+              # interrupt main thread if read thread failed
+              raise @shared_info[:read_exception] unless @shared_info[:read_exception].nil?
+              begin
+                if @upload_version.eql?(API_V1)
+                  slice_info[:data] = Base64.strict_encode64(slice_bin_data)
+                  ws_snd_json(MSG_SEND_SLICE_UPLOAD, slice_info)
+                else
+                  # send once, before data, at beginning
+                  ws_snd_json(MSG_SEND_SLICE_UPLOAD, slice_info) if slice_info[:slice].eql?(0)
+                  ws_send(ws_type: :binary, data: slice_bin_data)
+                  Log.log.debug{"#{LOG_WS_SEND}buffer: file: #{file_index}, slice: #{slice_info[:slice]}/#{slice_info[:last_slice]}"}
+                  # send once, after data, at end
+                  ws_snd_json(MSG_SEND_SLICE_UPLOAD, slice_info) if slice_info[:slice].eql?(slice_info[:last_slice])
+                end
+              rescue Errno::EPIPE => e
+                raise @shared_info[:read_exception] unless @shared_info[:read_exception].nil?
+                raise e
+              rescue Net::ReadTimeout => e
+                Log.log.warn{'A timeout condition using HTTPGW may signal a permission problem on destination. Check ascp logs on httpgw.'}
+                raise e
+              end
+              session_sent_bytes += slice_bin_data.length
+              @notify_cb&.call(type: :transfer, session_id: session_id, info: session_sent_bytes)
+              slice_info[:slice] += 1
+            end
+          ensure
+            file.close
+          end
+          file_index += 1
+        end
+        # throttling may have skipped last one
+        @notify_cb&.call(type: :transfer, session_id: session_id, info: session_sent_bytes)
+        @notify_cb&.call(type: :end, session_id: session_id)
+        ws_send(ws_type: :close, data: nil)
+        Log.log.debug("Finished upload, waiting for end of #{THR_RECV} thread.")
+        @ws_read_thread.join
+        Log.log.debug{'Read thread joined'}
+        # session no more used
+        @ws_io = nil
+        http_session&.finish
+      end
+
+      def download(transfer_spec)
+        transfer_spec['zip_required'] ||= false
+        transfer_spec['source_root'] ||= '/'
+        # is normally provided by application, like package name
+        if !transfer_spec.key?('download_name')
+          # by default it is the name of first file
+          download_name = File.basename(transfer_spec['paths'].first['source'], '.*')
+          # ands add indication of number of files if there is more than one
+          if transfer_spec['paths'].length > 1
+            download_name += " #{transfer_spec['paths'].length} Files"
+          end
+          transfer_spec['download_name'] = download_name
+        end
+        creation = create('download', {'transfer_spec' => transfer_spec})[:data]
+        transfer_uuid = creation['url'].split('/').last
+        file_name =
+          if transfer_spec['zip_required'] || transfer_spec['paths'].length > 1
+            # it is a zip file if zip is required or there is more than 1 file
+            transfer_spec['download_name'] + '.zip'
+          else
+            # it is a plain file if we don't require zip and there is only one file
+            File.basename(transfer_spec['paths'].first['source'])
+          end
+        file_path = File.join(transfer_spec['destination_root'], file_name)
+        call(operation: 'GET', subpath: "download/#{transfer_uuid}", save_to_file: file_path)
+      end
+
+      def info
+        return @api_info
+      end
+
+      # @param url [String] URL of the HTTP Gateway, without version
+      def initialize(
+        url:,
+        api_version:       API_V2,
+        upload_chunk_size: 64_000,
+        synchronous:       false,
+        notify_cb:         nil,
+        **opts
+      )
+        # add scheme if missing
+        url = "https://#{url}" unless url.match?(%r{^[a-z]{1,6}://})
+        raise 'GW URL shall be with scheme https' unless url.start_with?('https://')
+        # remove trailing slash and version if any
+        url = url.gsub(%r{/+$}, '').gsub(%r{/#{API_V1}$}o, '')
+        url = File.join(url, DEFAULT_BASE_PATH) unless url.end_with?(DEFAULT_BASE_PATH)
+        @gw_root_url = url
+        super(base_url: "#{@gw_root_url}/#{API_V1}", **opts)
+        @upload_version = api_version
+        @upload_chunk_size = upload_chunk_size
+        @synchronous = synchronous
+        @notify_cb = notify_cb
+        # get API info
+        @api_info = read('info')[:data].freeze
+        Log.log.debug{Log.dump(:api_info, @api_info)}
+        # web socket endpoint: by default use v2 (newer gateways), without base64 encoding
+        # is the latest supported? else revert to old api
+        if !@upload_version.eql?(API_V1)
+          if !@api_info['endpoints'].any?{|i|i.include?(@upload_version)}
+            Log.log.warn{"API version #{@upload_version} not supported, reverting to #{API_V1}"}
+            @upload_version = API_V1
+          end
+        end
+      end
+    end
+  end
+end

data/lib/aspera/api/node.rb

@@ -14,21 +14,23 @@ module Aspera
   module Api
     # Provides additional functions using node API with gen4 extensions (access keys)
     class Node < Aspera::Rest
-      # permissions
+      # node api permissions
       ACCESS_LEVELS = %w[delete list mkdir preview read rename write].freeze
+      HEADER_X_ASPERA_ACCESS_KEY = 'X-Aspera-AccessKey'
+      SCOPE_SEPARATOR = ':'
+      SCOPE_USER = 'user:all'
+      SCOPE_ADMIN = 'admin:all'
+      SCOPE_NODE_PREFIX = 'node.'
       # prefix for ruby code for filter (deprecated)
       MATCH_EXEC_PREFIX = 'exec:'
       MATCH_TYPES = [String, Proc, Regexp, NilClass].freeze
-      HEADER_X_ASPERA_ACCESS_KEY = 'X-Aspera-AccessKey'
       PATH_SEPARATOR = '/'
-      TS_FIELDS_TO_COPY = %w[remote_host remote_user ssh_port fasp_port wss_enabled wss_port].freeze
-      SCOPE_USER = 'user:all'
-      SCOPE_ADMIN = 'admin:all'
-      SCOPE_PREFIX = 'node.'
-      SCOPE_SEPARATOR = ':'
       SIGNATURE_DELIMITER = '==SIGNATURE=='
       BEARER_TOKEN_VALIDITY_DEFAULT = 86400
       BEARER_TOKEN_SCOPE_DEFAULT = SCOPE_USER
+      private_constant :MATCH_EXEC_PREFIX, :MATCH_TYPES,
+        :SIGNATURE_DELIMITER, :BEARER_TOKEN_VALIDITY_DEFAULT, :BEARER_TOKEN_SCOPE_DEFAULT,
+        :SCOPE_SEPARATOR, :SCOPE_NODE_PREFIX
 
       # register node special token decoder
       OAuth::Factory.instance.register_decoder(lambda{|token|Node.decode_bearer_token(token)})
@@ -62,14 +64,14 @@ module Aspera
 
         # node API scopes
         def token_scope(access_key, scope)
-          return [SCOPE_PREFIX, access_key, SCOPE_SEPARATOR, scope].join('')
+          return [SCOPE_NODE_PREFIX, access_key, SCOPE_SEPARATOR, scope].join('')
         end
 
         def decode_scope(scope)
           items = scope.split(SCOPE_SEPARATOR, 2)
           Aspera.assert(items.length.eql?(2)){"invalid scope: #{scope}"}
-          Aspera.assert(items[0].start_with?(SCOPE_PREFIX)){"invalid scope: #{scope}"}
-          return {access_key: items[0][SCOPE_PREFIX.length..-1], scope: items[1]}
+          Aspera.assert(items[0].start_with?(SCOPE_NODE_PREFIX)){"invalid scope: #{scope}"}
+          return {access_key: items[0][SCOPE_NODE_PREFIX.length..-1], scope: items[1]}
         end
 
         # Create an Aspera Node bearer token
@@ -134,6 +136,7 @@ module Aspera
         @app_info = app_info
         # this is added to transfer spec, for instance to add tags (COS)
         @add_tspec = add_tspec
+        @std_t_spec_cache = nil
         if !@app_info.nil?
           REQUIRED_APP_INFO_FIELDS.each do |field|
             Aspera.assert(@app_info.key?(field)){"app_info lacks field #{field}"}
@@ -205,7 +208,7 @@ module Aspera
             end
           end
         end
-      end # process_folder_tree
+      end
 
       # Navigate the path from given file id
       # @param top_file_id [String] id initial file id
@@ -271,7 +274,23 @@ module Aspera
       end
 
       def refreshed_transfer_token
-        return oauth_token(force_refresh: true)
+        return oauth.token(refresh: true)
+      end
+
+      # @return part of transfer spec with transport parameters only
+      def transport_params
+        if @std_t_spec_cache.nil?
+          # retrieve values from API (and keep a copy/cache)
+          full_spec = create(
+            'files/download_setup',
+            {transfer_requests: [{transfer_request: {paths: [{source: '/'}]}}]}
+          )[:data]['transfer_specs'].first['transfer_spec']
+          # set available fields
+          @std_t_spec_cache = Transfer::Spec::TRANSPORT_FIELDS.each_with_object({}) do |i, h|
+            h[i] = full_spec[i] if full_spec.key?(i)
+          end
+        end
+        return @std_t_spec_cache
       end
 
       # Create transfer spec for gen4
@@ -285,9 +304,9 @@ module Aspera
           ak_token = Rest.basic_token(auth_params[:username], auth_params[:password])
         when :oauth2
           ak_name = params[:headers][HEADER_X_ASPERA_ACCESS_KEY]
-          # TODO: token_generation_lambda = lambda{|do_refresh|oauth_token(force_refresh: do_refresh)}
+          # TODO: token_generation_lambda = lambda{|do_refresh|oauth.token(refresh: do_refresh)}
           # get bearer token, possibly use cache
-          ak_token = oauth_token(force_refresh: false)
+          ak_token = oauth.token(refresh: false)
         else Aspera.error_unexpected_value(auth_params[:type])
         end
         transfer_spec = {
@@ -327,13 +346,7 @@ module Aspera
             transfer_spec[i] = settings[i] if settings.key?(i)
           end if settings.is_a?(Hash)
         else
-          # retrieve values from API (and keep a copy/cache)
-          @std_t_spec_cache ||= create(
-            'files/download_setup',
-            {transfer_requests: [{ transfer_request: {paths: [{'source' => '/'}] } }] }
-          )[:data]['transfer_specs'].first['transfer_spec']
-          # copy some parts
-          TS_FIELDS_TO_COPY.each {|i| transfer_spec[i] = @std_t_spec_cache[i] if @std_t_spec_cache.key?(i)}
+          transfer_spec.merge!(transport_params)
         end
         Log.log.warn{"Expected transfer user: #{Transfer::Spec::ACCESS_KEY_TRANSFER_USER}, but have #{transfer_spec['remote_user']}"} \
           unless transfer_spec['remote_user'].eql?(Transfer::Spec::ACCESS_KEY_TRANSFER_USER)

data/lib/aspera/ascmd.rb

@@ -22,6 +22,7 @@ module Aspera
       mv:     2,
       rm:     1
     }.freeze
+    private_constant :OPS_ARGS
     # list of supported actions
     OPERATIONS = OPS_ARGS.keys.freeze
 
@@ -94,7 +95,7 @@ module Aspera
       raise Error.new(result[:errno], result[:errstr], action_sym, arguments) if
         result.is_a?(Hash) && (result.keys.sort == TYPES_DESCR[:error][:fields].map{|i|i[:name]}.sort)
       return result
-    end # execute_single
+    end
 
     # This exception is raised when +ascmd+ returns an error.
     class Error < StandardError
@@ -103,7 +104,7 @@ module Aspera
 
       def message; "ascmd: #{@errstr} (#{@errno})"; end
       def extended_message; "ascmd: errno=#{@errno} errstr=\"#{@errstr}\" command=#{@command} arguments=#{@arguments&.join(',')}"; end
-    end # Error
+    end
 
     # description of result structures (see ascmdtypes.h). Base types are big endian
     # key = name of type
@@ -211,7 +212,7 @@ module Aspera
             end
           end
         else Aspera.error_unexpected_value(type_descr[:decode])
-        end # is_a
+        end
         return result
       end
     end

data/lib/aspera/ascp/installation.rb

@@ -173,8 +173,7 @@ module Aspera
         return exe_version
       end
 
-      def ascp_info
-        data = file_paths
+      def ascp_add_pvcl(data)
         # read PATHs from ascp directly, and pvcl modules as well
         Open3.popen3(data['ascp'], '-DDL-') do |_stdin, _stdout, stderr, thread|
           last_line = ''
@@ -202,15 +201,24 @@ module Aspera
             raise last_line
           end
         end
-        # ascp's openssl directory
+      end
+
+      # extract some stings from ascp binary
+      def ascp_add_openssl(data)
         ascp_file = data['ascp']
-        File.binread(ascp_file).scan(/[\x20-\x7E]{4,}/) do |match|
-          if (m = match.match(/OPENSSLDIR.*"(.*)"/))
+        File.binread(ascp_file).scan(/[\x20-\x7E]{10,}/) do |bin_string|
+          if (m = bin_string.match(/OPENSSLDIR.*"(.*)"/))
             data['openssldir'] = m[1]
+          elsif (m = bin_string.match(/OpenSSL (\d[^ -]+)/))
+            data['openssl_version'] = m[1]
           end
         end if File.file?(ascp_file)
-        # log is "-" no need to display
-        data.delete('log')
+      end
+
+      def ascp_info
+        data = file_paths
+        ascp_add_pvcl(data)
+        ascp_add_openssl(data)
         return data
       end
 

data/lib/aspera/ascp/management.rb

@@ -209,7 +209,7 @@ module Aspera
             h[new_name] = value
           end
         end
-      end # class << self
+      end
 
       def initialize
         # current event being parsed line by line
@@ -236,7 +236,7 @@ module Aspera
           return @last_event
         else
           raise "mgt port: unexpected line: [#{line}]"
-        end # case
+        end
         return nil
       end
     end

data/lib/aspera/cli/basic_auth_plugin.rb

@@ -8,10 +8,7 @@ module Aspera
     # base class for applications supporting basic authentication
     class BasicAuthPlugin < Cli::Plugin
       class << self
-        #@@basic_options_declared = false # rubocop:disable Style/ClassVars
-        def declare_options(options) # , force: false
-          #return if @@basic_options_declared && !force
-          #@@basic_options_declared = true # rubocop:disable Style/ClassVars
+        def declare_options(options)
           options.declare(:url, 'URL of application, e.g. https://faspex.example.com/aspera/faspex')
           options.declare(:username, "User's name to log in")
           options.declare(:password, "User's password")
@@ -21,14 +18,13 @@ module Aspera
 
       def initialize(basic_options: true, **env)
         super(**env)
-        # , force: env[:all_manuals]
         BasicAuthPlugin.declare_options(options) if basic_options
       end
 
       # returns a Rest object with basic auth
       def basic_auth_params(subpath=nil)
         api_url = options.get_option(:url, mandatory: true)
-        api_url = api_url + '/' + subpath unless subpath.nil?
+        api_url = "#{api_url}/#{subpath}" unless subpath.nil?
         return {
           base_url: api_url,
           auth:     {
@@ -41,6 +37,6 @@ module Aspera
       def basic_auth_api(subpath=nil)
         return Rest.new(**basic_auth_params(subpath))
       end
-    end # BasicAuthPlugin
-  end # Cli
-end # Aspera
+    end
+  end
+end

data/lib/aspera/cli/extended_value.rb

@@ -22,6 +22,10 @@ module Aspera
       ALL = 'ALL'
       DEF = 'DEF'
 
+      MARKER_START = '@'
+      MARKER_END = ':'
+      MARKER_IN_END = '@'
+
       class << self
         # decode comma separated table text
         def decode_csvt(value)
@@ -61,10 +65,11 @@ module Aspera
           list:   lambda{|v|v[1..-1].split(v[0])},
           none:   lambda{|v|ExtendedValue.assert_no_value(v, :none); nil}, # rubocop:disable Style/Semicolon
           path:   lambda{|v|File.expand_path(v)},
-          re:     lambda{|v|Regexp.new(v)},
+          re:     lambda{|v|Regexp.new(v, Regexp::MULTILINE)},
           ruby:   lambda{|v|Environment.secure_eval(v, __FILE__, __LINE__)},
           secret: lambda{|v|prompt = v.empty? ? 'secret' : v; $stdin.getpass("#{prompt}> ")}, # rubocop:disable Style/Semicolon
           stdin:  lambda{|v|ExtendedValue.assert_no_value(v, :stdin); $stdin.read}, # rubocop:disable Style/Semicolon
+          stdbin: lambda{|v|ExtendedValue.assert_no_value(v, :stdin); $stdin.binmode.read}, # rubocop:disable Style/Semicolon
           yaml:   lambda{|v|YAML.load(v)},
           zlib:   lambda{|v|Zlib::Inflate.inflate(v)},
           extend: lambda{|v|ExtendedValue.instance.evaluate_all(v)}
@@ -84,14 +89,14 @@ module Aspera
 
       # Regex to match an extended value
       def ext_re
-        "@(#{modifiers.join('|')}):"
+        "#{MARKER_START}(#{modifiers.join('|')})#{MARKER_END}"
       end
 
       # parse an option value if it is a String using supported extended value modifiers
       # other value types are returned as is
       def evaluate(value)
         return value unless value.is_a?(String)
-        regex = Regexp.new("^#{ext_re}(.*)$")
+        regex = Regexp.new("^#{ext_re}(.*)$", Regexp::MULTILINE)
         # first determine decoders, in reversed order
         handlers_reversed = []
         while (m = value.match(regex))
@@ -101,18 +106,19 @@ module Aspera
           # stop processing if handler is extend (it will be processed later)
           break if handler.eql?(:extend)
         end
+        Log.log.trace1{"evaluating: #{handlers_reversed}, value: #{value}"}
         handlers_reversed.each do |handler|
           value = @handlers[handler].call(value)
         end
         return value
-      end # evaluate
+      end
 
       # find inner extended values
       def evaluate_all(value)
-        regex = Regexp.new("^(.*)#{ext_re}([^@]*)@(.*)$")
+        regex = Regexp.new("^(.*)#{ext_re}([^#{MARKER_IN_END}]*)#{MARKER_IN_END}(.*)$", Regexp::MULTILINE)
         while (m = value.match(regex))
           sub_value = "@#{m[2]}:#{m[3]}"
-          Log.log.debug("evaluating #{sub_value}")
+          Log.log.debug{"evaluating #{sub_value}"}
           value = m[1] + evaluate(sub_value) + m[4]
         end
         return value