aspera-cli 4.9.0 → 4.10.0

data/lib/aspera/environment.rb

@@ -85,10 +85,28 @@ module Aspera
         if force || !File.exist?(path)
           File.unlink(path) rescue nil # Windows may give error
           File.write(path,yield)
-          File.chmod(0400,path)
+          restrict_file_access(path)
         end
         return path
       end
+
+      def restrict_file_access(path,mode: nil)
+        begin
+          if mode.nil?
+            # or FileUtils ?
+            if File.file?(path)
+              mode=0600
+            elsif File.directory?(path)
+              mode=0700
+            else
+              Log.log.debug("No restriction can be set for #{path}");
+            end
+          end
+          File.chmod(mode,path) unless mode.nil?
+        rescue => e
+          Log.log.warn(e.message)
+        end
+      end
     end
   end
 end

data/lib/aspera/fasp/agent_direct.rb

@@ -272,6 +272,10 @@ module Aspera
                   env_args[:env]['ASPERA_SCP_TOKEN'] = session[:options][:regenerate_token].call(true)
                 end
               end
+              # cannot resolve address
+              #if last_status_event['Code'].to_i.eql?(14)
+              #  Log.log.warn("host: #{}")
+              #end
               raise Fasp::Error.new(last_status_event['Description'],last_status_event['Code'].to_i)
             else # case
               raise "unexpected last event type: #{last_status_event['Type']}"

data/lib/aspera/fasp/agent_httpgw.rb

@@ -4,8 +4,8 @@ require 'aspera/fasp/agent_base'
 require 'aspera/fasp/transfer_spec'
 require 'aspera/log'
 require 'aspera/rest'
-require 'websocket-client-simple'
 require 'securerandom'
+require 'websocket'
 require 'base64'
 require 'json'
 
@@ -13,16 +13,32 @@ require 'json'
 # https://developer.ibm.com/apis/catalog?search=%22aspera%20http%22
 module Aspera
   module Fasp
-    # start a transfer using Aspera HTTP Gateway, using web socket session
+    # start a transfer using Aspera HTTP Gateway, using web socket session for uploads
     class AgentHttpgw < AgentBase
       # message returned by HTTP GW in case of success
-      OK_MESSAGE = 'end upload'
-      # refresh rate for progress
-      UPLOAD_REFRESH_SEC = 0.5
-      private_constant :OK_MESSAGE,:UPLOAD_REFRESH_SEC
+      MSG_END_UPLOAD = 'end upload'
+      MSG_END_SLICE = 'end_slice_upload'
+      DEFAULT_OPTIONS = {
+        url:                    nil,
+        upload_chunksize:       64_000,
+        upload_bar_refresh_sec: 0.5
+      }.freeze
+      DEFAULT_BASE_PATH='/aspera/http-gwy'
+      # upload endpoints
+      V1_UPLOAD='/v1/upload'
+      V2_UPLOAD='/v2/upload'
+      private_constant :DEFAULT_OPTIONS,:MSG_END_UPLOAD,:MSG_END_SLICE,:V1_UPLOAD,:V2_UPLOAD
+
       # send message on http gw web socket
-      def ws_send(ws,type,data)
-        ws.send(JSON.generate({type => data}))
+      def ws_snd_json(data)
+        @slice_uploads += 1 if data.has_key?(:slice_upload)
+        Log.log.debug{JSON.generate(data)}
+        ws_send(JSON.generate(data))
+      end
+
+      def ws_send(data, type: :text)
+        frame = ::WebSocket::Frame::Outgoing::Client.new(data: data, type: type, version: @ws_handshake.version)
+        @ws_io.write(frame.to_s)
       end
 
       def upload(transfer_spec)
@@ -48,93 +64,129 @@ module Aspera
           # save so that we can actually read the file later
           source_paths.push(full_src_filepath)
         end
-
+        # identify this session uniquely
         session_id = SecureRandom.uuid
-        ws = ::WebSocket::Client::Simple::Client.new
-        # error message if any in callback
-        error = nil
-        # number of files totally sent
-        received = 0
-        # setup callbacks on websocket
-        ws.on(:message) do |msg|
-          Log.log.info("ws: message: #{msg.data}")
-          message = msg.data
-          if message.eql?(OK_MESSAGE)
-            received += 1
-          else
-            message.chomp!
-            error =
-              if message.start_with?('"') && message.end_with?('"')
-                JSON.parse(Base64.strict_decode64(message.chomp[1..-2]))['message']
-              else
-                "expecting quotes in [#{message}]"
+        @slice_uploads=0
+        # web socket endpoint: by default use v2 (newer gateways), without base64 encoding
+        upload_api_version = V2_UPLOAD
+        # is the latest supported? else revert to old api
+        upload_api_version=V1_UPLOAD unless @api_info['endpoints'].any?{|i|i.include?(upload_api_version)}
+        Log.log.debug{"api version: #{upload_api_version}"}
+        url=File.join(@gw_api.params[:base_url],upload_api_version)
+        #uri = URI.parse(url)
+        # open web socket to end point (equivalent to Net::HTTP.start)
+        http_socket = Rest.start_http_session(url)
+        @ws_io = http_socket.instance_variable_get(:@socket)
+        #@ws_io.debug_output = Log.log
+        @ws_handshake = ::WebSocket::Handshake::Client.new(url: url, headers: {})
+        @ws_io.write(@ws_handshake.to_s)
+        sleep(0.1)
+        @ws_handshake << @ws_io.readuntil("\r\n\r\n")
+        raise 'Error in websocket handshake' unless @ws_handshake.finished?
+        Log.log.debug('ws: handshake success')
+        # data shared between main thread and read thread
+        shared_info={
+          read_exception: nil, # error message if any in callback
+          end_uploads:    0 # number of files totally sent
+          #mutex: Mutex.new
+          #cond_var: ConditionVariable.new
+        }
+        # start read thread
+        ws_read_thread = Thread.new do
+          Log.log.debug('ws: thread: started')
+          frame = ::WebSocket::Frame::Incoming::Client.new
+          loop do
+            begin
+              frame << @ws_io.readuntil("\n")
+              while (msg = frame.next)
+                Log.log.debug("ws: thread: message: #{msg.data} #{shared_info[:end_uploads]}")
+                message = msg.data
+                if message.eql?(MSG_END_UPLOAD)
+                  shared_info[:end_uploads] += 1
+                elsif message.eql?(MSG_END_SLICE)
+                else
+                  message.chomp!
+                  error_message =
+                    if message.start_with?('"') && message.end_with?('"')
+                      JSON.parse(Base64.strict_decode64(message.chomp[1..-2]))['message']
+                    elsif message.start_with?('{') && message.end_with?('}')
+                      JSON.parse(message)['message']
+                    else
+                      "unknown message from gateway: [#{message}]"
+                    end
+                  raise error_message
+                end
               end
+            rescue => e
+              shared_info[:read_exception] = e unless e.is_a?(EOFError)
+              break
+            end
           end
-        end
-        ws.on(:error) do |e|
-          error = e
-        end
-        ws.on(:open) do
-          Log.log.info('ws: open')
-        end
-        ws.on(:close) do
-          Log.log.info('ws: close')
-        end
-        # open web socket to end point
-        ws.connect("#{@gw_api.params[:base_url]}/upload")
-        # async wait ready
-        while !ws.open? && error.nil?
-          Log.log.info('ws: wait')
-          sleep(0.2)
+          Log.log.debug("ws: thread: stopping #{shared_info[:read_exception]} #{shared_info[:read_exception].class}")
         end
         # notify progress bar
         notify_begin(session_id,total_size)
         # first step send transfer spec
         Log.dump(:ws_spec,transfer_spec)
-        ws_send(ws,:transfer_spec,transfer_spec)
+        ws_snd_json(transfer_spec: transfer_spec)
         # current file index
         file_index = 0
         # aggregate size sent
         sent_bytes = 0
         # last progress event
-        lastevent = nil
-        transfer_spec['paths'].each do |item|
-          # TODO: get mime type?
-          file_mime_type = ''
-          file_size = item['file_size']
-          file_name = File.basename(item[item['destination'].nil? ? 'source' : 'destination'])
-          # compute total number of slices
-          numslices = 1 + ((file_size - 1) / @upload_chunksize)
-          File.open(source_paths[file_index]) do |file|
-            # current slice index
-            slicenum = 0
-            while !file.eof?
-              data = file.read(@upload_chunksize)
-              slice_data = {
-                name:         file_name,
-                type:         file_mime_type,
-                size:         file_size,
-                data:         Base64.strict_encode64(data),
-                slice:        slicenum,
-                total_slices: numslices,
-                fileIndex:    file_index
-              }
-              # log without data
-              Log.dump(:slide_data,slice_data.keys.each_with_object({}){|i,m|m[i] = i.eql?(:data) ? 'base64 data' : slice_data[i];}) if slicenum.eql?(0)
-              ws_send(ws,:slice_upload, slice_data)
-              sent_bytes += data.length
-              currenttime = Time.now
-              if lastevent.nil? || ((currenttime - lastevent) > UPLOAD_REFRESH_SEC)
-                notify_progress(session_id,sent_bytes)
-                lastevent = currenttime
+        last_progress_time = nil
+        begin
+          transfer_spec['paths'].each do |item|
+            # TODO: get mime type?
+            file_mime_type = ''
+            file_size = item['file_size']
+            file_name = File.basename(item[item['destination'].nil? ? 'source' : 'destination'])
+            # compute total number of slices
+            numslices = 1 + ((file_size - 1) / @options[:upload_chunksize])
+            File.open(source_paths[file_index]) do |file|
+              # current slice index
+              slicenum = 0
+              while !file.eof?
+                # interrupt main thread if read thread failed
+                raise shared_info[:read_exception] unless shared_info[:read_exception].nil?
+                data = file.read(@options[:upload_chunksize])
+                slice_data = {
+                  name:         file_name,
+                  type:         file_mime_type,
+                  size:         file_size,
+                  slice:        slicenum,
+                  total_slices: numslices,
+                  fileIndex:    file_index
+                }
+                #Log.dump(:slice_data,slice_data) #if slicenum.eql?(0)
+                if upload_api_version.eql?(V1_UPLOAD)
+                  slice_data[:data] = Base64.strict_encode64(data)
+                  ws_snd_json(slice_upload: slice_data)
+                else
+                  ws_snd_json(slice_upload: slice_data) if slicenum.eql?(0)
+                  ws_send(data,type: :binary)
+                  Log.log.debug{"ws: sent buffer: #{file_index} / #{slicenum}"}
+                  ws_snd_json(slice_upload: slice_data) if slicenum.eql?(numslices-1)
+                end
+                sent_bytes += data.length
+                currenttime = Time.now
+                if last_progress_time.nil? || ((currenttime - last_progress_time) > @options[:upload_bar_refresh_sec])
+                  notify_progress(session_id,sent_bytes)
+                  last_progress_time = currenttime
+                end
+                slicenum += 1
               end
-              slicenum += 1
-              raise error unless error.nil?
             end
+            file_index += 1
           end
-          file_index += 1
         end
-        ws.close
+        Log.log.debug('Finished upload')
+        ws_read_thread.join
+        Log.log.debug{"result: #{shared_info[:end_uploads]} / #{@slice_uploads}"}
+        ws_send(nil, type: :close) unless @ws_io.nil?
+        @ws_io = nil
+        http_socket&.finish
+        notify_progress(session_id,sent_bytes)
         notify_end(session_id)
       end
 
@@ -153,7 +205,7 @@ module Aspera
           end
           transfer_spec['download_name'] = dname
         end
-        creation = @gw_api.create('download',{'transfer_spec' => transfer_spec})[:data]
+        creation = @gw_api.create('v1/download',{'transfer_spec' => transfer_spec})[:data]
         transfer_uuid = creation['url'].split('/').last
         file_dest =
           if transfer_spec['zip_required'] || transfer_spec['paths'].length > 1
@@ -164,7 +216,7 @@ module Aspera
             File.basename(transfer_spec['paths'].first['source'])
           end
         file_dest = File.join(transfer_spec['destination_root'],file_dest)
-        @gw_api.call({operation: 'GET',subpath: "download/#{transfer_uuid}",save_to_file: file_dest})
+        @gw_api.call({operation: 'GET',subpath: "v1/download/#{transfer_uuid}",save_to_file: file_dest})
       end
 
       # start FASP transfer based on transfer spec (hash table)
@@ -200,15 +252,22 @@ module Aspera
 
       private
 
-      def initialize(params)
-        raise 'params must be Hash' unless params.is_a?(Hash)
-        params = params.symbolize_keys
-        raise 'must have only one param: url' unless params.keys.eql?([:url])
+      def initialize(opts)
+        Log.log.debug("local options= #{opts}")
+        # set default options and override if specified
+        @options = DEFAULT_OPTIONS.dup
+        raise "httpgw agent parameters (transfer_info): expecting Hash, but have #{opts.class}" unless opts.is_a?(Hash)
+        opts.symbolize_keys.each do |k,v|
+          raise "httpgw agent parameter: Unknown: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map(&:to_s).join(',')}" unless DEFAULT_OPTIONS.has_key?(k)
+          @options[k] = v
+        end
+        raise 'missing param: url' if @options[:url].nil?
+        # remove /v1 from end
+        @options[:url].gsub(%r{/v1/*$},'')
         super()
-        @gw_api = Rest.new({base_url: params[:url]})
-        api_info = @gw_api.read('info')[:data]
-        Log.log.info(api_info.to_s)
-        @upload_chunksize = 128_000 # TODO: configurable ?
+        @gw_api = Rest.new({base_url: @options[:url]})
+        @api_info = @gw_api.read('v1/info')[:data]
+        Log.log.info(@api_info.to_s)
       end
     end # AgentHttpgw
   end

data/lib/aspera/fasp/installation.rb

@@ -219,7 +219,7 @@ module Aspera
         return nil unless File.exist?(exe_path)
         exe_version = nil
         cmd_out = %x("#{exe_path}" #{vers_arg})
-        raise "An error occured when testing #{ascp_filename}: #{cmd_out}" unless $CHILD_STATUS == 0
+        raise "An error occurred when testing #{ascp_filename}: #{cmd_out}" unless $CHILD_STATUS == 0
         # get version from ascp, only after full extract, as windows requires DLLs (SSL/TLS/etc...)
         m = cmd_out.match(/ version ([0-9.]+)/)
         exe_version = m[1] unless m.nil?
@@ -269,12 +269,12 @@ module Aspera
         path(:aspera_conf)
         ascp_path = File.join(sdk_folder,ascp_filename)
         raise "No #{ascp_filename} found in SDK archive" unless File.exist?(ascp_path)
-        FileUtils.chmod(0755,ascp_path)
-        FileUtils.chmod(0755,ascp_path.gsub('ascp','ascp4'))
+        Environment.restrict_file_access(ascp_path, mode: 0755)
+        Environment.restrict_file_access(ascp_path.gsub('ascp','ascp4'), mode: 0755)
         ascp_version = get_ascp_version(File.join(sdk_folder,ascp_filename))
         trd_path = transferd_filepath
         Log.log.warn("No #{trd_path} in SDK archive") unless File.exist?(trd_path)
-        FileUtils.chmod(0755,trd_path) if File.exist?(trd_path)
+        Environment.restrict_file_access(trd_path, mode: 0755) if File.exist?(trd_path)
         transferd_version = get_exe_version(trd_path,'version')
         sdk_version = transferd_version || ascp_version
         File.write(File.join(sdk_folder,PRODUCT_INFO),"<product><name>IBM Aspera SDK</name><version>#{sdk_version}</version></product>")

data/lib/aspera/fasp/parameters.rb

@@ -51,7 +51,19 @@ module Aspera
             param[:cli] =
               case i[:cltype]
               when :envvar then 'env:' + i[:clvarname]
-              when :opt_without_arg,:opt_with_arg then i[:clswitch]
+              when :opt_without_arg then i[:clswitch]
+              when :opt_with_arg
+                values=if i.has_key?(:enum)
+                  ['enum']
+                elsif i[:accepted_types].is_a?(Array)
+                  i[:accepted_types]
+                elsif !i[:accepted_types].nil?
+                  [i[:accepted_types]]
+                else
+                  raise "error: #{param}"
+                end.map{|n|"{#{n}}"}.join('|')
+                conv=i.has_key?(:clconvert) ? '(conversion)' : ''
+                "#{i[:clswitch]} #{conv}#{values}"
               else ''
               end
             if i.has_key?(:enum)

data/lib/aspera/fasp/parameters.yaml

@@ -4,7 +4,7 @@
 # enum      : set with list of values for enum types accepted in transfer spec
 # tragents  : supported agents (for doc only)
 # required  : optional, default: false
-# cltype    : ascp: type of parameter
+# cltype    : ascp: type of parameter: :opt_with_arg,:opt_without_arg,:envvar,:defer,:ignore,
 # clswitch  : ascp: switch for ascp command line
 # clconvert : ascp: transform value: either a Hash with conversion values, or name of class
 # clvarname : ascp: name of env var
@@ -144,8 +144,11 @@ https_fallback_port:
   :cltype: :opt_with_arg
   :clswitch: "-t"
 move_after_transfer:
-  :desc: The relative path to which the files will be moved after the transfer at the source side.
+  :desc: The relative path to which the files will be moved after the transfer at the source side. Available as of 3.8.0.
   :cltype: :opt_with_arg
+  :tragents:
+  - :direct
+  - :node
 multi_session:
   :desc: |
     Use multi-session transfer. max 128.

data/lib/aspera/fasp/resume_policy.rb

@@ -45,7 +45,7 @@ module Aspera
             yield
             break
           rescue Fasp::Error => e
-            Log.log.warn("An error occured: #{e.message}");
+            Log.log.warn("An error occurred: #{e.message}");
             # failure in ascp
             if e.retryable?
               # exit if we exceed the max number of retry

data/lib/aspera/keychain/encrypted_hash.rb

@@ -1,134 +1,72 @@
 # frozen_string_literal: true
 
 require 'aspera/hash_ext'
-require 'openssl'
+require 'symmetric_encryption/core'
+require 'yaml'
 
 module Aspera
   module Keychain
-    class SimpleCipher
-      def initialize(key)
-        @key = Digest::SHA1.hexdigest(key+('*'*23))[0..23]
-        @cipher = OpenSSL::Cipher.new('DES-EDE3-CBC')
-      end
-
-      def encrypt(value)
-        @cipher.encrypt
-        @cipher.key = @key
-        s = @cipher.update(value) + @cipher.final
-        s.unpack1('H*')
-      end
-
-      def decrypt(value)
-        @cipher.decrypt
-        @cipher.key = @key
-        s = [value].pack('H*').unpack('C*').pack('c*')
-        @cipher.update(s) + @cipher.final
-      end
-    end
-
     # Manage secrets in a simple Hash
     class EncryptedHash
-      SEPARATOR = '%'
-      ACCEPTED_KEYS = %i[username url secret description].freeze
-      private_constant :SEPARATOR
-      attr_reader :legacy_detected
-      def initialize(values)
-        raise 'values shall be Hash' unless values.is_a?(Hash)
-        @all_secrets = values
+      CIPHER_NAME='aes-256-cbc'
+      ACCEPTED_KEYS = %i[label username password url description].freeze
+      def initialize(path,current_password)
+        @path=path
+        self.password=current_password
+        raise 'path to vault file shall be String' unless @path.is_a?(String)
+        @all_secrets=File.exist?(@path) ? YAML.load_stream(@cipher.decrypt(File.read(@path))).first : {}
       end
 
-      def identifier(options)
-        return options[:username] if options[:url].to_s.empty?
-        %i[url username].map{|s|options[s]}.join(SEPARATOR)
+      def password=(new_password)
+        key_bytes=CIPHER_NAME.split('-')[1].to_i/8
+        # derive key from passphrase
+        key="#{new_password}#{"\x0"*key_bytes}"[0..(key_bytes-1)]
+        Log.log.debug("key=[#{key}],#{key.length}")
+        SymmetricEncryption.cipher=@cipher = SymmetricEncryption::Cipher.new(cipher_name: CIPHER_NAME,key: key,encoding: :none)
+      end
+
+      def save
+        File.write(@path, @cipher.encrypt(YAML.dump(@all_secrets)),encoding: 'BINARY')
       end
 
       def set(options)
         raise 'options shall be Hash' unless options.is_a?(Hash)
         unsupported = options.keys - ACCEPTED_KEYS
         raise "unsupported options: #{unsupported}" unless unsupported.empty?
-        username = options[:username]
-        raise 'options shall have username' if username.nil?
-        url = options[:url]
-        raise 'options shall have username' if url.nil?
-        secret = options[:secret]
-        raise 'options shall have secret' if secret.nil?
-        key = identifier(options)
-        raise "secret #{key} already exist, delete first" if @all_secrets.has_key?(key)
-        obj = {username: username, url: url, secret: SimpleCipher.new(key).encrypt(secret)}
-        obj[:description] = options[:description] if options.has_key?(:description)
-        @all_secrets[key] = obj.stringify_keys
-        nil
+        label = options.delete(:label)
+        raise "secret #{label} already exist, delete first" if @all_secrets.has_key?(label)
+        @all_secrets[label] = options.symbolize_keys
+        save
       end
 
       def list
         result = []
-        legacy_detected=false
-        @all_secrets.each do |name,value|
-          normal = # normalized version
-            case value
-            when String
-              legacy_detected=true
-              {username: name, url: '', secret: value}
-            when Hash then value.symbolize_keys
-            else raise 'error secret must be String (legacy) or Hash (new)'
-            end
-          normal[:description] = '' unless normal.has_key?(:description)
-          extraneous_keys=normal.keys - ACCEPTED_KEYS
-          Log.log.error("wrongs keys in secret hash: #{extraneous_keys.map(&:to_s).join(',')}") unless extraneous_keys.empty?
+        @all_secrets.each do |label,values|
+          normal = values.symbolize_keys
+          normal[:label] = label
+          ACCEPTED_KEYS.each{|k|normal[k] = '' unless normal.has_key?(k)}
           result.push(normal)
         end
-        Log.log.warn('Legacy vault format detected in config file, please refer to documentation to convert to new format.') if legacy_detected
         return result
       end
 
       def delete(options)
         raise 'options shall be Hash' unless options.is_a?(Hash)
-        unsupported = options.keys - %i[username url]
+        unsupported = options.keys - %i[label]
         raise "unsupported options: #{unsupported}" unless unsupported.empty?
-        username = options[:username]
-        raise 'options shall have username' if username.nil?
-        url = options[:url]
-        key = nil
-        if !url.nil?
-          extk = identifier(options)
-          key = extk if @all_secrets.has_key?(extk)
-        end
-        # backward compatibility: TODO: remove in future ? (make url mandatory ?)
-        key = username if key.nil? && @all_secrets.has_key?(username)
-        raise 'no such secret' if key.nil?
-        @all_secrets.delete(key)
+        label=options[:label]
+        @all_secrets.delete(label)
+        save
       end
 
       def get(options)
         raise 'options shall be Hash' unless options.is_a?(Hash)
-        unsupported = options.keys - %i[username url]
+        unsupported = options.keys - %i[label]
         raise "unsupported options: #{unsupported}" unless unsupported.empty?
-        username = options[:username]
-        raise 'options shall have username' if username.nil?
-        url = options[:url]
-        info = nil
-        if !url.nil?
-          info = @all_secrets[identifier(options)]
-        end
-        # backward compatibility: TODO: remove in future ? (make url mandatory ?)
-        if info.nil?
-          info = @all_secrets[username]
-        end
-        result = options.clone
-        case info
-        when NilClass
-          raise "no such secret: [#{url}|#{username}] in #{@all_secrets.keys.join(',')}"
-        when String
-          result[:secret] = info
-          result[:description] = ''
-        when Hash
-          info=info.symbolize_keys
-          key = identifier(options)
-          plain = SimpleCipher.new(key).decrypt(info[:secret]) rescue info[:secret]
-          result[:secret] = plain
-          result[:description] = info[:description]
-        else raise "#{info.class} is not an expected type"
-        end
+        label=options[:label]
+        result = @all_secrets[label].clone
+        raise "no such entry #{label}" if result.nil?
+        result[:label]=label
         return result
       end
     end