aspera-cli 4.22.0 → 4.24.0

data/lib/aspera/api/aoc.rb

@@ -34,6 +34,8 @@ module Aspera
       # types of events for shared folder creation
       # Node events: permission.created permission.modified permission.deleted
       PERMISSIONS_CREATED = ['permission.created'].freeze
+      # Special name when creating workspace shared folders
+      ID_AK_ADMIN = 'ASPERA_ACCESS_KEY_ADMIN'
 
       private_constant :MAX_AOC_URL_REDIRECT,
         :CLIENT_ID_PREFIX,
@@ -43,7 +45,8 @@ module Aspera
         :JWT_AUDIENCE,
         :OAUTH_API_SUBPATH,
         :USER_INFO_FIELDS_MIN,
-        :PERMISSIONS_CREATED
+        :PERMISSIONS_CREATED,
+        :ID_AK_ADMIN
 
       # various API scopes supported
       SCOPE_FILES_SELF = 'self'
@@ -58,7 +61,7 @@ module Aspera
       # class static methods
       class << self
         # strings /Applications/Aspera\ Drive.app/Contents/MacOS/AsperaDrive|grep -E '.{100}==$'|base64 --decode
-        def get_client_info(client_name=nil)
+        def get_client_info(client_name = nil)
           client_key = client_name.nil? ? GLOBAL_CLIENT_APPS.first : client_name.to_sym
           return client_key, DataRepository.instance.item(client_key)
         end
@@ -79,23 +82,25 @@ module Aspera
         end
 
         def saas_url?(url)
-          url.include?(SAAS_DOMAIN_PROD)
+          URI.parse(url).host&.end_with?(".#{SAAS_DOMAIN_PROD}")
+        rescue URI::InvalidURIError
+          false
         end
 
         # @param url [String] URL of AoC public link
         # @return [Hash] information about public link, or nil if not a public link
         def link_info(url)
           final_uri = Rest.new(base_url: url, redirect_max: MAX_AOC_URL_REDIRECT).call(operation: 'GET')[:http].uri
-          Log.log.trace1{Log.dump(:final_uri, final_uri)}
+          Log.dump(:final_uri, final_uri, level: :trace1)
           org_domain = split_org_domain(final_uri)
           if (m = final_uri.path.match(%r{/oauth2/([^/]+)/login$}))
             org_domain[:organization] = m[1] if org_domain[:organization].nil?
           else
             Log.log.debug{"path=#{final_uri.path} does not end with /login"}
           end
-          raise 'AoC shall redirect to login page with a query' if final_uri.query.nil?
+          raise Error, 'AoC shall redirect to login page with a query' if final_uri.query.nil?
           query = Rest.query_to_h(final_uri.query)
-          Log.log.trace1{Log.dump(:query, query)}
+          Log.dump(:query, query, level: :trace1)
           # is that a public link ?
           if query.key?('token')
             Log.log.warn{"Unknown pub link path: #{final_uri.path}"} unless PUBLIC_LINK_PATHS.include?(final_uri.path)
@@ -106,7 +111,7 @@ module Aspera
               token:           query['token']
             }
           end
-          if query['state']
+          if query.key?('state')
             # can be a private link
             state_uri = URI.parse(query['state'])
             if state_uri.query && query['redirect_uri']
@@ -129,7 +134,7 @@ module Aspera
               end
             end
           end
-          Log.log.debug{Log.dump(:org_domain, org_domain)}
+          Log.dump(:org_domain, org_domain)
           return {
             instance_domain: org_domain[:domain],
             organization:    org_domain[:organization]
@@ -163,7 +168,7 @@ module Aspera
         }
         # analyze type of url
         url_info = AoC.link_info(url)
-        Log.log.debug{Log.dump(:url_info, url_info)}
+        Log.dump(:url_info, url_info)
         @private_link = url_info[:private_link]
         auth_params[:grant_method] = if url_info.key?(:token)
           :url_json
@@ -209,7 +214,7 @@ module Aspera
       end
 
       def public_link
-        return nil unless auth_params[:grant_method].eql?(:url_json)
+        return unless auth_params[:grant_method].eql?(:url_json)
         return @cache_url_token_info unless @cache_url_token_info.nil?
         # TODO: can there be several in list ?
         @cache_url_token_info = read('url_tokens').first
@@ -242,12 +247,12 @@ module Aspera
       end
 
       def workspace
-        raise 'internal error: AoC workspace context is not set' if @workspace_info.nil?
+        Aspera.assert(!@workspace_info.nil?){'AoC workspace context is not set'}
         @workspace_info
       end
 
       def home
-        raise 'internal error: AoC home context is not set' if @home_info.nil?
+        Aspera.assert(!@home_info.nil?){'AoC home context is not set'}
         @home_info
       end
 
@@ -291,8 +296,8 @@ module Aspera
               name: ws_info['name']
             }
           end
-        Log.log.debug{Log.dump(:context, @workspace_info)}
-        return nil unless application.eql?(:files)
+        Log.dump(:context, @workspace_info)
+        return unless application.eql?(:files)
         @home_info =
           if !public_link.nil?
             assert_public_link_types(['view_shared_file'])
@@ -319,7 +324,7 @@ module Aspera
             }
           end
         raise "Cannot get user's home node id, check your default workspace or specify one" if @home_info[:node_id].to_s.empty?
-        Log.log.debug{Log.dump(:context, @home_info)}
+        Log.dump(:context, @home_info)
       end
 
       # @param node_id [String] identifier of node in AoC
@@ -331,9 +336,7 @@ module Aspera
       def node_api_from(node_id:, workspace_id: nil, workspace_name: nil, scope: Node::SCOPE_USER, package_info: nil)
         Aspera.assert_type(node_id, String)
         node_info = read("nodes/#{node_id}")
-        if workspace_name.nil? && !workspace_id.nil?
-          workspace_name = read("workspaces/#{workspace_id}")['name']
-        end
+        workspace_name = read("workspaces/#{workspace_id}")['name'] if workspace_name.nil? && !workspace_id.nil?
         app_info = {
           api:            self, # for callback
           app:            package_info.nil? ? FILES_APP : PACKAGES_APP,
@@ -342,7 +345,7 @@ module Aspera
           workspace_name: workspace_name
         }
         if PACKAGES_APP.eql?(app_info[:app])
-          raise 'package info required' if package_info.nil?
+          Aspera.assert(!package_info.nil?){'package info required'}
           app_info[:package_id] = package_info['id']
           app_info[:package_name] = package_info['name']
         end
@@ -380,7 +383,7 @@ module Aspera
         Aspera.assert(pkg_data.key?('metadata')){"package requires metadata: #{meta_schema}"}
         pkg_meta = pkg_data['metadata']
         Aspera.assert_type(pkg_meta, Array){'metadata'}
-        Log.log.debug{Log.dump(:metadata, pkg_meta)}
+        Log.dump(:metadata, pkg_meta)
         pkg_meta.each do |field|
           Aspera.assert_type(field, Hash){'metadata field'}
           Aspera.assert(field.key?('name')){'metadata field must have name'}
@@ -440,7 +443,7 @@ module Aspera
         end
         # replace with resolved elements
         package_data[recipient_list_field] = resolved_list
-        return nil
+        return
       end
 
       # CLI allows simplified format for metadata: transform if necessary for API
@@ -458,7 +461,7 @@ module Aspera
           pkg_data['metadata'] = api_meta
         else Aspera.error_unexpected_value(pkg_meta.class)
         end
-        return nil
+        return
       end
 
       # create a package
@@ -477,18 +480,24 @@ module Aspera
 
         validate_metadata(package_data) if validate_meta
 
+        # tell AoC what to expect in package: 1 transfer (can also be done after transfer)
+        # TODO: if multi session was used we should probably tell
+        # also, currently no "multi-source" , i.e. only from client-side files, unless "node" agent is used
+        # `single_source` is required to allow web UI to ask for CSEAR password on download, see API doc
+        package_data.merge!({
+          'single_source'      => true,
+          'sent'               => true,
+          'transfers_expected' => 1
+        })
+
         #  create a new package container
         created_package = create('packages', package_data)
 
         package_node_api = node_api_from(
           node_id: created_package['node_id'],
           workspace_id: created_package['workspace_id'],
-          package_info: created_package)
-
-        # tell AoC what to expect in package: 1 transfer (can also be done after transfer)
-        # TODO: if multi session was used we should probably tell
-        # also, currently no "multi-source" , i.e. only from client-side files, unless "node" agent is used
-        update("packages/#{created_package['id']}", {'sent' => true, 'transfers_expected' => 1})
+          package_info: created_package
+        )
 
         return {
           spec: package_node_api.transfer_spec_gen4(created_package['contents_file_id'], Transfer::Spec::DIRECTION_SEND),
@@ -507,8 +516,10 @@ module Aspera
         transfer_spec.deep_merge!({
           'tags' => {
             Transfer::Spec::TAG_RESERVED => {
+              'app'      => app_info[:app],
               'usage_id' => "aspera.files.workspace.#{app_info[:workspace_id]}", # activity tracking
               'files'    => {
+                'node_id'               => app_info[:node_info]['id'],
                 'files_transfer_action' => "#{transfer_type}_#{app_info[:app].gsub(/s$/, '')}",
                 'workspace_name'        => app_info[:workspace_name], # activity tracking
                 'workspace_id'          => app_info[:workspace_id]
@@ -527,8 +538,15 @@ module Aspera
         case app_info[:app]
         when FILES_APP
           file_id = transfer_spec['tags'][Transfer::Spec::TAG_RESERVED]['node']['file_id']
-          transfer_spec.deep_merge!({'tags' => {Transfer::Spec::TAG_RESERVED => {'files' => {'parentCwd' => "#{app_info[:node_info]['id']}:#{file_id}"}}}}) \
-            unless transfer_spec.key?('remote_access_key')
+          transfer_spec.deep_merge!({
+            'tags' => {
+              Transfer::Spec::TAG_RESERVED => {
+                'files' => {
+                  'parentCwd' => "#{app_info[:node_info]['id']}:#{file_id}"
+                }
+              }
+            }
+          }) unless transfer_spec.key?('remote_access_key')
         when PACKAGES_APP
           transfer_spec.deep_merge!({
             'tags' => {
@@ -542,21 +560,14 @@ module Aspera
             }
           })
         end
-        transfer_spec['tags'][Transfer::Spec::TAG_RESERVED]['files']['node_id'] = app_info[:node_info]['id']
-        transfer_spec['tags'][Transfer::Spec::TAG_RESERVED]['app'] = app_info[:app]
       end
 
-      ID_AK_ADMIN = 'ASPERA_ACCESS_KEY_ADMIN'
       # Callback from Plugins::Node
       # add application specific tags to permissions creation
       # @param perm_data [Hash] parameters for creating permissions
       # @param app_info [Hash] application information
       def permissions_set_create_params(perm_data:, app_info:)
-        # workspace shared folder:
-        # access_id = "#{ID_AK_ADMIN}_WS_#{app_info[:workspace_id]}"
         defaults = {
-          # 'access_type'   => 'user', # mandatory: user or group
-          # 'access_id'     => access_id, # id of user or group or special
           'tags' => {
             Transfer::Spec::TAG_RESERVED => {
               'files' => {
@@ -567,8 +578,6 @@ module Aspera
                   'shared_by_user_id' => current_user_info['id'],
                   'shared_by_name'    => current_user_info['name'],
                   'shared_by_email'   => current_user_info['email'],
-                  # 'shared_with_name'  => access_id,
-                  # 'share_as'  => new_name_for_folder,
                   'access_key'        => app_info[:node_info]['access_key'],
                   'node'              => app_info[:node_info]['name']
                 }
@@ -578,19 +587,20 @@ module Aspera
         }
         perm_data.deep_merge!(defaults)
         tag_workspace = perm_data['tags'][Transfer::Spec::TAG_RESERVED]['files']['workspace']
-        case perm_data['with']
+        shared_with = perm_data.delete('with')
+        case shared_with
         when NilClass
         when ''
+          # workspace shared folder
           perm_data['access_type'] = 'user'
           perm_data['access_id'] = "#{ID_AK_ADMIN}_WS_#{app_info[:workspace_id]}"
           tag_workspace['shared_with_name'] = perm_data['access_id']
         else
-          entity_info = lookup_by_name('contacts', perm_data['with'], query: {'current_workspace_id' => app_info[:workspace_id]})
+          entity_info = lookup_by_name('contacts', shared_with, query: {'current_workspace_id' => app_info[:workspace_id]})
           perm_data['access_type'] = entity_info['source_type']
           perm_data['access_id'] = entity_info['source_id']
-          tag_workspace['shared_with_name'] = entity_info['email']
+          tag_workspace['shared_with_name'] = entity_info['email'] # TODO: check that ???
         end
-        perm_data.delete('with')
         if perm_data.key?('as')
           tag_workspace['share_as'] = perm_data['as']
           perm_data.delete('as')

data/lib/aspera/api/cos_node.rb

@@ -14,13 +14,13 @@ module Aspera
         def parameters_from_svc_credentials(service_credentials, bucket_region)
           # check necessary contents
           Aspera.assert_type(service_credentials, Hash){'service_credentials'}
-          Aspera::Log.dump('service_credentials', service_credentials)
+          Log.dump(:service_credentials, service_credentials)
           %w[apikey resource_instance_id endpoints].each do |field|
             Aspera.assert(service_credentials.key?(field)){"service_credentials must have a field: #{field}"}
           end
           # read endpoints from service provided in service credentials
           endpoints = Aspera::Rest.new(base_url: service_credentials['endpoints']).read('')
-          Aspera::Log.dump('endpoints', endpoints)
+          Log.dump(:endpoints, endpoints)
           endpoint = endpoints.dig('service-endpoints', 'regional', bucket_region, 'public', bucket_region)
           raise "no such region: #{bucket_region}" if endpoint.nil?
           return {
@@ -48,7 +48,8 @@ module Aspera
             grant_type:    'urn:ibm:params:oauth:grant-type:apikey',
             response_type: 'cloud_iam',
             apikey:        @api_key
-          })
+          }
+        )
         # read FASP connection information for bucket
         xml_result_text = s3_api.call(
           operation: 'GET',
@@ -57,7 +58,7 @@ module Aspera
           query:     {'faspConnectionInfo' => nil}
         )[:http].body
         ats_info = XmlSimple.xml_in(xml_result_text, {'ForceArray' => false})
-        Aspera::Log.dump('ats_info', ats_info)
+        Log.dump(:ats_info, ats_info)
         @storage_credentials = {
           'type'  => 'token',
           'token' => {TOKEN_FIELD => nil}
@@ -67,7 +68,8 @@ module Aspera
           auth:     {
             type:     :basic,
             username: ats_info['AccessKey']['Id'],
-            password: ats_info['AccessKey']['Secret']},
+            password: ats_info['AccessKey']['Secret']
+          },
           add_tspec: {'tags'=>{Transfer::Spec::TAG_RESERVED=>{'node'=>{'storage_credentials'=>@storage_credentials}}}}
           )
         # update storage_credentials AND Rest params

data/lib/aspera/api/httpgw.rb

@@ -67,9 +67,7 @@ module Aspera
                 @shared_info[:read_exception].nil? &&
                 (((@shared_info[:count][:sent_general] - @shared_info[:count][:received_general]) > 1) ||
                   ((@shared_info[:count][:received_v2_delimiter] - @shared_info[:count][:sent_v2_delimiter]) > 1))
-              if !@shared_info[:cond_var].wait(@shared_info[:mutex], 2.0)
-                Log.log.trace1{"#{LOG_WS_SEND}#{'timeout'.blue}: #{@shared_info[:count]}"}
-              end
+              Log.log.trace1{"#{LOG_WS_SEND}#{'timeout'.blue}: #{@shared_info[:count]}"} if !@shared_info[:cond_var].wait(@shared_info[:mutex], 2.0)
             end
           end
         end
@@ -110,7 +108,7 @@ module Aspera
         Log.log.debug{"#{LOG_WS_RECV}read thread started"}
         frame_parser = ::WebSocket::Frame::Incoming::Client.new(version: @ws_handshake.version)
         until @ws_io.eof?
-          begin # rubocop:disable Style/RedundantBegin
+          begin
             # ready byte by byte until frame is ready
             # blocking read
             byte = @ws_io.read(1)
@@ -138,16 +136,16 @@ module Aspera
       def upload(transfer_spec)
         # identify this session uniquely
         session_id = SecureRandom.uuid
-        @notify_cb&.call(:pre_start, session_id: nil, info: 'starting')
+        @notify_cb&.call(:sessions_init, info: 'starting')
         # process files to send, modify `paths` in transfer_spec
         files_to_send = process_upload_list(transfer_spec)
         # total size of all files is last element
         total_bytes_to_transfer = files_to_send.pop
-        Log.log.trace1{Log.dump(:modified_tspec, transfer_spec)}
-        Log.log.trace1{Log.dump(:files_to_send, files_to_send)}
+        Log.dump(:modified_tspec, transfer_spec, level: :trace1)
+        Log.dump(:files_to_send, files_to_send, level: :trace1)
         # TODO: check that this is available in endpoints: @api_info['endpoints']
         upload_url = File.join(@gw_root_url, @upload_version, 'upload')
-        @notify_cb&.call(:pre_start, session_id: nil, info: 'connecting wss')
+        @notify_cb&.call(:sessions_init, info: 'connecting wss')
         # open web socket to end point (equivalent to Net::HTTP.start)
         http_session = Rest.start_http_session(upload_url)
         # get the underlying socket i/o
@@ -233,7 +231,8 @@ module Aspera
         end
         # throttling may have skipped last one
         @notify_cb&.call(:transfer, session_id: session_id, info: session_sent_bytes)
-        @notify_cb&.call(:end, session_id: session_id)
+        @notify_cb&.call(:session_end, session_id: session_id)
+        @notify_cb&.call(:end)
         ws_send(ws_type: :close, data: nil)
         Log.log.debug("Finished upload, waiting for end of #{THR_RECV} thread.")
         @ws_read_thread.join
@@ -244,27 +243,29 @@ module Aspera
       end
 
       def download(transfer_spec)
-        transfer_spec['zip_required'] ||= false
         transfer_spec['source_root'] ||= '/'
+        default_file_name = transfer_spec['paths'].first['source']
+        source_is_folder = %w[. /].include?(default_file_name)
+        default_file_name = 'http_download' if source_is_folder
+        transfer_spec['zip_required'] ||= source_is_folder || transfer_spec['paths'].length > 1
         # is normally provided by application, like package name
         if !transfer_spec.key?('download_name')
           # by default it is the name of first file
-          download_name = File.basename(transfer_spec['paths'].first['source'], '.*')
-          # ands add indication of number of files if there is more than one
-          if transfer_spec['paths'].length > 1
-            download_name += " #{transfer_spec['paths'].length} Files"
-          end
+          download_name = File.basename(default_file_name, '.*')
+          # add indication of number of files if there is more than one
+          download_name += " #{transfer_spec['paths'].length} Files" if transfer_spec['paths'].length > 1
           transfer_spec['download_name'] = download_name
         end
+        # start transfer session on httpgw
         creation = create('download', {'transfer_spec' => transfer_spec})
         transfer_uuid = creation['url'].split('/').last
         file_name =
           if transfer_spec['zip_required'] || transfer_spec['paths'].length > 1
             # it is a zip file if zip is required or there is more than 1 file
-            transfer_spec['download_name'] + '.zip'
+            "#{transfer_spec['download_name']}.zip"
           else
             # it is a plain file if we don't require zip and there is only one file
-            File.basename(transfer_spec['paths'].first['source'])
+            File.basename(default_file_name)
           end
         file_path = File.join(transfer_spec['destination_root'], file_name)
         call(operation: 'GET', subpath: "download/#{transfer_uuid}", save_to_file: file_path)
@@ -288,13 +289,13 @@ module Aspera
         notify_cb:         nil,
         **opts
       )
-        Log.log.debug{Log.dump(:gw_url, url)}
+        Log.dump(:gw_url, url)
         # add scheme if missing
         url = "https://#{url}" unless url.match?(%r{^[a-z]{1,6}://})
-        raise 'GW URL shall be with scheme https' unless url.start_with?('https://')
+        raise Error, 'GW URL shall be with scheme https' unless url.start_with?('https://')
         # remove trailing slash and version (o=only once) if present
         # TODO: issue warning ?
-        url = url.gsub(%r{/+$}, '').gsub(%r{/#{API_V1}$}o, '')
+        url = url.chomp('/').gsub(%r{/#{API_V1}$}o, '')
         # assume GW is always under specific path (TODO: remove this ?)
         url = File.join(url, DEFAULT_BASE_PATH) unless url.end_with?(DEFAULT_BASE_PATH)
         @gw_root_url = url
@@ -305,7 +306,7 @@ module Aspera
         @notify_cb = notify_cb
         # get API info
         @api_info = read('info').freeze
-        Log.log.debug{Log.dump(:api_info, @api_info)}
+        Log.dump(:api_info, @api_info)
         # web socket endpoint: by default use v2 (newer gateways), without base64 encoding
         # is the latest supported? else revert to old api
         if !@upload_version.eql?(API_V1)
@@ -328,7 +329,7 @@ module Aspera
       # @return [Array] info on files to send
       def process_upload_list(transfer_spec)
         total_bytes_to_transfer = 0
-        source_prefix = transfer_spec.key?('source_root') && !transfer_spec['source_root'].empty? ? transfer_spec['source_root'] + '/' : ''
+        source_prefix = transfer_spec.key?('source_root') && !transfer_spec['source_root'].empty? ? "#{transfer_spec['source_root']}/" : ''
         files_to_send = []
         transfer_spec['paths'].each do |one_path|
           source_path = source_prefix + one_path['source']