aspera-cli 4.20.0 → 4.21.0

data/bin/asession

@@ -6,7 +6,7 @@ Kernel.load(File.join(gem_lib_folder, 'aspera/coverage.rb'))
 $LOAD_PATH.unshift(gem_lib_folder)
 require 'aspera/agent/direct'
 require 'aspera/cli/extended_value'
-require 'aspera/ascp/installation'
+require 'aspera/products/trsdk'
 require 'aspera/log'
 require 'json'
 # extended transfer spec parameter (only used in asession)
@@ -77,7 +77,7 @@ if session_spec.key?(PARAM_TMP_FILE_LIST_FOLDER)
   Aspera::Transfer::Parameters.file_list_folder = session_spec[PARAM_TMP_FILE_LIST_FOLDER]
 end
 session_spec[PARAM_SDK] = File.join(Dir.home, '.aspera', 'sdk') unless session_spec.key?(PARAM_SDK)
-Aspera::Ascp::Installation.instance.sdk_folder = session_spec[PARAM_SDK]
+Aspera::Products::Trsdk.sdk_directory = session_spec[PARAM_SDK]
 session_spec[PARAM_AGENT] = {} unless session_spec.key?(PARAM_AGENT)
 agent_params = session_spec[PARAM_AGENT]
 agent_params['quiet'] = true

data/lib/aspera/agent/alpha.rb

@@ -2,9 +2,9 @@
 
 require 'aspera/agent/base'
 require 'aspera/rest'
-require 'aspera/log'
-require 'aspera/json_rpc'
 require 'aspera/environment'
+require 'aspera/json_rpc'
+require 'aspera/products/alpha'
 require 'securerandom'
 
 module Aspera
@@ -15,9 +15,8 @@ module Aspera
       START_URIS = ['aspera://', 'aspera://', 'aspera://']
       # delay between each try to start the app
       SLEEP_SEC_BETWEEN_RETRY = 5
-      APP_IDENTIFIER = 'com.ibm.software.aspera.desktop'
-      APP_NAME = 'Aspera Desktop Alpha Client'
       private_constant :START_URIS, :SLEEP_SEC_BETWEEN_RETRY
+
       def initialize(**base_options)
         @application_id = SecureRandom.uuid
         @xfer_id = nil
@@ -34,23 +33,19 @@ module Aspera
         rescue Errno::ECONNREFUSED => e
           start_url = START_URIS[method_index]
           method_index += 1
-          raise StandardError, "Unable to start #{APP_NAME} #{method_index} times" if start_url.nil?
-          Log.log.warn{"#{APP_NAME} is not started (#{e}). Trying to start it ##{method_index}..."}
+          raise StandardError, "Unable to start #{Products::Alpha::APP_NAME} #{method_index} times" if start_url.nil?
+          Log.log.warn{"#{Products::Alpha::APP_NAME} is not started (#{e}). Trying to start it ##{method_index}..."}
           if !Environment.open_uri_graphical(start_url)
             Environment.open_uri_graphical('https://www.ibm.com/aspera/connect/')
-            raise StandardError, "#{APP_NAME} is not installed"
+            raise StandardError, "#{Products::Alpha::APP_NAME} is not installed"
           end
           sleep(SLEEP_SEC_BETWEEN_RETRY)
           retry
         end
       end
 
-      def sdk_log_file
-        File.join(Dir.home, 'Library', 'Logs', APP_IDENTIFIER, 'ibm-aspera-desktop.log')
-      end
-
       def aspera_client_api_url
-        log_file = sdk_log_file
+        log_file = Products::Alpha.log_file
         url = nil
         File.open(log_file, 'r') do |file|
           file.each_line do |line|

data/lib/aspera/agent/connect.rb

@@ -1,7 +1,10 @@
 # frozen_string_literal: true
 
 require 'aspera/agent/base'
+require 'aspera/products/connect'
+require 'aspera/products/other'
 require 'aspera/rest'
+require 'aspera/environment'
 require 'securerandom'
 
 module Aspera
@@ -20,7 +23,7 @@ module Aspera
         raise 'Using connect requires a graphical environment' if !Environment.default_gui_mode.eql?(:graphical)
         method_index = 0
         begin
-          connect_url = Ascp::Products.connect_uri
+          connect_url = connect_api_url
           Log.log.debug{"found: #{connect_url}"}
           @connect_api = Rest.new(
             base_url: "#{connect_url}/v5/connect", # could use v6 also now
@@ -43,6 +46,21 @@ module Aspera
         end
       end
 
+      # @return the file path of local connect where API's URI can be read
+      def connect_api_url
+        connect_locations = Products::Other.find(Products::Connect.locations).first
+        raise "Product: #{name} not found, please install." if connect_locations.nil?
+        folder = File.join(connect_locations[:run_root], 'var', 'run')
+        ['', 's'].each do |ext|
+          uri_file = File.join(folder, "http#{ext}.uri")
+          Log.log.debug{"checking connect port file: #{uri_file}"}
+          if File.exist?(uri_file)
+            return File.open(uri_file, &:gets).strip
+          end
+        end
+        raise "no connect uri file found in #{folder}"
+      end
+
       def start_transfer(transfer_spec, token_regenerator: nil)
         if transfer_spec['direction'] == 'send'
           Log.log.warn{"Connect requires upload selection using GUI, ignoring #{transfer_spec['paths']}".red}

data/lib/aspera/agent/direct.rb

@@ -72,7 +72,7 @@ module Aspera
         session = {
           id:                nil, # SessionId from INIT message in mgt port
           job_id:            SecureRandom.uuid, # job id (regroup sessions)
-          ts:                transfer_spec,     # transfer spec
+          ts:                transfer_spec,     # global transfer spec
           thread:            nil,               # Thread object monitoring management port, not nil when pushed to :sessions
           error:             nil,               # exception if failed
           io:                nil,               # management port server socket
@@ -84,7 +84,7 @@ module Aspera
         if multi_session_info.nil?
           Log.log.debug('Starting single session thread')
           # single session for transfer : simple
-          session[:thread] = Thread.new(session) {|session_info|transfer_thread_entry(session_info)}
+          session[:thread] = Thread.new {transfer_thread_entry(session)}
           @sessions.push(session)
         else
           Log.log.debug('Starting multi session threads')
@@ -101,7 +101,7 @@ module Aspera
             # option: increment (default as per ascp manual) or not (cluster on other side ?)
             args.unshift('-O', (multi_session_info[:udp_base] + i - 1).to_s) if @multi_incr_udp
             # finally start the thread
-            this_session[:thread] = Thread.new(this_session) {|session_info|transfer_thread_entry(session_info)}
+            this_session[:thread] = Thread.new {transfer_thread_entry(this_session)}
             @sessions.push(this_session)
           end
         end
@@ -132,18 +132,18 @@ module Aspera
 
       # @return [Array] list of sessions for a job
       def sessions_by_job(job_id)
-        @sessions.select{|session_info| session_info[:job_id].eql?(job_id)}
+        @sessions.select{|session| session[:job_id].eql?(job_id)}
       end
 
-      # This is the low level method to start the transfer process
+      # This is the low level method to start the transfer process.
+      # Typically started in a thread.
       # Start process with management port.
-      # returns
-      # raises FaspError on error
       # @param session this session information, keys :io and :token_regenerator
-      # @param env  [Hash]   environment variables
-      # @param name [Symbol] name of executable: :ascp, :ascp4 or :async
-      # @param args [Array]  command line arguments
+      # @param env  [Hash]   environment variables (comes from ascp_args)
+      # @param name [Symbol] name of executable: :ascp, :ascp4 or :async (comes from ascp_args)
+      # @param args [Array]  command line arguments (comes from ascp_args)
       # @return [nil] when process has exited
+      # @throw FaspError on error
       def start_and_monitor_process(
         session:,
         env:,
@@ -159,6 +159,8 @@ module Aspera
           mgt_server_socket = socket_class.new(Socket::AF_INET, Socket::SOCK_STREAM, 0)
           # open any available (0) local TCP port for use as management port
           mgt_server_socket.bind(Addrinfo.tcp(LISTEN_LOCAL_ADDRESS, SELECT_AVAILABLE_PORT))
+          # make port ready to accept connections, before starting ascp
+          mgt_server_socket.listen(1)
           # build arguments and add mgt port
           command_arguments = if name.eql?(:async)
             ["--exclusive-mgmt-port=#{mgt_server_socket.local_address.ip_port}"]
@@ -170,8 +172,8 @@ module Aspera
           command_path = Ascp::Installation.instance.path(name)
           command_pid = Environment.secure_spawn(env: env, exec: command_path, args: command_arguments)
           notify_progress(:pre_start, session_id: nil, info: "waiting for #{name} to start")
-          mgt_server_socket.listen(1)
           # TODO: timeout does not work when Process.spawn is used... until process exits, then it works
+          # So we use select to detect that anything happens on the socket (connection)
           Log.log.debug{"before select, timeout: #{@spawn_timeout_sec}"}
           readable, _, _ = IO.select([mgt_server_socket], nil, nil, @spawn_timeout_sec)
           Log.log.debug('after select, before accept')
@@ -192,6 +194,8 @@ module Aspera
             next unless event
             # event is ready
             Log.log.trace1{Log.dump(:management_port, event)}
+            # store latest event by type
+            session[:id] = event['SessionId'] if event['Type'].eql?('INIT')
             @management_cb&.call(event)
             process_progress(event)
             Log.log.error((event['Description']).to_s) if event['Type'].eql?('FILEERROR') # cspell:disable-line
@@ -242,10 +246,13 @@ module Aspera
         nil
       end
 
+      attr_reader :sessions
+
       private
 
       # notify progress to callback
       # @param event management port event
+      # @param session sessin object
       def process_progress(event)
         session_id = event['SessionId']
         case event['Type']
@@ -282,14 +289,6 @@ module Aspera
         end
       end
 
-      # @return [Hash] session information
-      def session_by_id(id)
-        matches = @sessions.select{|session_info| session_info[:id].eql?(id)}
-        raise 'no such session' if matches.empty?
-        raise 'more than one session' if matches.length > 1
-        return matches.first
-      end
-
       # send command to management port of command (used in `asession)
       # @param job_id identified transfer process
       # @param session_index index of session (for multi session)
@@ -297,18 +296,10 @@ module Aspera
       # {'type'=>'START','source'=>_path_,'destination'=>_path_}
       # {'type'=>'DONE'}
       def send_command(job_id, data)
-        session = session_by_id(job_id)
+        session = @sessions.find{|session| session[:job_id].eql?(job_id)}
         Log.log.debug{"command: #{data}"}
-        # build command
-        command = data
-          .keys
-          .map{|k|"#{k.capitalize}: #{data[k]}"}
-          .unshift(MGT_HEADER)
-          .push('', '')
-          .join("\n")
-        session[:io].puts(command)
+        session[:io].puts(Ascp::Management.command_to_stream(data))
       end
-      attr_reader :sessions
 
       # options for initialize (same as values in option transfer_info)
       # @param ascp_args [Array] additional arguments to ascp

data/lib/aspera/agent/node.rb

@@ -50,14 +50,6 @@ module Aspera
         return @node_api
       end
 
-      # use this to set the node_api end point before using the class.
-      def node_api=(new_value)
-        if !@node_api.nil? && !new_value.nil?
-          Log.log.warn('overriding existing node api value')
-        end
-        @node_api = new_value
-      end
-
       # generic method
       def start_transfer(transfer_spec, token_regenerator: nil)
         # add root id if access key
@@ -120,9 +112,7 @@ module Aspera
             # Bug in HSTS ? transfer is marked failed, but there is no reason
             break if transfer_data['error_code'].eql?(0) && transfer_data['error_desc'].empty?
             raise Transfer::Error, "status: #{transfer_data['status']}. code: #{transfer_data['error_code']}. description: #{transfer_data['error_desc']}"
-          else
-            Log.log.warn{"transfer_data -> #{transfer_data}"}
-            raise Transfer::Error, "status: #{transfer_data['status']}. code: #{transfer_data['error_code']}. description: #{transfer_data['error_desc']}"
+          else Aspera.error_unexpected_value(transfer_data['status']){"transfer_data -> #{transfer_data}"}
           end
           sleep(1.0)
         end

data/lib/aspera/agent/trsdk.rb

@@ -1,10 +1,8 @@
 # frozen_string_literal: true
 
 require 'aspera/agent/base'
-require 'aspera/ascp/installation'
+require 'aspera/products/trsdk'
 require 'aspera/temp_file_manager'
-require 'aspera/log'
-require 'aspera/assert'
 require 'json'
 require 'uri'
 require 'transfer_services_pb'
@@ -18,25 +16,6 @@ module Aspera
       PORT_SEP = ':'
       # port zero means select a random available high port
       AUTO_LOCAL_TCP_PORT = "#{PORT_SEP}0"
-      class << self
-        # Well, the port number is only in log file
-        def daemon_port_from_log(log_file)
-          result = nil
-          # if port is zero, a dynamic port was created, get it
-          File.open(log_file, 'r') do |file|
-            file.each_line do |line|
-              # Well, it's tricky to depend on log
-              if (m = line.match(/Info: API Server: Listening on ([^:]+):(\d+) /))
-                result = m[2].to_i
-                # no "break" , need to read last matching log line
-              end
-            end
-          end
-          raise 'Port not found in daemon logs' if result.nil?
-          Log.log.debug{"Got port #{result} from log"}
-          return result
-        end
-      end
 
       # @param url [String] URL of the transfer manager daemon
       # @param external [Boolean] if true, expect that an external daemon is already running
@@ -83,8 +62,8 @@ module Aspera
             fasp_runtime: {
               use_embedded: false,
               user_defined: {
-                bin: Ascp::Installation.instance.sdk_folder,
-                etc: Ascp::Installation.instance.sdk_folder
+                bin: Products::Trsdk.sdk_directory,
+                etc: Products::Trsdk.sdk_directory
               }
             }
           }
@@ -110,7 +89,7 @@ module Aspera
           Process.detach(@daemon_pid) if @keep
           at_exit {shutdown}
           # update port for next connection attempt (if auto high port was requested)
-          daemon_endpoint = "#{LOCAL_SOCKET_ADDR}#{PORT_SEP}#{self.class.daemon_port_from_log(log_stdout)}" if is_local_auto_port
+          daemon_endpoint = "#{LOCAL_SOCKET_ADDR}#{PORT_SEP}#{Products::Trsdk.daemon_port_from_log(log_stdout)}" if is_local_auto_port
           # local daemon started, try again
           retry
         end

data/lib/aspera/api/aoc.rb

@@ -70,6 +70,7 @@ module Aspera
 
         # split host of http://myorg.asperafiles.com in org and domain
         def split_org_domain(uri)
+          Aspera.assert_type(uri, URI)
           raise "No host found in URL.Please check URL format: https://myorg.#{SAAS_DOMAIN_PROD}" if uri.host.nil?
           parts = uri.host.split('.', 2)
           Aspera.assert(parts.length == 2){"expecting a public FQDN for #{PRODUCT_NAME}"}
@@ -77,6 +78,10 @@ module Aspera
           return %i{organization domain}.zip(parts).to_h
         end
 
+        def saas_url?(url)
+          url.include?(SAAS_DOMAIN_PROD)
+        end
+
         # @param url [String] URL of AoC public link
         # @return [Hash] information about public link, or nil if not a public link
         def link_info(url)

data/lib/aspera/api/node.rb

@@ -16,8 +16,6 @@ module Aspera
     class Node < Aspera::Rest
       SCOPE_SEPARATOR = ':'
       SCOPE_NODE_PREFIX = 'node.'
-      # prefix for ruby code for filter (deprecated)
-      MATCH_EXEC_PREFIX = 'exec:'
       MATCH_TYPES = [String, Proc, Regexp, NilClass].freeze
       SIGNATURE_DELIMITER = '==SIGNATURE=='
       BEARER_TOKEN_VALIDITY_DEFAULT = 86400
@@ -25,7 +23,7 @@ module Aspera
       REQUIRED_APP_INFO_FIELDS = %i[api app node_info workspace_id workspace_name].freeze
       # methods of @app_info[:api]
       REQUIRED_APP_API_METHODS = %i[node_api_from add_ts_tags].freeze
-      private_constant :SCOPE_SEPARATOR, :SCOPE_NODE_PREFIX, :MATCH_EXEC_PREFIX, :MATCH_TYPES,
+      private_constant :SCOPE_SEPARATOR, :SCOPE_NODE_PREFIX, :MATCH_TYPES,
         :SIGNATURE_DELIMITER, :BEARER_TOKEN_VALIDITY_DEFAULT,
         :REQUIRED_APP_INFO_FIELDS, :REQUIRED_APP_API_METHODS
 
@@ -47,27 +45,20 @@ module Aspera
       @use_node_cache = true
 
       class << self
+        # set to false to read transfer parameters from download_setup
         attr_accessor :use_standard_ports
+        # set to false to bypass cache in redis
         attr_accessor :use_node_cache
 
-        def cache_control_headers
-          h = {'Accept' => 'application/json'}
-          h[HEADER_X_CACHE_CONTROL] = 'no-cache' unless use_node_cache
-          h
-        end
-
         # For access keys: provide expression to match entry in folder
+        # @param match_expression one of supported types
+        # @return lambda function
         def file_matcher(match_expression)
           case match_expression
           when Proc then return match_expression
           when Regexp then return ->(f){f['name'].match?(match_expression)}
           when String
-            if match_expression.start_with?(MATCH_EXEC_PREFIX)
-              code = "->(f){#{match_expression[MATCH_EXEC_PREFIX.length..-1]}}"
-              Log.log.warn{"Use of prefix #{MATCH_EXEC_PREFIX} is deprecated (4.15), instead use: @ruby:'#{code}'"}
-              return Environment.secure_eval(code, __FILE__, __LINE__)
-            end
-            return lambda{|f|File.fnmatch(match_expression, f['name'], File::FNM_DOTMATCH)}
+            return ->(f){File.fnmatch(match_expression, f['name'], File::FNM_DOTMATCH)}
           when NilClass then return ->(_){true}
           else Aspera.error_unexpected_value(match_expression.class.name, exception_class: Cli::BadArgument)
           end
@@ -158,7 +149,13 @@ module Aspera
 
       # Call node API, possibly adding cache control header, as globally specified
       def read_with_cache(subpath, query=nil)
-        return call(operation: 'GET', subpath: subpath, headers: self.class.cache_control_headers, query: query)[:data]
+        headers = {'Accept' => 'application/json'}
+        headers[HEADER_X_CACHE_CONTROL] = 'no-cache' unless self.class.use_node_cache
+        return call(
+          operation: 'GET',
+          subpath:   subpath,
+          headers:   headers,
+          query:     query)[:data]
       end
 
       # update transfer spec with special additional tags
@@ -198,11 +195,11 @@ module Aspera
       # Recursively browse in a folder (with non-recursive method)
       # sub folders are processed if the processing method returns true
       # links are processed on the respective node
+      # @param method_sym [Symbol] processing method, arguments: entry, path, state
       # @param state [Object] state object sent to processing method
       # @param top_file_id [String] file id to start at (default = access key root file id)
       # @param top_file_path [String] path of top folder (default = /)
-      # @param block [Proc] processing method, arguments: entry, path, state
-      def process_folder_tree(method_sym:, state:, top_file_id:, top_file_path: '/')
+      def process_folder_tree(method_sym:, state:, top_file_id:, top_file_path: '/', query: nil)
         Aspera.assert(!top_file_path.nil?){'top_file_path not set'}
         Log.log.debug{"process_folder_tree: node=#{@app_info ? @app_info[:node_info]['id'] : 'nil'}, file id=#{top_file_id},  path=#{top_file_path}"}
         # start at top folder
@@ -215,7 +212,8 @@ module Aspera
           # get folder content
           folder_contents =
             begin
-              read("files/#{current_item[:id]}/files")
+              # TODO: use header
+              read_with_cache("files/#{current_item[:id]}/files")
             rescue StandardError => e
               Log.log.warn{"#{current_item[:path]}: #{e.class} #{e.message}"}
               []
@@ -228,21 +226,21 @@ module Aspera
               end
               next
             end
-            relative_path = File.join(current_item[:path], entry['name'])
-            Log.log.debug{"process_folder_tree: checking #{relative_path}"}
+            current_path = File.join(current_item[:path], entry['name'])
+            Log.log.debug{"process_folder_tree: checking #{current_path}"}
             # call block, continue only if method returns true
-            next unless send(method_sym, entry, relative_path, state)
+            next unless send(method_sym, entry, current_path, state)
             # entry type is file, folder or link
             case entry['type']
             when 'folder'
-              folders_to_explore.push({id: entry['id'], path: relative_path})
+              folders_to_explore.push({id: entry['id'], path: current_path})
             when 'link'
               if entry_has_link_information(entry)
                 node_id_to_node(entry['target_node_id'])&.process_folder_tree(
                   method_sym:    method_sym,
                   state:         state,
                   top_file_id:   entry['target_id'],
-                  top_file_path: relative_path)
+                  top_file_path: current_path)
               end
             end
           end
@@ -268,13 +266,19 @@ module Aspera
         return resolve_state[:result]
       end
 
-      def find_files(top_file_id, test_block)
+      def find_files(top_file_id, test_lambda)
         Log.log.debug{"find_files: file id=#{top_file_id}"}
-        find_state = {found: [], test_block: test_block}
+        find_state = {found: [], test_lambda: test_lambda}
         process_folder_tree(method_sym: :process_find_files, state: find_state, top_file_id: top_file_id)
         return find_state[:found]
       end
 
+      def list_files(top_file_id)
+        find_state = {found: []}
+        process_folder_tree(method_sym: :process_list_files, state: find_state, top_file_id: top_file_id)
+        return find_state[:found]
+      end
+
       def refreshed_transfer_token
         return oauth.token(refresh: true)
       end
@@ -296,6 +300,9 @@ module Aspera
       end
 
       # Create transfer spec for gen4
+      # @param file_id destination or source folder (id)
+      # @param direction one of Transfer::Spec::DIRECTION_SEND, Transfer::Spec::DIRECTION_RECEIVE
+      # @param ts_merge additional transfer spec to merge
       def transfer_spec_gen4(file_id, direction, ts_merge=nil)
         ak_name = nil
         ak_token = nil
@@ -309,6 +316,9 @@ module Aspera
           # TODO: token_generation_lambda = lambda{|do_refresh|oauth.token(refresh: do_refresh)}
           # get bearer token, possibly use cache
           ak_token = oauth.token
+        when :none
+          ak_name = params[:headers][HEADER_X_ASPERA_ACCESS_KEY]
+          ak_token = params[:headers]['Authorization']
         else Aspera.error_unexpected_value(auth_params[:type])
         end
         transfer_spec = {
@@ -357,8 +367,8 @@ module Aspera
 
       private
 
+      # method called in loop for each entry for `resolve_api_fid`
       def process_api_fid(entry, path, state)
-        # this block is called recursively for each entry in folder
         # stop digging here if not in right path
         return false unless entry['name'].eql?(state[:path].first)
         # ok it matches, so we remove the match, and continue digging
@@ -401,11 +411,18 @@ module Aspera
         return true
       end
 
+      # method called in loop for each entry for `find_files`
       def process_find_files(entry, path, state)
-        state[:found].push(entry.merge({'path' => path})) if state[:test_block].call(entry)
+        state[:found].push(entry.merge({'path' => path})) if state[:test_lambda].call(entry)
         # test all files deeply
         return true
       end
+
+      # method called in loop for each entry for `list_files`
+      def process_list_files(entry, path, state)
+        state[:found].push(entry.merge({'path' => path}))
+        return false
+      end
     end
   end
 end