aspera-cli 4.14.0 → 4.16.0

data/lib/aspera/node_simulator.rb

@@ -0,0 +1,214 @@
+# frozen_string_literal: true
+
+require 'aspera/log'
+require 'aspera/fasp/installation'
+require 'aspera/fasp/agent_direct'
+require 'webrick'
+require 'json'
+
+module Aspera
+  # this class answers the Faspex /send API and creates a package on Aspera on Cloud
+  class NodeSimulatorServlet < WEBrick::HTTPServlet::AbstractServlet
+    PATH_TRANSFERS = '/ops/transfers'
+    PATH_ONE_TRANSFER = %r{/ops/transfers/(.+)$}
+    # @param app_api [Aspera::AoC]
+    # @param app_context [String]
+    def initialize(server, credentials, transfer)
+      super(server)
+      @credentials = credentials
+      @xfer_manager = Aspera::Fasp::AgentDirect.new
+    end
+
+    def do_POST(request, response)
+      case request.path
+      when PATH_TRANSFERS
+        job_id = @xfer_manager.start_transfer(JSON.parse(request.body))
+        session = @xfer_manager.sessions_by_job(job_id).first
+        result = session[:ts].clone
+        result['id'] = job_id
+        set_json_response(response, result)
+        Log.log.debug{">>> transfer started: #{job_id}"}
+      else
+        set_json_response(response, [{error: 'Bad request'}], code: 400)
+      end
+    end
+
+    def do_GET(request, response)
+      case request.path
+      when '/info'
+        info = Aspera::Fasp::Installation.instance.ascp_info
+        set_json_response(response, {
+          application:                           'node',
+          current_time:                          Time.now.utc.iso8601(0),
+          version:                               info['ascp_version'].gsub(/ .*$/, ''),
+          license_expiration_date:               info['expiration_date'],
+          license_max_rate:                      info['maximum_bandwidth'],
+          os:                                    %x(uname -srv).chomp,
+          aej_status:                            'disconnected',
+          async_reporting:                       'yes',
+          transfer_activity_reporting:           'yes',
+          transfer_user:                         'xfer',
+          docroot:                               'file:////data/aoc/eudemo-sedemo',
+          node_id:                               '2bbdcc39-f789-4d47-8163-6767fc14f421',
+          cluster_id:                            '6dae2844-d1a9-47a5-916d-9b3eac3ea466',
+          acls:                                  [],
+          access_key_configuration_capabilities: {
+            transfer: %w[
+              cipher
+              policy
+              target_rate_cap_kbps
+              target_rate_kbps
+              preserve_timestamps
+              content_protection_secret
+              aggressiveness
+              token_encryption_key
+              byok_enabled
+              bandwidth_flow_network_rc_module
+              file_checksum_type],
+            server:   %w[
+              activity_event_logging
+              activity_file_event_logging
+              recursive_counts
+              aej_logging
+              wss_enabled
+              activity_transfer_ignore_skipped_files
+              activity_files_max
+              access_key_credentials_encryption_type
+              discovery
+              auto_delete
+              allow
+              deny]
+          },
+          capabilities:                          [
+            {name:  'sync', value: true},
+            {name:  'watchfolder', value: true},
+            {name:  'symbolic_links', value: true},
+            {name:  'move_file', value: true},
+            {name:  'move_directory', value: true},
+            {name:  'filelock', value: false},
+            {name:  'ssh_fingerprint', value: false},
+            {name:  'aej_version', value: '1.0'},
+            {name:  'page', value: true},
+            {name:  'file_id_version', value: '2.0'},
+            {name:  'auto_delete', value: false}],
+          settings:                              [
+            {name:  'content_protection_required', value: false},
+            {name:  'content_protection_strong_pass_required', value: false},
+            {name:  'filelock_restriction', value: 'none'},
+            {name:  'ssh_fingerprint', value: nil},
+            {name:  'wss_enabled', value: false},
+            {name:  'wss_port', value: 443}
+          ]})
+      when PATH_TRANSFERS
+        result = @xfer_manager.sessions.map { |session| job_to_transfer(session) }
+        set_json_response(response, result)
+      when PATH_ONE_TRANSFER
+        job_id = request.path.match(PATH_ONE_TRANSFER)[1]
+        set_json_response(response, job_to_transfer(@xfer_manager.sessions_by_job(job_id).first))
+      else
+        set_json_response(response, [{error: 'Unknown request'}], code: 400)
+      end
+    end
+
+    def set_json_response(response, json, code: 200)
+      response.status = code
+      response['Content-Type'] = 'application/json'
+      response.body = json.to_json
+      Log.log.trace1{Log.dump('response', json)}
+    end
+
+    def job_to_transfer(job)
+      session = {
+        id:                    'bafc72b8-366c-4501-8095-47208183d6b8',
+        client_node_id:        '',
+        server_node_id:        '2bbdcc39-f789-4d47-8163-6767fc14f421',
+        client_ip_address:     '192.168.0.100',
+        server_ip_address:     '5.10.114.4',
+        status:                'running',
+        retry_timeout:         3600,
+        retry_count:           0,
+        start_time_usec:       1701094040000000,
+        end_time_usec:         nil,
+        elapsed_usec:          405312,
+        bytes_transferred:     26,
+        bytes_written:         26,
+        bytes_lost:            0,
+        files_completed:       1,
+        directories_completed: 0,
+        target_rate_kbps:      500000,
+        min_rate_kbps:         0,
+        calc_rate_kbps:        9900,
+        network_delay_usec:    40000,
+        avg_rate_kbps:         0.51,
+        error_code:            0,
+        error_desc:            '',
+        source_statistics:     {
+          args_scan_attempted:  1,
+          args_scan_completed:  1,
+          paths_scan_attempted: 1,
+          paths_scan_failed:    0,
+          paths_scan_skipped:   0,
+          paths_scan_excluded:  0,
+          dirs_scan_completed:  0,
+          files_scan_completed: 1,
+          dirs_xfer_attempted:  0,
+          dirs_xfer_fail:       0,
+          files_xfer_attempted: 1,
+          files_xfer_fail:      0,
+          files_xfer_noxfer:    0
+        },
+        precalc:               {
+          enabled:              true,
+          status:               'ready',
+          bytes_expected:       0,
+          directories_expected: 0,
+          files_expected:       0,
+          files_excluded:       0,
+          files_special:        0,
+          files_failed:         1
+        }}
+      return {
+        id:                    '609a667d-642e-4290-9312-b4d20d3c0159',
+        status:                'running',
+        start_spec:            job[:ts],
+        sessions:              [session],
+        bytes_transferred:     26,
+        bytes_written:         26,
+        bytes_lost:            0,
+        avg_rate_kbps:         0.51,
+        files_completed:       1,
+        files_skipped:         0,
+        directories_completed: 0,
+        start_time_usec:       1701094040000000,
+        end_time_usec:         1701094040405312,
+        elapsed_usec:          405312,
+        error_code:            0,
+        error_desc:            '',
+        precalc:               {
+          status:               'ready',
+          bytes_expected:       0,
+          files_expected:       0,
+          directories_expected: 0,
+          files_special:        0,
+          files_failed:         1
+        },
+        files:                 [{
+          id:              'd1b5c112-82b75425-860745fc-93851671-64541bdd',
+          path:            '/workspaces/45071/packages/bYA_ilq73g.asp-package/contents/data_file.bin',
+          start_time_usec: 1701094040000000,
+          elapsed_usec:    105616,
+          end_time_usec:   1701094040001355,
+          status:          'completed',
+          error_code:      0,
+          error_desc:      '',
+          size:            26,
+          type:            'file',
+          checksum_type:   'none',
+          checksum:        nil,
+          start_byte:      0,
+          bytes_written:   26,
+          session_id:      'bafc72b8-366c-4501-8095-47208183d6b8'}]
+      }
+    end
+  end # NodeSimulatorServlet
+end # Aspera

data/lib/aspera/oauth.rb

@@ -3,6 +3,8 @@
 require 'aspera/open_application'
 require 'aspera/web_auth'
 require 'aspera/id_generator'
+require 'aspera/log'
+require 'aspera/assert'
 require 'base64'
 require 'date'
 require 'socket'
@@ -24,18 +26,23 @@ module Aspera
     # OAuth methods supported by default
     STD_AUTH_TYPES = %i[web jwt].freeze
 
-    # remove 5 minutes to account for time offset between client and server (TODO: configurable?)
-    JWT_ACCEPTED_OFFSET_SEC = 300
-    # one hour validity (TODO: configurable?)
-    JWT_EXPIRY_OFFSET_SEC = 3600
-    # tokens older than 30 minutes will be discarded from cache
-    TOKEN_CACHE_EXPIRY_SEC = 1800
-    # tokens valid for less than this duration will be regenerated
-    TOKEN_EXPIRATION_GUARD_SEC = 120
+    @@globals = { # rubocop:disable Style/ClassVars
+      # remove 5 minutes to account for time offset between client and server (TODO: configurable?)
+      jwt_accepted_offset_sec:    300,
+      # one hour validity (TODO: configurable?)
+      jwt_expiry_offset_sec:      3600,
+      # tokens older than 30 minutes will be discarded from cache
+      token_cache_expiry_sec:     1800,
+      # tokens valid for less than this duration will be regenerated
+      token_expiration_guard_sec: 120
+    }
+
     # a prefix for persistency of tokens (simplify garbage collect)
     PERSIST_CATEGORY_TOKEN = 'token'
+    # prefix for bearer token when in header
+    BEARER_PREFIX = 'Bearer '
 
-    private_constant :JWT_ACCEPTED_OFFSET_SEC, :JWT_EXPIRY_OFFSET_SEC, :TOKEN_CACHE_EXPIRY_SEC, :PERSIST_CATEGORY_TOKEN, :TOKEN_EXPIRATION_GUARD_SEC
+    private_constant :PERSIST_CATEGORY_TOKEN, :BEARER_PREFIX
 
     # persistency manager
     @persist = nil
@@ -45,10 +52,23 @@ module Aspera
     @id_handlers = {}
 
     class << self
+      def bearer_build(token)
+        return BEARER_PREFIX + token
+      end
+
+      def bearer_extract(token)
+        assert(bearer?(token)){'not a bearer token, wrong prefix'}
+        return token[BEARER_PREFIX.length..-1]
+      end
+
+      def bearer?(token)
+        return token.start_with?(BEARER_PREFIX)
+      end
+
       def persist_mgr=(manager)
         @persist = manager
         # cleanup expired tokens
-        @persist.garbage_collect(PERSIST_CATEGORY_TOKEN, TOKEN_CACHE_EXPIRY_SEC)
+        @persist.garbage_collect(PERSIST_CATEGORY_TOKEN, @@globals[:token_cache_expiry_sec])
       end
 
       def persist_mgr
@@ -88,26 +108,28 @@ module Aspera
       # @param id_create called to generate unique id for token, for cache
       def register_token_creator(id, lambda_create, id_create)
         Log.log.debug{"registering token creator #{id}"}
-        raise 'ERROR: requites Symbol and 2 lambdas' unless id.is_a?(Symbol) && lambda_create.is_a?(Proc) && id_create.is_a?(Proc)
+        assert_type(id, Symbol)
+        assert_type(lambda_create, Proc)
+        assert_type(id_create, Proc)
         @create_handlers[id] = lambda_create
         @id_handlers[id] = id_create
       end
 
       # @return one of the registered creators for the given create type
       def token_creator(id)
-        raise "token grant method unknown: '#{id}' (#{id.class})" unless @create_handlers.key?(id)
+        assert(@create_handlers.key?(id)){"token grant method unknown: '#{id}' (#{id.class})"}
         @create_handlers[id]
       end
 
       # list of identifiers found in creation parameters that can be used to uniquely identify the token
       def id_creator(id)
-        raise "id creator type unknown: #{id}/#{id.class}" unless @id_handlers.key?(id)
+        assert(@id_handlers.key?(id)){"id creator type unknown: #{id}/#{id.class}"}
         @id_handlers[id]
       end
     end # self
 
     # JSON Web Signature (JWS) compact serialization: https://datatracker.ietf.org/doc/html/rfc7515
-    register_decoder lambda { |token| parts = token.split('.'); raise 'not aoc token' unless parts.length.eql?(3); JSON.parse(Base64.decode64(parts[1]))} # rubocop:disable Style/Semicolon, Layout/LineLength
+    register_decoder lambda { |token| parts = token.split('.'); assert(parts.length.eql?(3)){'not aoc token'}; JSON.parse(Base64.decode64(parts[1]))} # rubocop:disable Style/Semicolon, Layout/LineLength
 
     # generic token creation, parameters are provided in :generic
     register_token_creator :generic, lambda { |oauth|
@@ -134,7 +156,7 @@ module Aspera
       OpenApplication.instance.uri(login_page_url)
       # wait for code in request
       received_params = web_server.received_request
-      raise 'wrong received state' unless random_state.eql?(received_params['state'])
+      assert(random_state.eql?(received_params['state'])){'wrong received state'}
       # exchange code for token
       return oauth.create_token(oauth.optional_scope_client_id(add_secret: true).merge(
         grant_type:   'authorization_code',
@@ -151,11 +173,11 @@ module Aspera
       require 'jwt'
       seconds_since_epoch = Time.new.to_i
       Log.log.info{"seconds=#{seconds_since_epoch}"}
-      raise 'missing JWT payload' unless oauth.specific_parameters[:payload].is_a?(Hash)
+      assert(oauth.specific_parameters[:payload].is_a?(Hash)){'missing JWT payload'}
       jwt_payload = {
-        exp: seconds_since_epoch + JWT_EXPIRY_OFFSET_SEC, # expiration time
-        nbf: seconds_since_epoch - JWT_ACCEPTED_OFFSET_SEC, # not before
-        iat: seconds_since_epoch - JWT_ACCEPTED_OFFSET_SEC + 1, # issued at (we tell a little in the past so that server always accepts)
+        exp: seconds_since_epoch + @@globals[:jwt_expiry_offset_sec], # expiration time
+        nbf: seconds_since_epoch - @@globals[:jwt_accepted_offset_sec], # not before
+        iat: seconds_since_epoch - @@globals[:jwt_accepted_offset_sec] + 1, # issued at (we tell a little in the past so that server always accepts)
         jti: SecureRandom.uuid # JWT id
       }.merge(oauth.specific_parameters[:payload])
       Log.log.debug{"JWT jwt_payload=[#{jwt_payload}]"}
@@ -173,14 +195,14 @@ module Aspera
     private
 
     # [M]=mandatory [D]=has default value [0]=accept nil
-    # :base_url            [M]  URL of authentication API
+    # :base_url            [M] URL of authentication API
     # :auth
-    # :grant_method        [M]  :generic, :web, :jwt, custom
+    # :grant_method        [M] :generic, :web, :jwt, [custom types]
     # :client_id           [0]
     # :client_secret       [0]
     # :scope               [0]
-    # :path_token          [D]  API end point to create a token
-    # :token_field         [D]  field in result that contains the token
+    # :path_token          [D] API end point to create a token
+    # :token_field         [D] field in result that contains the token
     # :jwt:private_key_obj [M] for type :jwt
     # :jwt:payload         [M] for type :jwt
     # :jwt:headers         [0] for type :jwt
@@ -189,18 +211,16 @@ module Aspera
     # :generic             [M] for type :generic
     def initialize(a_params)
       Log.log.debug{"auth=#{a_params}"}
-      # replace default values
+      # set default values if not set in parameters common to all types
       @generic_parameters = DEFAULT_CREATE_PARAMS.deep_merge(a_params)
-      # legacy
-      @generic_parameters[:grant_method] ||= @generic_parameters.delete(:crtype) if @generic_parameters.key?(:crtype) # cspell: disable-line
       # check that type is known
       self.class.token_creator(@generic_parameters[:grant_method])
       # specific parameters for the creation type
       @specific_parameters = @generic_parameters[@generic_parameters[:grant_method]]
       if @generic_parameters[:grant_method].eql?(:web) && @specific_parameters.key?(:redirect_uri)
         uri = URI.parse(@specific_parameters[:redirect_uri])
-        raise 'redirect_uri scheme must be http or https' unless %w[http https].include?(uri.scheme)
-        raise 'redirect_uri must have a port' if uri.port.nil?
+        assert(%w[http https].include?(uri.scheme)){'redirect_uri scheme must be http or https'}
+        assert(!uri.port.nil?){'redirect_uri must have a port'}
         # TODO: we could check that host is localhost or local address
       end
       rest_params = {
@@ -208,13 +228,14 @@ module Aspera
         redirect_max: 2
       }
       rest_params[:auth] = a_params[:auth] if a_params.key?(:auth)
+      # this is the OAuth API
       @api = Rest.new(rest_params)
-      # if needed use from api
+      # if those are needed use from @api
       @generic_parameters.delete(:base_url)
       @generic_parameters.delete(:auth)
       @generic_parameters.delete(@generic_parameters[:grant_method])
-      Log.dump(:generic_parameters, @generic_parameters)
-      Log.dump(:specific_parameters, @specific_parameters)
+      Log.log.debug{Log.dump(:generic_parameters, @generic_parameters)}
+      Log.log.debug{Log.dump(:specific_parameters, @specific_parameters)}
     end
 
     public
@@ -248,7 +269,7 @@ module Aspera
         @generic_parameters[:grant_method],
         self.class.id_creator(@generic_parameters[:grant_method]).call(self), # array, so we flatten later
         @generic_parameters[:scope],
-        @api.params.dig(%i[auth username])
+        @api.params.dig(*%i[auth username])
       ].flatten)
 
       # get token_data from cache (or nil), token_data is what is returned by /token
@@ -259,14 +280,14 @@ module Aspera
       # `direct` agent is equipped with refresh code
       if !use_refresh_token && !token_data.nil?
         decoded_token = self.class.decode_token(token_data[@generic_parameters[:token_field]])
-        Log.dump('decoded_token', decoded_token) unless decoded_token.nil?
+        Log.log.debug{Log.dump('decoded_token', decoded_token)} unless decoded_token.nil?
         if decoded_token.is_a?(Hash)
           expires_at_sec =
             if    decoded_token['expires_at'].is_a?(String) then DateTime.parse(decoded_token['expires_at']).to_time
             elsif decoded_token['exp'].is_a?(Integer)       then Time.at(decoded_token['exp'])
             end
           # force refresh if we see a token too close from expiration
-          use_refresh_token = true if expires_at_sec.is_a?(Time) && (expires_at_sec - Time.now) < TOKEN_EXPIRATION_GUARD_SEC
+          use_refresh_token = true if expires_at_sec.is_a?(Time) && (expires_at_sec - Time.now) < @@globals[:token_expiration_guard_sec]
           Log.log.debug{"Expiration: #{expires_at_sec} / #{use_refresh_token}"}
         end
       end
@@ -304,9 +325,9 @@ module Aspera
         token_data = JSON.parse(json_data)
         self.class.persist_mgr.put(token_id, json_data)
       end # if ! in_cache
-      raise "API error: No such field in answer: #{@generic_parameters[:token_field]}" unless token_data.key?(@generic_parameters[:token_field])
+      assert(token_data.key?(@generic_parameters[:token_field])){"API error: No such field in answer: #{@generic_parameters[:token_field]}"}
       # ok we shall have a token here
-      return 'Bearer ' + token_data[@generic_parameters[:token_field]]
+      return self.class.bearer_build(token_data[@generic_parameters[:token_field]])
     end
   end # OAuth
 end # Aspera

data/lib/aspera/open_application.rb

@@ -27,7 +27,7 @@ module Aspera
       def uri_graphical(uri)
         case Aspera::Environment.os
         when Aspera::Environment::OS_X       then return system('open', uri.to_s)
-        when Aspera::Environment::OS_WINDOWS then return system('start', 'explorer', '"' + uri.to_s + '"')
+        when Aspera::Environment::OS_WINDOWS then return system('start', 'explorer', %Q{"#{uri}"})
         when Aspera::Environment::OS_LINUX   then return system('xdg-open', uri.to_s)
         else
           raise "no graphical open method for #{Aspera::Environment.os}"
@@ -38,7 +38,7 @@ module Aspera
         if ENV.key?('EDITOR')
           system(ENV['EDITOR'], file_path.to_s)
         elsif Aspera::Environment.os.eql?(Aspera::Environment::OS_WINDOWS)
-          system('notepad.exe', '"' + file_path.to_s + '"')
+          system('notepad.exe', %Q{"#{file_path}"})
         else
           uri_graphical(file_path.to_s)
         end
@@ -59,9 +59,9 @@ module Aspera
       when :text
         case the_url.to_s
         when /^http/
-          puts "USER ACTION: please enter this url in a browser:\n" + the_url.to_s.red + "\n"
+          puts "USER ACTION: please enter this url in a browser:\n#{the_url.to_s.red}\n"
         else
-          puts "USER ACTION: open this:\n" + the_url.to_s.red + "\n"
+          puts "USER ACTION: open this:\n#{the_url.to_s.red}\n"
         end
       else
         raise StandardError, "unsupported url open method: #{@url_method}"

data/lib/aspera/persistency_action_once.rb

@@ -2,6 +2,7 @@
 
 require 'json'
 require 'aspera/log'
+require 'aspera/assert'
 
 module Aspera
   # Persist data on file system
@@ -13,21 +14,19 @@ module Aspera
     # @param :parse    Optional  parse method (default to JSON)
     # @param :format   Optional  dump method (default to JSON)
     # @param :merge    Optional  merge data from file to current data
-    def initialize(options)
-      Log.log.debug{"persistency: #{options}"}
-      raise 'options shall be Hash' unless options.is_a?(Hash)
-      raise 'mandatory :manager' if options[:manager].nil?
-      raise 'mandatory :data' if options[:data].nil?
-      raise 'mandatory :id (String)' unless options[:id].is_a?(String)
-      raise 'mandatory 1 element in :id' unless options[:id].length >= 1
-      @manager = options[:manager]
-      @persisted_object = options[:data]
-      @object_id = options[:id]
+    def initialize(manager:, data:, id:, delete: nil, parse: nil, format: nil, merge: nil)
+      assert(!manager.nil?)
+      assert(!data.nil?)
+      assert_type(id, String)
+      assert(!id.empty?)
+      @manager = manager
+      @persisted_object = data
+      @object_id = id
       # by default , at save time, file is deleted if data is nil
-      @delete_condition = options[:delete] || lambda{|d|d.empty?}
-      @persist_format = options[:format] || lambda {|h| JSON.generate(h)}
-      persist_parse = options[:parse] || lambda {|t| JSON.parse(t)}
-      persist_merge = options[:merge] || lambda {|current, file| current.concat(file).uniq rescue current}
+      @delete_condition = delete || lambda{|d|d.empty?}
+      @persist_format = format || lambda {|h| JSON.generate(h)}
+      persist_parse = parse || lambda {|t| JSON.parse(t)}
+      persist_merge = merge || lambda {|current, file| current.concat(file).uniq rescue current}
       value = @manager.get(@object_id)
       persist_merge.call(@persisted_object, persist_parse.call(value)) unless value.nil?
     end

data/lib/aspera/persistency_folder.rb

@@ -2,6 +2,7 @@
 
 require 'fileutils'
 require 'aspera/log'
+require 'aspera/assert'
 require 'aspera/environment'
 
 # search: persistency_folder PersistencyFolder
@@ -34,10 +35,10 @@ module Aspera
     end
 
     def put(object_id, value)
-      raise 'value: only String supported' unless value.is_a?(String)
+      assert_type(value, String)
       persist_filepath = id_to_filepath(object_id)
       Log.log.debug{"persistency saving: #{persist_filepath}"}
-      File.delete(persist_filepath) if File.exist?(persist_filepath)
+      FileUtils.rm_f(persist_filepath)
       File.write(persist_filepath, value)
       Environment.restrict_file_access(persist_filepath)
       @cache[object_id] = value
@@ -46,7 +47,7 @@ module Aspera
     def delete(object_id)
       persist_filepath = id_to_filepath(object_id)
       Log.log.debug{"persistency deleting: #{persist_filepath}"}
-      File.delete(persist_filepath) if File.exist?(persist_filepath)
+      FileUtils.rm_f(persist_filepath)
       @cache.delete(object_id)
     end
 
@@ -67,7 +68,7 @@ module Aspera
 
     # @param object_id String or Array
     def id_to_filepath(object_id)
-      raise 'object_id: only String supported' unless object_id.is_a?(String)
+      assert_type(object_id, String)
       FileUtils.mkdir_p(@folder)
       Environment.restrict_file_access(@folder)
       return File.join(@folder, "#{object_id}#{FILE_SUFFIX}")