aspera-cli 4.18.1 → 4.19.0

data/lib/aspera/transfer/sync.rb

@@ -4,6 +4,7 @@
 
 require 'aspera/command_line_builder'
 require 'aspera/ascp/installation'
+require 'aspera/agent/direct'
 require 'aspera/log'
 require 'aspera/assert'
 require 'json'
@@ -13,12 +14,12 @@ require 'English'
 
 module Aspera
   module Transfer
-    # builds command line arg for async
+    # builds command line arg for async and execute it
     module Sync
       # sync direction, default is push
       DIRECTIONS = %i[push pull bidi].freeze
-      # custom JSON for async instance command line options
-      PARAMS_VX_INSTANCE =
+      # JSON for async instance command line options
+      CMDLINE_PARAMS_INSTANCE =
         {
           'alt_logdir'          => { cli: { type: :opt_with_arg}, accepted_types: :string},
           'watchd'              => { cli: { type: :opt_with_arg}, accepted_types: :string},
@@ -28,7 +29,7 @@ module Aspera
         }.freeze
 
       # map sync session parameters to transfer spec: sync -> ts, true if same
-      PARAMS_VX_SESSION =
+      CMDLINE_PARAMS_SESSION =
         {
           'name'                       => { cli: { type: :opt_with_arg}, accepted_types: :string},
           'local_dir'                  => { cli: { type: :opt_with_arg}, accepted_types: :string},
@@ -65,13 +66,13 @@ module Aspera
           'license'                    => { cli: { type: :envvar, variable: 'ASPERA_SCP_LICENSE'}}
         }.freeze
 
-      CommandLineBuilder.normalize_description(PARAMS_VX_INSTANCE)
-      CommandLineBuilder.normalize_description(PARAMS_VX_SESSION)
+      CommandLineBuilder.normalize_description(CMDLINE_PARAMS_INSTANCE)
+      CommandLineBuilder.normalize_description(CMDLINE_PARAMS_SESSION)
 
-      PARAMS_VX_KEYS = %w[instance sessions].freeze
+      CMDLINE_PARAMS_KEYS = %w[instance sessions].freeze
 
       # Translation of transfer spec parameters to async v2 API (asyncs)
-      TS_TO_PARAMS_V2 = {
+      TSPEC_TO_ASYNC_CONF = {
         'remote_host'     => 'remote.host',
         'remote_user'     => 'remote.user',
         'remote_password' => 'remote.pass',
@@ -83,10 +84,9 @@ module Aspera
         'tags'            => 'tags'
       }.freeze
 
-      ASYNC_EXECUTABLE = 'async'
       ASYNC_ADMIN_EXECUTABLE = 'asyncadmin'
 
-      private_constant :PARAMS_VX_INSTANCE, :PARAMS_VX_SESSION, :PARAMS_VX_KEYS, :TS_TO_PARAMS_V2, :ASYNC_EXECUTABLE, :ASYNC_ADMIN_EXECUTABLE
+      private_constant :CMDLINE_PARAMS_INSTANCE, :CMDLINE_PARAMS_SESSION, :CMDLINE_PARAMS_KEYS, :TSPEC_TO_ASYNC_CONF, :ASYNC_ADMIN_EXECUTABLE
 
       class << self
         # Set remote_dir in sync parameters based on transfer spec
@@ -126,7 +126,7 @@ module Aspera
             remote.delete('ws_port')
             # add SSH bypass keys when authentication is token and no auth is provided
             if remote.key?('token') && !remote.key?('pass')
-              certificates_to_use.concat(Ascp::Installation.instance.aspera_token_ssh_key_paths)
+              certificates_to_use.concat(Ascp::Installation.instance.aspera_token_ssh_key_paths(:rsa))
             end
           end
           return certificates_to_use
@@ -134,23 +134,27 @@ module Aspera
 
         # @param sync_params [Hash] sync parameters, old or new format
         # @param block [nil, Proc] block to generate transfer spec, takes: direction (one of DIRECTIONS), local_dir, remote_dir
-        def start(sync_params, &block)
+        def start(
+          sync_params,
+          &block
+        )
+          Log.log.debug{Log.dump(:sync_params_initial, sync_params)}
           Aspera.assert_type(sync_params, Hash)
           env_args = {
             args: [],
             env:  {}
           }
           if sync_params.key?('local')
+            # async native JSON format (conf option)
+            Aspera.assert_type(sync_params['local'], Hash){'local'}
             remote = sync_params['remote']
-            # async native JSON format (v2)
-            Aspera.assert_type(remote, Hash)
+            Aspera.assert_type(remote, Hash){'remote'}
+            Aspera.assert_type(remote['path'], String){'remote path'}
             # get transfer spec if possible, and feed back to new structure
             if block
               transfer_spec = yield((sync_params['direction'] || 'push').to_sym, sync_params['local']['path'], remote['path'])
-              # async native JSON format
-              Aspera.assert_type(sync_params['local'], Hash)
               # translate transfer spec to async parameters
-              TS_TO_PARAMS_V2.each do |ts_param, sy_path|
+              TSPEC_TO_ASYNC_CONF.each do |ts_param, sy_path|
                 next unless transfer_spec.key?(ts_param)
                 sy_dig = sy_path.split('.')
                 param = sy_dig.pop
@@ -160,36 +164,41 @@ module Aspera
               end
               update_remote_dir(remote, 'path', transfer_spec)
             end
-            remote['connect_mode'] ||= remote.key?('ws_port') ? 'ws' : 'ssh'
+            remote['connect_mode'] ||= transfer_spec['wss_enabled'] ? 'ws' : 'ssh'
             add_certificates = remote_certificates(remote)
             if !add_certificates.empty?
               remote['private_key_paths'] ||= []
               remote['private_key_paths'].concat(add_certificates)
             end
-            Aspera.assert_type(sync_params, Hash)
+            # '--exclusive-mgmt-port=12345', '--arg-err-path=-',
             env_args[:args] = ["--conf64=#{Base64.strict_encode64(JSON.generate(sync_params))}"]
+            Log.log.debug{Log.dump(:sync_params_enriched, sync_params)}
+            agent = Agent::Direct.new
+            agent.start_and_monitor_process(session: {}, name: :async, **env_args)
           elsif sync_params.key?('sessions')
-            # ascli JSON format (v1)
+            # ascli JSON format (cmdline)
+            raise StandardError, "Only 'sessions', and optionally 'instance' keys are allowed" unless
+              sync_params.keys.push('instance').uniq.sort.eql?(CMDLINE_PARAMS_KEYS)
+            Aspera.assert_type(sync_params['sessions'], Array)
+            Aspera.assert_type(sync_params['sessions'].first, Hash)
             if block
               sync_params['sessions'].each do |session|
+                Aspera.assert_type(session['local_dir'], String){'local_dir'}
+                Aspera.assert_type(session['remote_dir'], String){'remote_dir'}
                 transfer_spec = yield((session['direction'] || 'push').to_sym, session['local_dir'], session['remote_dir'])
-                PARAMS_VX_SESSION.each do |async_param, behavior|
+                CMDLINE_PARAMS_SESSION.each do |async_param, behavior|
                   if behavior.key?(:ts)
                     tspec_param = behavior[:ts].is_a?(TrueClass) ? async_param : behavior[:ts].to_s
                     session[async_param] ||= transfer_spec[tspec_param] if transfer_spec.key?(tspec_param)
                   end
                 end
-                session['private_key_paths'] = Ascp::Installation.instance.aspera_token_ssh_key_paths if transfer_spec.key?('token')
+                session['private_key_paths'] = Ascp::Installation.instance.aspera_token_ssh_key_paths(:rsa) if transfer_spec.key?('token')
                 update_remote_dir(session, 'remote_dir', transfer_spec)
               end
             end
-            raise StandardError, "Only 'sessions', and optionally 'instance' keys are allowed" unless
-              sync_params.keys.push('instance').uniq.sort.eql?(PARAMS_VX_KEYS)
-            Aspera.assert_type(sync_params['sessions'], Array)
-            Aspera.assert_type(sync_params['sessions'].first, Hash)
             if sync_params.key?('instance')
               Aspera.assert_type(sync_params['instance'], Hash)
-              instance_builder = CommandLineBuilder.new(sync_params['instance'], PARAMS_VX_INSTANCE)
+              instance_builder = CommandLineBuilder.new(sync_params['instance'], CMDLINE_PARAMS_INSTANCE)
               instance_builder.process_params
               instance_builder.add_env_args(env_args)
             end
@@ -197,23 +206,19 @@ module Aspera
             sync_params['sessions'].each do |session_params|
               Aspera.assert_type(session_params, Hash)
               Aspera.assert(session_params.key?('name')){'session must contain at least name'}
-              session_builder = CommandLineBuilder.new(session_params, PARAMS_VX_SESSION)
+              session_builder = CommandLineBuilder.new(session_params, CMDLINE_PARAMS_SESSION)
               session_builder.process_params
               session_builder.add_env_args(env_args)
             end
+            async_exec = Ascp::Installation.instance.path(:async)
+            Process.wait(Environment.secure_spawn(env: env_args[:env], exec: async_exec, args: env_args[:args]))
+            if $CHILD_STATUS.exitstatus != 0
+              raise "Sync failed with exit: #{$CHILD_STATUS.exitstatus}"
+            end
           else
             raise 'At least one of `local` or `sessions` must be present in async parameters'
           end
-          Log.log.debug{Log.dump(:sync_params, sync_params)}
-          Log.log.debug{"execute: #{env_args[:env].map{|k, v| "#{k}=\"#{v}\""}.join(' ')} \"#{ASYNC_EXECUTABLE}\" \"#{env_args[:args].join('" "')}\""}
-          res = system(env_args[:env], [ASYNC_EXECUTABLE, ASYNC_EXECUTABLE], *env_args[:args])
-          Log.log.debug{"result=#{res}"}
-          case res
-          when true then return nil
-          when false then raise "failed: #{$CHILD_STATUS}"
-          when nil then raise "not started: #{$CHILD_STATUS}"
-          else Aspera.error_unexpected_value(res)
-          end
+          return nil
         end
 
         def parse_status(stdout)
@@ -234,15 +239,15 @@ module Aspera
         end
 
         def admin_status(sync_params, session_name)
-          command_line = [ASYNC_ADMIN_EXECUTABLE, '--quiet']
+          arguments = ['--quiet']
           if sync_params.key?('local')
             Aspera.assert(!sync_params['name'].nil?){'Missing session name'}
             Aspera.assert(session_name.nil? || session_name.eql?(sync_params['name'])){'Session not found'}
-            command_line.push("--name=#{sync_params['name']}")
+            arguments.push("--name=#{sync_params['name']}")
             if sync_params.key?('local_db_dir')
-              command_line.push("--local-db-dir=#{sync_params['local_db_dir']}")
+              arguments.push("--local-db-dir=#{sync_params['local_db_dir']}")
             elsif sync_params.dig('local', 'path')
-              command_line.push("--local-dir=#{sync_params.dig('local', 'path')}")
+              arguments.push("--local-dir=#{sync_params.dig('local', 'path')}")
             else
               raise 'Missing either local_db_dir or local.path'
             end
@@ -250,19 +255,19 @@ module Aspera
             session = session_name.nil? ? sync_params['sessions'].first : sync_params['sessions'].find{|s|s['name'].eql?(session_name)}
             raise "Session #{session_name} not found in #{sync_params['sessions'].map{|s|s['name']}.join(',')}" if session.nil?
             raise 'Missing session name' if session['name'].nil?
-            command_line.push("--name=#{session['name']}")
+            arguments.push("--name=#{session['name']}")
             if session.key?('local_db_dir')
-              command_line.push("--local-db-dir=#{session['local_db_dir']}")
+              arguments.push("--local-db-dir=#{session['local_db_dir']}")
             elsif session.key?('local_dir')
-              command_line.push("--local-dir=#{session['local_dir']}")
+              arguments.push("--local-dir=#{session['local_dir']}")
             else
               raise 'Missing either local_db_dir or local_dir'
             end
           else
             raise 'At least one of `local` or `sessions` must be present in async parameters'
           end
-          Log.log.debug{"execute: #{command_line.join(' ')}"}
-          stdout, stderr, status = Open3.capture3(*command_line)
+          Environment.secure_spawn(env: {}, exec: ASYNC_ADMIN_EXECUTABLE, args: arguments, log_only: true)
+          stdout, stderr, status = Open3.capture3(*[ASYNC_ADMIN_EXECUTABLE].concat(arguments))
           Log.log.debug{"status=#{status}, stderr=#{stderr}"}
           Log.log.trace1{"stdout=#{stdout}"}
           raise "Sync failed: #{status.exitstatus} : #{stderr}" unless status.success?

data/lib/aspera/web_auth.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require 'aspera/web_server_simple'
-require 'stringio'
 
 module Aspera
   # servlet called on callback: it records the callback request

data/lib/aspera/web_server_simple.rb

@@ -4,12 +4,13 @@ require 'webrick'
 require 'webrick/https'
 require 'aspera/log'
 require 'aspera/assert'
+require 'aspera/hash_ext'
 require 'openssl'
 
 module Aspera
   # Simple WEBrick server with HTTPS support
   class WebServerSimple < WEBrick::HTTPServer
-    CERT_PARAMETERS = %i[key cert chain].freeze
+    CERT_PARAMETERS = %i[key cert chain pkcs12].freeze
     GENERIC_ISSUER = '/C=FR/O=Test/OU=Test/CN=Test'
     ONE_YEAR_SECONDS = 365 * 24 * 60 * 60
 
@@ -58,19 +59,29 @@ module Aspera
           Aspera.assert_type(certificate, Hash)
           certificate = certificate.symbolize_keys
           raise "unexpected key in certificate config: only: #{CERT_PARAMETERS.join(', ')}" if certificate.keys.any?{|key|!CERT_PARAMETERS.include?(key)}
-          webrick_options[:SSLPrivateKey] = if certificate.key?(:key)
-            OpenSSL::PKey::RSA.new(File.read(certificate[:key]))
+          if certificate.key?(:pkcs12)
+            Log.log.debug('Using PKCS12 certificate')
+            raise 'pkcs12 requires a password' unless certificate.key?(:key)
+            pkcs12 = OpenSSL::PKCS12.new(File.read(certificate[:pkcs12]), certificate[:key])
+            webrick_options[:SSLCertificate] = pkcs12.certificate
+            webrick_options[:SSLPrivateKey] = pkcs12.key
+            webrick_options[:SSLExtraChainCert] = pkcs12.ca_certs
           else
-            OpenSSL::PKey::RSA.new(4096)
-          end
-          if certificate.key?(:cert)
-            webrick_options[:SSLCertificate] = OpenSSL::X509::Certificate.new(File.read(certificate[:cert]))
-          else
-            webrick_options[:SSLCertificate] = OpenSSL::X509::Certificate.new
-            self.class.fill_self_signed_cert(webrick_options[:SSLCertificate], webrick_options[:SSLPrivateKey])
-          end
-          if certificate.key?(:chain)
-            webrick_options[:SSLExtraChainCert] = [OpenSSL::X509::Certificate.new(File.read(certificate[:chain]))]
+            Log.log.debug('Using PEM certificate')
+            webrick_options[:SSLPrivateKey] = if certificate.key?(:key)
+              OpenSSL::PKey::RSA.new(File.read(certificate[:key]))
+            else
+              OpenSSL::PKey::RSA.new(4096)
+            end
+            if certificate.key?(:cert)
+              webrick_options[:SSLCertificate] = OpenSSL::X509::Certificate.new(File.read(certificate[:cert]))
+            else
+              webrick_options[:SSLCertificate] = OpenSSL::X509::Certificate.new
+              self.class.fill_self_signed_cert(webrick_options[:SSLCertificate], webrick_options[:SSLPrivateKey])
+            end
+            if certificate.key?(:chain)
+              webrick_options[:SSLExtraChainCert] = [OpenSSL::X509::Certificate.new(File.read(certificate[:chain]))]
+            end
           end
         end
       end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: aspera-cli
 version: !ruby/object:Gem::Version
-  version: 4.18.1
+  version: 4.19.0
 platform: ruby
 authors:
 - Laurent Martin
@@ -37,7 +37,7 @@ cert_chain:
   eTf9kxhVM40wGQOECVNA8UsEEZHD48eF+csUYZtAJOF5oxTI8UyV9T/o6CgO0c9/
   Gzz+Qm5ULOUcPiJLjSpaiTrkiIVYiDGnqNSr6R1Hb1c=
   -----END CERTIFICATE-----
-date: 2024-08-21 00:00:00.000000000 Z
+date: 2024-10-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: blankslate
@@ -390,10 +390,10 @@ files:
 - README.md
 - bin/ascli
 - bin/asession
+- examples/build_exec
 - examples/build_package.sh
 - examples/dascli
 - examples/proxy.pac
-- examples/rubyc
 - lib/aspera/agent/alpha.rb
 - lib/aspera/agent/base.rb
 - lib/aspera/agent/connect.rb

data/examples/rubyc

@@ -1,24 +0,0 @@
-#!/bin/bash
-# https://github.com/you54f/ruby-packer
-# https://github.com/YOU54F/ruby-packer/releases
-set -e
-FOLDER="$(dirname $0)/../tmp"
-RUBYC="$FOLDER/rubyc"
-if test ! -e "$RUBYC"; then
-    mkdir -p "$FOLDER"
-    case $(uname -sm|tr ' ' -) in
-        Darwin-arm64)
-            curl -L https://github.com/YOU54F/ruby-packer/releases/download/rel-20230812/rubyc-Darwin-arm64.tar.gz | tar -xz -C "$FOLDER"
-            mv "$FOLDER/rubyc-Darwin-arm64" "$RUBYC"
-            ;;
-        Linux-x86_64)
-            curl -L https://github.com/YOU54F/ruby-packer/releases/download/rel-20230812/rubyc-Linux-x86_64.tar.gz | tar -xz -C "$FOLDER"
-            mv "$FOLDER/rubyc-Linux-x86_64" "$RUBYC"
-            ;;
-        *)
-            echo "This architecture is not supported." >&2
-            exit 1
-            ;;
-    esac
-fi
-exec "$RUBYC" "$@"