aspera-cli 4.10.0 → 4.11.0

data/lib/aspera/sync.rb

@@ -1,90 +1,204 @@
 # frozen_string_literal: true
 
 require 'aspera/command_line_builder'
+require 'aspera/fasp/installation'
+require 'json'
+require 'base64'
 
 module Aspera
   # builds command line arg for async
   class Sync
-    INSTANCE_PARAMS =
+    PARAMS_VX_INSTANCE =
       {
         'alt_logdir'          => { cltype: :opt_with_arg, accepted_types: :string},
         'watchd'              => { cltype: :opt_with_arg, accepted_types: :string},
         'apply_local_docroot' => { cltype: :opt_without_arg},
-        'quiet'               => { cltype: :opt_without_arg}
+        'quiet'               => { cltype: :opt_without_arg},
+        'ws_connect'          => { cltype: :opt_without_arg}
       }.freeze
-    SESSION_PARAMS =
+
+    # map sync session parameters to transfer spec: sync -> ts, true if same
+    PARAMS_VX_SESSION =
       {
         'name'                       => { cltype: :opt_with_arg, accepted_types: :string},
         'local_dir'                  => { cltype: :opt_with_arg, accepted_types: :string},
         'remote_dir'                 => { cltype: :opt_with_arg, accepted_types: :string},
         'local_db_dir'               => { cltype: :opt_with_arg, accepted_types: :string},
         'remote_db_dir'              => { cltype: :opt_with_arg, accepted_types: :string},
-        'host'                       => { cltype: :opt_with_arg, accepted_types: :string},
-        'user'                       => { cltype: :opt_with_arg, accepted_types: :string},
-        'private_key_path'           => { cltype: :opt_with_arg, accepted_types: :string},
+        'host'                       => { cltype: :opt_with_arg, accepted_types: :string, ts: :remote_host},
+        'user'                       => { cltype: :opt_with_arg, accepted_types: :string, ts: :remote_user},
+        'private_key_paths'          => { cltype: :opt_with_arg, accepted_types: :array, clswitch: '--private-key-path'},
         'direction'                  => { cltype: :opt_with_arg, accepted_types: :string},
         'checksum'                   => { cltype: :opt_with_arg, accepted_types: :string},
-        'tcp_port'                   => { cltype: :opt_with_arg, accepted_types: :int},
+        'tags'                       => { cltype: :opt_with_arg, accepted_types: :hash, ts: true,
+                                          clswitch: '--tags64', clconvert: 'Aspera::Fasp::Parameters.clconv_json64'},
+        'tcp_port'                   => { cltype: :opt_with_arg, accepted_types: :int, ts: :ssh_port},
         'rate_policy'                => { cltype: :opt_with_arg, accepted_types: :string},
         'target_rate'                => { cltype: :opt_with_arg, accepted_types: :string},
         'cooloff'                    => { cltype: :opt_with_arg, accepted_types: :int},
         'pending_max'                => { cltype: :opt_with_arg, accepted_types: :int},
         'scan_intensity'             => { cltype: :opt_with_arg, accepted_types: :string},
-        'cipher'                     => { cltype: :opt_with_arg, accepted_types: :string},
+        'cipher'                     => { cltype: :opt_with_arg, accepted_types: :string, ts: true},
         'transfer_threads'           => { cltype: :opt_with_arg, accepted_types: :int},
-        'preserve_time'              => { cltype: :opt_without_arg},
-        'preserve_access_time'       => { cltype: :opt_without_arg},
-        'preserve_modification_time' => { cltype: :opt_without_arg},
-        'preserve_uid'               => { cltype: :opt_without_arg},
-        'preserve_gid'               => { cltype: :opt_without_arg},
-        'create_dir'                 => { cltype: :opt_without_arg},
+        'preserve_time'              => { cltype: :opt_without_arg, ts: :preserve_times},
+        'preserve_access_time'       => { cltype: :opt_without_arg, ts: nil},
+        'preserve_modification_time' => { cltype: :opt_without_arg, ts: nil},
+        'preserve_uid'               => { cltype: :opt_without_arg, ts: :preserve_file_owner_uid},
+        'preserve_gid'               => { cltype: :opt_without_arg, ts: :preserve_file_owner_gid},
+        'create_dir'                 => { cltype: :opt_without_arg, ts: true},
         'reset'                      => { cltype: :opt_without_arg},
-        # note: only one env var, but multiple sessions... may be a problem
-        'remote_password'            => { cltype: :envvar, clvarname: 'ASPERA_SCP_PASS'},
-        'cookie'                     => { cltype: :envvar, clvarname: 'ASPERA_SCP_COOKIE'},
-        'token'                      => { cltype: :envvar, clvarname: 'ASPERA_SCP_TOKEN'},
+        # NOTE: only one env var, but multiple sessions... could be a problem
+        'remote_password'            => { cltype: :envvar, clvarname: 'ASPERA_SCP_PASS', ts: true},
+        'cookie'                     => { cltype: :envvar, clvarname: 'ASPERA_SCP_COOKIE', ts: true},
+        'token'                      => { cltype: :envvar, clvarname: 'ASPERA_SCP_TOKEN', ts: true},
         'license'                    => { cltype: :envvar, clvarname: 'ASPERA_SCP_LICENSE'}
       }.freeze
 
-    Aspera::CommandLineBuilder.normalize_description(INSTANCE_PARAMS)
-    Aspera::CommandLineBuilder.normalize_description(SESSION_PARAMS)
+    Aspera::CommandLineBuilder.normalize_description(PARAMS_VX_INSTANCE)
+    Aspera::CommandLineBuilder.normalize_description(PARAMS_VX_SESSION)
 
-    private_constant :INSTANCE_PARAMS,:SESSION_PARAMS
+    PARAMS_VX_KEYS = %w[instance sessions].freeze
 
-    def initialize(sync_params)
-      @sync_params = sync_params
-    end
+    TS_TO_PARAMS = {
+      'remote_host'     => 'remote.host',
+      'remote_user'     => 'remote.user',
+      'remote_password' => 'remote.pass',
+      'sshfp'           => 'remote.fingerprint',
+      'ssh_port'        => 'remote.port',
+      'wss_port'        => 'remote.ws_port',
+      'proxy'           => 'remote.proxy',
+      'token'           => 'remote.token',
+      'tags'            => 'tags'
+    }.freeze
+
+    ASYNC_EXECUTABLE = 'async'
 
-    MANDATORY_KEYS = %w[instance sessions].freeze
+    private_constant :PARAMS_VX_INSTANCE, :PARAMS_VX_SESSION, :PARAMS_VX_KEYS, :ASYNC_EXECUTABLE
 
-    def compute_args
-      raise StandardError,'parameter must be Hash' unless @sync_params.is_a?(Hash)
-      raise StandardError,"parameter hash must have at least 'sessions', and optionally 'instance' keys." unless
-        @sync_params.keys.push('instance').uniq.sort.eql?(MANDATORY_KEYS)
-      raise StandardError,'sessions key must be Array' unless @sync_params['sessions'].is_a?(Array)
-      raise StandardError,'sessions key must has at least one element (hash)' unless @sync_params['sessions'].first.is_a?(Hash)
+    class << self
+      def update_parameters_with_transfer_spec(sync_params, transfer_spec)
+        if sync_params.key?('local')
+          # async native JSON format
+          raise StandardError, 'local must be Hash' unless sync_params['local'].is_a?(Hash)
+          TS_TO_PARAMS.each do |ts_param, sy_path|
+            next unless transfer_spec.key?(ts_param)
+            sy_dig = sy_path.split('.')
+            param = sy_dig.pop
+            hash = sy_dig.empty? ? sync_params : sync_params[sy_dig.first]
+            hash = sync_params[sy_dig.first] = {} if hash.nil?
+            hash[param] = transfer_spec[ts_param]
+          end
+          # 'remote.path',
+          sync_params['remote']['connect_mode'] ||= sync_params['remote'].key?('ws_port') ? 'ws' : 'ssh'
+          sync_params['remote']['private_key_paths'] ||= Fasp::Installation.instance.bypass_keys if transfer_spec.key?('token')
+          sync_params['remote']['path'] ||= '/' if transfer_spec.dig(*%w[tags aspera node file_id])
+        elsif sync_params.key?('sessions')
+          sync_params['sessions'].each do |session|
+            PARAMS_VX_SESSION.each do |async_param, behaviour|
+              if behaviour.key?(:ts)
+                tspec_param = behaviour[:ts].is_a?(TrueClass) ? async_param : behaviour[:ts].to_s
+                session[async_param] ||= transfer_spec[tspec_param] if transfer_spec.key?(tspec_param)
+              end
+            end
+            session['private_key_paths'] = Fasp::Installation.instance.bypass_keys if transfer_spec.key?('token')
+            session['remote_dir'] = '/' if transfer_spec.dig(*%w[tags aspera node file_id])
+          end
+        else
+          raise 'At least one of `local` or `sessions` must be present in async parameters'
+        end
+        Log.dump(:sync, sync_params)
+      end
+    end
 
-      env_args = {
+    attr_reader :env_args
+
+    def initialize(sync_params)
+      raise StandardError, 'parameter must be Hash' unless sync_params.is_a?(Hash)
+      @env_args = {
         args: [],
         env:  {}
       }
+      if sync_params.key?('local')
+        # async native JSON format
+        raise StandardError, 'remote must be Hash' unless sync_params['remote'].is_a?(Hash)
+        @env_args[:args] = "--conf64=#{Base64.strict_encode64(JSON.generate(sync_params))}"
+      elsif sync_params.key?('sessions')
+        # ascli JSON format
+        raise StandardError, "Only 'sessions', and optionally 'instance' keys are allowed" unless
+          sync_params.keys.push('instance').uniq.sort.eql?(PARAMS_VX_KEYS)
+        raise StandardError, 'sessions key must be Array' unless sync_params['sessions'].is_a?(Array)
+        raise StandardError, 'sessions key requires at least one Hash' unless sync_params['sessions'].first.is_a?(Hash)
 
-      if @sync_params.has_key?('instance')
-        raise StandardError,'instance key must be hash' unless @sync_params['instance'].is_a?(Hash)
-        instance_builder = CommandLineBuilder.new(@sync_params['instance'],INSTANCE_PARAMS)
-        instance_builder.process_params
-        instance_builder.add_env_args(env_args[:env],env_args[:args])
+        if sync_params.key?('instance')
+          raise StandardError, 'instance key must be Hash' unless sync_params['instance'].is_a?(Hash)
+          instance_builder = Aspera::CommandLineBuilder.new(sync_params['instance'], PARAMS_VX_INSTANCE)
+          instance_builder.process_params
+          instance_builder.add_env_args(@env_args[:env], @env_args[:args])
+        end
+
+        sync_params['sessions'].each do |session_params|
+          raise StandardError, 'sessions must contain hashes' unless session_params.is_a?(Hash)
+          raise StandardError, 'session must contain at leat name' unless session_params.key?('name')
+          session_builder = Aspera::CommandLineBuilder.new(session_params, PARAMS_VX_SESSION)
+          session_builder.process_params
+          session_builder.add_env_args(@env_args[:env], @env_args[:args])
+        end
+      else
+        raise 'At least one of `local` or `sessions` must be present in async parameters'
       end
+    end
 
-      @sync_params['sessions'].each do |session_params|
-        raise StandardError,'sessions must contain hashes' unless session_params.is_a?(Hash)
-        raise StandardError,'session must contain at leat name' unless session_params.has_key?('name')
-        session_builder = CommandLineBuilder.new(session_params,SESSION_PARAMS)
-        session_builder.process_params
-        session_builder.add_env_args(env_args[:env],env_args[:args])
+    def start
+      Log.log.debug{"execute: #{@env_args[:env].map{|k, v| "#{k}=\"#{v}\""}.join(' ')} \"#{ASYNC_EXECUTABLE}\" \"#{@env_args[:args].join('" "')}\""}
+      res = system(@env_args[:env], [ASYNC_EXECUTABLE, ASYNC_EXECUTABLE], *@env_args[:args])
+      Log.log.debug{"result=#{res}"}
+      case res
+      when true then return nil
+      when false then raise "failed: #{$CHILD_STATUS}"
+      when nil then raise "not started: #{$CHILD_STATUS}"
+      else raise 'internal error: unspecified case'
       end
+    end
+  end
+
+  class SyncAdmin
+    ASYNC_ADMIN_EXECUTABLE = 'asyncadmin'
+    private_constant :ASYNC_ADMIN_EXECUTABLE
+    def initialize(sync_params, session_name)
+      @cmdline = [ASYNC_ADMIN_EXECUTABLE, '--quiet']
+      if sync_params.key?('local')
+        raise 'Missing session name' if sync_params['name'].nil?
+        raise 'Session not found' unless session_name.nil? || session_name.eql?(sync_params['name'])
+        @cmdline.push("--name=#{sync_params['name']}")
+        if sync_params.key?('local_db_dir')
+          @cmdline.push("--local-db-dir=#{sync_params['local_db_dir']}")
+        elsif sync_params.dig('local', 'path')
+          @cmdline.push("--local-dir=#{sync_params.dig('local', 'path')}")
+        else
+          raise 'Missing either local_db_dir or local.path'
+        end
+      elsif sync_params.key?('sessions')
+        session = session_name.nil? ? sync_params['sessions'].first : sync_params['sessions'].find{|s|s['name'].eql?(session_name)}
+        raise 'Session not found' if session.nil?
+        raise 'Missing session name' if session['name'].nil?
+        @cmdline.push("--name=#{session['name']}")
+        if session.key?('local_db_dir')
+          @cmdline.push("--local-db-dir=#{session['local_db_dir']}")
+        elsif session.key?('local_dir')
+          @cmdline.push("--local-dir=#{session['local_dir']}")
+        else
+          raise 'Missing either local_db_dir or local_dir'
+        end
+      else
+        raise 'At least one of `local` or `sessions` must be present in async parameters'
+      end
+    end
 
-      return env_args
+    def status
+      stdout, stderr, status = Open3.capture3(*@cmdline)
+      Log.log.debug{"status=#{status}, stderr=#{stderr}"}
+      raise "Sync failed: #{status.exitstatus} : #{stderr}" unless status.success?
+      return stdout.split("\n").each_with_object({}){|l, m|i = l.split(/:  */); m[i.first.lstrip] = i.last.lstrip} # rubocop:disable Style/Semicolon
     end
   end
 end

data/lib/aspera/temp_file_manager.rb

@@ -11,7 +11,7 @@ module Aspera
     SEC_IN_DAY = 86_400
     # assume no transfer last longer than this
     # (garbage collect file list which were not deleted after transfer)
-    FILE_LIST_AGE_MAX_SEC = 5 * SEC_IN_DAY
+    FILE_LIST_AGE_MAX_SEC = SEC_IN_DAY * 5
     private_constant :SEC_IN_DAY, :FILE_LIST_AGE_MAX_SEC
     include Singleton
     def initialize
@@ -53,7 +53,7 @@ module Aspera
         age_sec = (Time.now - File.stat(file_path).mtime).to_i
         # check age of file, delete too old
         if File.file?(file_path) && (age_sec > FILE_LIST_AGE_MAX_SEC)
-          Log.log.debug("garbage collecting #{name}")
+          Log.log.debug{"garbage collecting #{name}"}
           File.delete(file_path)
         end
       end

data/lib/aspera/uri_reader.rb

@@ -10,12 +10,12 @@ module Aspera
       def read(uri_to_read)
         proxy_uri = URI.parse(uri_to_read)
         case proxy_uri.scheme
-        when 'http','https'
-          return Rest.new(base_url: uri_to_read,redirect_max: 5).call(operation: 'GET', subpath: '', headers: {'Accept' => 'text/plain'})[:data]
-        when 'file',NilClass
+        when 'http', 'https'
+          return Rest.new(base_url: uri_to_read, redirect_max: 5).call(operation: 'GET', subpath: '', headers: {'Accept' => 'text/plain'})[:data]
+        when 'file', NilClass
           local_file_path = proxy_uri.path
           raise 'URL shall have a path, check syntax' if local_file_path.nil?
-          local_file_path = File.expand_path(local_file_path.gsub(/^\//,'')) if /^\/(~|.|..)\//.match?(local_file_path)
+          local_file_path = File.expand_path(local_file_path.gsub(%r{^/}, '')) if %r{^/(~|.|..)/}.match?(local_file_path)
           return File.read(local_file_path)
         else
           raise "unknown scheme: [#{proxy_uri.scheme}] for [#{uri_to_read}]"

data/lib/aspera/web_auth.rb

@@ -7,16 +7,16 @@ require 'stringio'
 module Aspera
   # servlet called on callback: it records the callback request
   class WebAuthServlet < WEBrick::HTTPServlet::AbstractServlet
-    def initialize(server,application) # additional args get here
+    def initialize(server, application) # additional args get here
       Log.log.debug('WebAuthServlet initialize')
       super(server)
       @app = application
     end
 
     def service(request, response)
-      Log.log.debug("received request from browser #{request.request_method} #{request.path}")
-      raise WEBrick::HTTPStatus::MethodNotAllowed,"unexpected method: #{request.request_method}" unless request.request_method.eql?('GET')
-      raise WEBrick::HTTPStatus::NotFound,"unexpected path: #{request.path}" unless request.path.eql?(@app.expected_path)
+      Log.log.debug{"received request from browser #{request.request_method} #{request.path}"}
+      raise WEBrick::HTTPStatus::MethodNotAllowed, "unexpected method: #{request.request_method}" unless request.request_method.eql?('GET')
+      raise WEBrick::HTTPStatus::NotFound, "unexpected path: #{request.path}" unless request.path.eql?(@app.expected_path)
       # acquire lock and signal change
       @app.mutex.synchronize do
         @app.query = request.query
@@ -30,7 +30,7 @@ module Aspera
   end # WebAuthServlet
 
   # generates and adds self signed cert to provided webrick options
-  #def fill_self_signed_cert(cert,key)
+  # def fill_self_signed_cert(cert,key)
   #  cert.subject = cert.issuer = OpenSSL::X509::Name.parse('/C=FR/O=Test/OU=Test/CN=Test')
   #  cert.not_before = Time.now
   #  cert.not_after = Time.now + 365 * 24 * 60 * 60
@@ -47,12 +47,13 @@ module Aspera
   #  ]
   #  cert.add_extension(ef.create_extension('authorityKeyIdentifier','keyid:always,issuer:always'))
   #  cert.sign(key, OpenSSL::Digest::SHA256.new)
-  #end
+  # end
 
   # start a local web server, then start a browser that will callback the local server upon authentication
   class WebAuth
-    attr_reader :expected_path,:mutex,:cond
+    attr_reader :expected_path, :mutex, :cond
     attr_writer :query
+
     # @param endpoint_url [String] e.g. 'https://127.0.0.1:12345'
     def initialize(endpoint_url)
       uri = URI.parse(endpoint_url)
@@ -74,14 +75,14 @@ module Aspera
       when 'https'
         webrick_options[:SSLEnable] = true
         # a- automatic certificate generation
-        webrick_options[:SSLCertName] = [['CN',WEBrick::Utils.getservername]]
+        webrick_options[:SSLCertName] = [['CN', WEBrick::Utils.getservername]]
         # b- generate self signed cert
-        #webrick_options[:SSLPrivateKey]   = OpenSSL::PKey::RSA.new(4096)
-        #webrick_options[:SSLCertificate]  = OpenSSL::X509::Certificate.new
-        #self.class.fill_self_signed_cert(webrick_options[:SSLCertificate],webrick_options[:SSLPrivateKey])
+        # webrick_options[:SSLPrivateKey]   = OpenSSL::PKey::RSA.new(4096)
+        # webrick_options[:SSLCertificate]  = OpenSSL::X509::Certificate.new
+        # self.class.fill_self_signed_cert(webrick_options[:SSLCertificate],webrick_options[:SSLPrivateKey])
         ## c- good cert
-        #webrick_options[:SSLPrivateKey]  = OpenSSL::PKey::RSA.new(File.read('.../myserver.key'))
-        #webrick_options[:SSLCertificate] = OpenSSL::X509::Certificate.new(File.read('.../myserver.crt'))
+        # webrick_options[:SSLPrivateKey]  = OpenSSL::PKey::RSA.new(File.read('.../myserver.key'))
+        # webrick_options[:SSLCertificate] = OpenSSL::X509::Certificate.new(File.read('.../myserver.crt'))
       end
       # self signed certificate generates characters on STDERR, see create_self_signed_cert in webrick/ssl.rb
       Log.capture_stderr { @server = WEBrick::HTTPServer.new(webrick_options) }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: aspera-cli
 version: !ruby/object:Gem::Version
-  version: 4.10.0
+  version: 4.11.0
 platform: ruby
 authors:
 - Laurent Martin
@@ -35,7 +35,7 @@ cert_chain:
   ZjkOWbUc1aLIsfaQFHWyNfisY9X2RgkFHjX0p5493wnoA7aWh52MUhc145npFh8z
   v4P9xwkT02Shkert4B4iwNvVjoAUGk+J4090svZCroAyXBjon5LV7MJ4fyw=
   -----END CERTIFICATE-----
-date: 2022-12-01 00:00:00.000000000 Z
+date: 2023-01-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: execjs
@@ -89,14 +89,14 @@ dependencies:
   name: net-ssh
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '6.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '6.0'
 - !ruby/object:Gem::Dependency
@@ -360,6 +360,9 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- BUGS.md
+- CHANGELOG.md
+- CONTRIBUTING.md
 - README.md
 - bin/ascli
 - bin/asession
@@ -485,7 +488,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements:
 - Read the manual for any requirement
-rubygems_version: 3.1.6
+rubygems_version: 3.4.1
 signing_key: 
 specification_version: 4
 summary: 'Execute actions using command line on IBM Aspera Server products: Aspera