aspera-cli 4.10.0 → 4.11.0

data/lib/aspera/cli/plugins/server.rb

@@ -1,98 +1,150 @@
 # frozen_string_literal: true
 
 require 'aspera/cli/basic_auth_plugin'
+require 'aspera/cli/plugins/sync'
 require 'aspera/ascmd'
 require 'aspera/fasp/transfer_spec'
 require 'aspera/ssh'
 require 'aspera/nagios'
 require 'tempfile'
+require 'open3'
 
 module Aspera
   module Cli
     module Plugins
       # implement basic remote access with FASP/SSH
-      class Server < BasicAuthPlugin
+      class Server < Aspera::Cli::BasicAuthPlugin
+        SSH_SCHEME = 'ssh'
+        URI_SCHEMES = %w[https local].push(SSH_SCHEME).freeze
+        ASCMD_ALIASES = {
+          browse: :ls,
+          delete: :rm,
+          rename: :mv
+        }.freeze
+        TRANSFER_COMMANDS = %i[sync upload download].freeze
+
+        private_constant :SSH_SCHEME, :URI_SCHEMES, :ASCMD_ALIASES, :TRANSFER_COMMANDS
+
         class LocalExecutor
-          def execute(cmd,_input=nil)
-            Log.log.debug("Executing: #{cmd}")
-            %x(#{cmd.join(' ')})
+          def execute(cmd, line)
+            # concatenate arguments, enclose in double quotes
+            cmd = cmd.map{|v|%Q("#{v}")}.join(' ') if cmd.is_a?(Array)
+            Log.log.debug{"Executing: #{cmd} with '#{line}'"}
+            stdout_str, stderr_str, status = Open3.capture3(cmd, stdin_data: line, binmode: true)
+            Log.log.debug(">> #{status} -> #{stderr_str}")
+            raise "command #{cmd} failed with code #{status.exitstatus} #{stderr_str}" unless status.success?
+            return stdout_str
           end
         end
 
         def initialize(env)
           super(env)
-          options.add_opt_simple(:ssh_keys,'SSH key path list (Array or single)')
-          options.add_opt_simple(:ssh_options,'SSH options (Hash)')
-          options.set_option(:ssh_keys,[])
-          options.set_option(:ssh_options,{})
+          options.add_opt_simple(:ssh_keys, 'SSH key path list (Array or single)')
+          options.add_opt_simple(:ssh_options, 'SSH options (Hash)')
           options.parse_options!
+          @ssh_opts = nil
         end
 
-        def key_symb_to_str_list(source)
-          return source.map(&:stringify_keys)
-        end
-
-        ACTIONS = %i[health download upload browse delete rename].concat(Aspera::AsCmd::OPERATIONS).freeze
-
-        def execute_action
-          server_uri = URI.parse(options.get_option(:url,is_type: :mandatory))
-          Log.log.debug("URI : #{server_uri}, port=#{server_uri.port}, scheme:#{server_uri.scheme}")
-          server_transfer_spec = {'remote_host' => server_uri.hostname}
-          shell_executor = nil
-          case server_uri.scheme
-          when 'local'
-            shell_executor = LocalExecutor.new
-          when 'https'
-            raise 'ERROR: transfer spec with token required' unless transfer.option_transfer_spec['token'].is_a?(String)
+        # Read command line options
+        # @return [Hash] transfer specification
+        def options_to_base_transfer_spec
+          url = options.get_option(:url, is_type: :mandatory)
+          server_transfer_spec = {}
+          server_uri = URI.parse(url)
+          Log.log.debug{"URI : #{server_uri}, port=#{server_uri.port}, scheme:#{server_uri.scheme}"}
+          server_transfer_spec['remote_host'] = server_uri.hostname
+          unless URI_SCHEMES.include?(server_uri.scheme)
+            Log.log.warn{"Scheme [#{server_uri.scheme}] not supported in #{url}, use one of: #{URI_SCHEMES.join(', ')}. Defaulting to #{SSH_SCHEME}."}
+            server_uri.scheme = SSH_SCHEME
+          end
+          if server_uri.scheme.eql?('local')
+            # Using local execution (mostly for testing)
+            return server_transfer_spec
+          elsif transfer.option_transfer_spec['token'].is_a?(String) && server_uri.scheme.eql?('https')
             server_transfer_spec['wss_enabled'] = true
             server_transfer_spec['wss_port'] = server_uri.port
-          else # when 'ssh'
-            Log.log.error("Scheme #{server_uri.scheme} not supported. Assuming SSH.") if !server_uri.scheme.eql?('ssh')
-            if options.get_option(:username).nil?
-              options.set_option(:username,Aspera::Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER)
-              Log.log.info("Using default transfer user: #{Aspera::Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER}")
-            end
-            server_transfer_spec['remote_user'] = options.get_option(:username,is_type: :mandatory)
-            ssh_options = options.get_option(:ssh_options)
-            raise 'expecting a Hash for ssh_options' unless ssh_options.is_a?(Hash)
-            ssh_options = ssh_options.symbolize_keys
-            if !server_uri.port.nil?
-              ssh_options[:port] = server_uri.port
-              server_transfer_spec['ssh_port'] = server_uri.port
-            end
-            cred_set = false
-            password = options.get_option(:password)
-            if !password.nil?
-              ssh_options[:password] = password
-              server_transfer_spec['remote_password'] = password
-              cred_set = true
-            end
-            ssh_keys = options.get_option(:ssh_keys)
-            if !ssh_keys.nil?
-              raise 'expecting single value or array for ssh_keys' unless ssh_keys.is_a?(Array) || ssh_keys.is_a?(String)
-              ssh_keys = [ssh_keys] if ssh_keys.is_a?(String)
-              ssh_keys.map!{|p|File.expand_path(p)}
-              Log.log.debug("ssh keys=#{ssh_keys}")
-              if !ssh_keys.empty?
-                ssh_options[:keys] = ssh_keys
-                server_transfer_spec['EX_ssh_key_paths'] = ssh_keys
-                ssh_keys.each do |k|
-                  Log.log.warn("no such key file: #{k}") unless File.exist?(k)
-                end
-                cred_set = true
+            # Using WSS
+            return server_transfer_spec
+          end
+          if !server_uri.scheme.eql?(SSH_SCHEME)
+            Log.log.warn('URL scheme is https but no token was provided in transfer spec.')
+            Log.log.warn("If you want to access the server, not using WSS for session, then use a URL with scheme \"#{SSH_SCHEME}\" and proper SSH port")
+            assumed_url = "#{SSH_SCHEME}://#{server_transfer_spec['remote_host']}:#{Aspera::Fasp::TransferSpec::SSH_PORT}"
+            Log.log.warn{"Assuming proper URL is: #{assumed_url}"}
+            server_uri = URI.parse(assumed_url)
+          end
+          # Scheme is SSH
+          if options.get_option(:username).nil?
+            options.set_option(:username, Aspera::Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER)
+            Log.log.info{"No username provided: Assuming default transfer user: #{Aspera::Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER}"}
+          end
+          server_transfer_spec['remote_user'] = options.get_option(:username, is_type: :mandatory)
+          ssh_args = options.get_option(:ssh_options)
+          ssh_args = {} if ssh_args.nil?
+          raise 'expecting a Hash for ssh_options' unless ssh_args.is_a?(Hash)
+          @ssh_opts = ssh_args.symbolize_keys
+          if !server_uri.port.nil?
+            @ssh_opts[:port] = server_uri.port
+            server_transfer_spec['ssh_port'] = server_uri.port
+          end
+          cred_set = false
+          password = options.get_option(:password)
+          if !password.nil?
+            @ssh_opts[:password] = password
+            server_transfer_spec['remote_password'] = password
+            cred_set = true
+          end
+          ssh_key_list = options.get_option(:ssh_keys)
+          if !ssh_key_list.nil?
+            raise 'Expecting single value or array for ssh_keys' unless ssh_key_list.is_a?(Array) || ssh_key_list.is_a?(String)
+            ssh_key_list = [ssh_key_list] if ssh_key_list.is_a?(String)
+            ssh_key_list.map!{|p|File.expand_path(p)}
+            Log.log.debug{"SSH keys=#{ssh_key_list}"}
+            if !ssh_key_list.empty?
+              @ssh_opts[:keys] = ssh_key_list
+              server_transfer_spec['EX_ssh_key_paths'] = ssh_key_list
+              ssh_key_list.each do |k|
+                Log.log.warn{"No such key file: #{k}"} unless File.exist?(k)
               end
+              cred_set = true
             end
-            # if user provided transfer spec has a token, we will use by pass keys
-            cred_set = true if transfer.option_transfer_spec['token'].is_a?(String)
-            raise 'either password, key , or transfer spec token must be provided' if !cred_set
-            shell_executor = Ssh.new(server_transfer_spec['remote_host'],server_transfer_spec['remote_user'],ssh_options)
           end
+          # if user provided transfer spec has a token, we will use bypass keys
+          cred_set = true if transfer.option_transfer_spec['token'].is_a?(String)
+          raise 'Either password, key , or transfer spec token must be provided' if !cred_set
+          return server_transfer_spec
+        end
+
+        def execute_transfer(command, transfer_spec)
+          case command
+          when :upload, :download
+            Fasp::TransferSpec.action_to_direction(transfer_spec, command)
+            return Main.result_transfer(transfer.start(transfer_spec))
+          when :sync
+            sync_plugin = Sync.new(@agents, transfer_spec: transfer_spec)
+            return sync_plugin.execute_action
+          end
+        end
 
-          # get command and set aliases
-          command = options.get_next_command(ACTIONS)
-          command = :ls if command.eql?(:browse)
-          command = :rm if command.eql?(:delete)
-          command = :mv if command.eql?(:rename)
+        # actions without ascmd
+        BASE_ACTIONS = %i[health].concat(TRANSFER_COMMANDS).freeze
+        # all actions
+        ACTIONS = [].concat(BASE_ACTIONS, Aspera::AsCmd::OPERATIONS, ASCMD_ALIASES.keys).freeze
+
+        def execute_action
+          server_transfer_spec = options_to_base_transfer_spec
+          ascmd_executor = if !@ssh_opts.nil?
+            Ssh.new(server_transfer_spec['remote_host'], server_transfer_spec['remote_user'], @ssh_opts)
+          elsif server_transfer_spec.key?('wss_enabled')
+            nil
+          else
+            LocalExecutor.new
+          end
+          # the set of available commands depends on SSH executor availability (i.e. no WSS)
+          available_commands = ascmd_executor.nil? ? BASE_ACTIONS : ACTIONS
+          # get command and translate aliases
+          command = options.get_next_command(available_commands)
+          command = ASCMD_ALIASES[command] if ASCMD_ALIASES.key?(command)
           case command
           when :health
             nagios = Nagios.new
@@ -107,35 +159,37 @@ module Aspera
                 'direction'     => 'send',
                 'cookie'        => 'aspera.sync', # hide in console
                 'resume_policy' => 'none',
-                'paths'         => [{'source' => filepath,'destination' => '.fasping'}]
+                'paths'         => [{'source' => filepath, 'destination' => '.fasping'}]
               })
-              statuses = transfer.start(probe_ts,{src: :direct})
+              statuses = transfer.start(probe_ts)
               file.unlink
               if TransferAgent.session_status(statuses).eql?(:success)
-                nagios.add_ok('transfer','ok')
+                nagios.add_ok('transfer', 'ok')
               else
-                nagios.add_critical('transfer',statuses.reject{|i|i.eql?(:success)}.first.to_s)
+                nagios.add_critical('transfer', statuses.reject{|i|i.eql?(:success)}.first.to_s)
               end
             else raise 'ERROR'
             end
             return nagios.result
-          when :upload
-            return Main.result_transfer(transfer.start(server_transfer_spec.merge('direction' => Fasp::TransferSpec::DIRECTION_SEND),{src: :direct}))
-          when :download
-            return Main.result_transfer(transfer.start(server_transfer_spec.merge('direction' => Fasp::TransferSpec::DIRECTION_RECEIVE),{src: :direct}))
+          when *TRANSFER_COMMANDS
+            return execute_transfer(command, server_transfer_spec)
           when *Aspera::AsCmd::OPERATIONS
-            args = options.get_next_argument('ascmd command arguments',expected: :multiple,mandatory: false)
-            ascmd = Aspera::AsCmd.new(shell_executor)
+            args = options.get_next_argument('ascmd command arguments', expected: :multiple, mandatory: false)
+            ascmd = Aspera::AsCmd.new(ascmd_executor)
             begin
-              result = ascmd.send(:execute_single,command,args)
+              result = ascmd.send(:execute_single, command, args)
               case command
-              when :mkdir,:mv,:cp,:rm then return Main.result_success
-              when :ls                then return {type: :object_list,data: key_symb_to_str_list(result),fields: %w[zmode zuid zgid size mtime name]}
-              when :df                then return {type: :object_list,data: key_symb_to_str_list(result)}
-              when :du,:md5sum,:info  then return {type: :single_object,data: result.stringify_keys}
+              when :mkdir, :mv, :cp, :rm
+                return Main.result_success
+              when :ls
+                return {type: :object_list, data: result.map(&:stringify_keys), fields: %w[zmode zuid zgid size mtime name]}
+              when :df
+                return {type: :object_list, data: result.map(&:stringify_keys)}
+              when :du, :md5sum, :info
+                return {type: :single_object, data: result.stringify_keys}
               end
             rescue Aspera::AsCmd::Error => e
-              raise CliBadArgument,e.extended_message
+              raise CliBadArgument, e.extended_message
             end
           else raise 'internal error: unexpected action'
           end

data/lib/aspera/cli/plugins/shares.rb

@@ -5,13 +5,14 @@ require 'aspera/cli/plugins/node'
 module Aspera
   module Cli
     module Plugins
-      class Shares < BasicAuthPlugin
+      class Shares < Aspera::Cli::BasicAuthPlugin
         class << self
           def detect(base_url)
             api = Rest.new({base_url: base_url})
             # Shares
             begin
               # shall fail: shares requires auth, but we check error message
+              # TODO: use ping instead ?
               api.read('node_api/app')
             rescue RestCallError => e
               if e.response.code.to_s.eql?('401') && e.response.body.eql?('{"error":{"user_message":"API user authentication failed"}}')
@@ -26,8 +27,8 @@ module Aspera
         #          super(env)
         #        end
 
-        SAML_IMPORT_MANDATORY=%w[id name_id].freeze
-        SAML_IMPORT_ALLOWED=[SAML_IMPORT_MANDATORY,%w[email given_name surname]].flatten.freeze
+        SAML_IMPORT_MANDATORY = %w[id name_id].freeze
+        SAML_IMPORT_ALLOWED = %w[email given_name surname].concat(SAML_IMPORT_MANDATORY).freeze
 
         ACTIONS = %i[health repository admin].freeze
 
@@ -37,23 +38,23 @@ module Aspera
           when :health
             nagios = Nagios.new
             begin
-              Rest.
-                new(base_url: options.get_option(:url,is_type: :mandatory)+'/node_api').
-                call(
+              Rest
+                .new(base_url: options.get_option(:url, is_type: :mandatory) + '/node_api')
+                .call(
                   operation: 'GET',
                   subpath: 'ping',
                   headers: {'content-type': 'application/json'},
                   return_error: true)
-              nagios.add_ok('shares api','accessible')
+              nagios.add_ok('shares api', 'accessible')
             rescue StandardError => e
-              nagios.add_critical('node api',e.to_s)
+              nagios.add_critical('node api', e.to_s)
             end
             return nagios.result
           when :repository
             api_shares_node = basic_auth_api('node_api')
-            command = options.get_next_command(Node::COMMON_ACTIONS)
+            command = options.get_next_command(Node::COMMANDS_SHARES)
             case command
-            when *Node::COMMON_ACTIONS then Node.new(@agents.merge(skip_basic_auth_options: true,node_api: api_shares_node)).execute_action(command)
+            when *Node::COMMANDS_SHARES then Node.new(@agents.merge(skip_basic_auth_options: true, node_api: api_shares_node)).execute_action(command)
             else raise "INTERNAL ERROR, unknown command: [#{command}]"
             end
           when :admin
@@ -65,30 +66,30 @@ module Aspera
               user_id = instance_identifier if %i[app_authorizations share_permissions].include?(command)
               case command
               when :list
-                return {type: :object_list,data: api_shares_admin.read('data/users')[:data],fields: %w[id username email directory_user urn]}
+                return {type: :object_list, data: api_shares_admin.read('data/users')[:data], fields: %w[id username email directory_user urn]}
               when :app_authorizations
-                return {type: :single_object,data: api_shares_admin.read("data/users/#{user_id}/app_authorizations")[:data]}
+                return {type: :single_object, data: api_shares_admin.read("data/users/#{user_id}/app_authorizations")[:data]}
               when :share_permissions
-                #share_name = options.get_next_argument('share name')
-                #all_shares = api_shares_admin.read('data/shares')[:data]
-                #share_id = all_shares.find{|s| s['name'].eql?(share_name)}['id']
-                return {type: :object_list,data: api_shares_admin.read("data/users/#{user_id}/share_permissions")[:data]}
+                # share_name = options.get_next_argument('share name')
+                # all_shares = api_shares_admin.read('data/shares')[:data]
+                # share_id = all_shares.find{|s| s['name'].eql?(share_name)}['id']
+                return {type: :object_list, data: api_shares_admin.read("data/users/#{user_id}/share_permissions")[:data]}
               when :saml_import
-                parameters = options.get_option(:value)
-                return do_bulk_operation(parameters,'created') do |user_params|
-                  user_params=user_params.transform_keys{|k|k.gsub(/\s+/,'_').downcase}
+                parameters = options.get_option(:value, is_type: :mandatory)
+                return do_bulk_operation(parameters, 'created') do |user_params|
+                  user_params = user_params.transform_keys{|k|k.gsub(/\s+/, '_').downcase}
                   raise 'expecting Hash' unless user_params.is_a?(Hash)
                   SAML_IMPORT_MANDATORY.each{|p|raise "missing mandatory field: #{p}" if user_params[p].nil?}
-                  user_params.keys.each do |p|
+                  user_params.each_key do |p|
                     raise "unsupported field: #{p}, use: #{SAML_IMPORT_ALLOWED.join(',')}" unless SAML_IMPORT_ALLOWED.include?(p)
                   end
-                  api_shares_admin.create('data/saml_users/import',user_params)[:data]
+                  api_shares_admin.create('data/saml_users/import', user_params)[:data]
                 end
               when :ldap_import
                 parameters = options.get_option(:value)
-                return do_bulk_operation(parameters,'created') do |user_name|
-                  raise 'expecting string (user name), have #{user_params.class}' unless user_params.is_a?(String)
-                  api_shares_admin.create('data/ldap_users',{'user'=>user_name})[:data]
+                return do_bulk_operation(parameters, 'created') do |user_name|
+                  raise "expecting string (user name), have #{user_name.class}" unless user_name.is_a?(String)
+                  api_shares_admin.create('data/ldap_users', {'user'=>user_name})[:data]
                 end
               end
             when :share
@@ -97,12 +98,12 @@ module Aspera
               all_shares = api_shares_admin.read('data/shares')[:data]
               case command
               when :list
-                return {type: :object_list,data: all_shares,fields: %w[id name status status_message]}
+                return {type: :object_list, data: all_shares, fields: %w[id name status status_message]}
               when :user_permissions
-                #share_name = options.get_next_argument('share name')
-                #share_id = all_shares.find{|s| s['name'].eql?(share_name)}['id']
-                #raise "NOT IMPLEMENTED: #{share_name} #{share_id}"
-                return {type: :object_list,data: api_shares_admin.read("data/shares/#{share_id}/user_permissions")[:data]}
+                # share_name = options.get_next_argument('share name')
+                # share_id = all_shares.find{|s| s['name'].eql?(share_name)}['id']
+                # raise "NOT IMPLEMENTED: #{share_name} #{share_id}"
+                return {type: :object_list, data: api_shares_admin.read("data/shares/#{share_id}/user_permissions")[:data]}
               end
             end
           end

data/lib/aspera/cli/plugins/sync.rb

@@ -9,13 +9,16 @@ require 'open3'
 module Aspera
   module Cli
     module Plugins
-      # list and download connect client versions, select FASP implementation
-      class Sync < Plugin
-        def initialize(env)
+      # Execute Aspera Sync
+      class Sync < Aspera::Cli::Plugin
+        def initialize(env, transfer_spec: nil)
           super(env)
-          options.add_opt_simple(:parameters,'extended value for session set definition')
-          options.add_opt_simple(:session_name,'name of session to use for admin commands, by default first one')
+          options.add_opt_simple(:sync_info, 'Information for sync instance and sessions (Hash)')
+          options.add_opt_simple(:sync_session, 'Name of session to use for admin commands. default: first in parameters')
           options.parse_options!
+          return if env[:man_only]
+          @params = options.get_option(:sync_info, is_type: :mandatory)
+          Aspera::Sync.update_parameters_with_transfer_spec(@params, transfer_spec) unless transfer_spec.nil?
         end
 
         ACTIONS = %i[start admin].freeze
@@ -24,38 +27,15 @@ module Aspera
           command = options.get_next_command(ACTIONS)
           case command
           when :start
-            env_args = Aspera::Sync.new(options.get_option(:parameters,is_type: :mandatory)).compute_args
-            async_bin = 'async'
-            Log.log.debug("execute: #{env_args[:env].map{|k,v| "#{k}=\"#{v}\""}.join(' ')} \"#{async_bin}\" \"#{env_args[:args].join('" "')}\"")
-            res = system(env_args[:env],[async_bin,async_bin],*env_args[:args])
-            Log.log.debug("result=#{res}")
-            case res
-            when true then return Main.result_success
-            when false then raise "failed: #{$CHILD_STATUS}"
-            when nil then return Main.result_status("not started: #{$CHILD_STATUS}")
-            else raise 'internal error: unspecified case'
-            end
+            Aspera::Sync.new(@params).start
+            return Main.result_success
           when :admin
-            p = options.get_option(:parameters,is_type: :mandatory)
-            n = options.get_option(:session_name)
-            cmdline = ['asyncadmin','--quiet']
-            session = n.nil? ? p['sessions'].first : p['sessions'].find{|s|s['name'].eql?(n)}
-            cmdline.push('--name=' + session['name'])
-            if session.has_key?('local_db_dir')
-              cmdline.push('--local-db-dir=' + session['local_db_dir'])
-            else
-              cmdline.push('--local-dir=' + session['local_dir'])
-            end
+            sync_admin = Aspera::SyncAdmin.new(@params, options.get_option(:sync_session))
             command2 = options.get_next_command([:status])
             case command2
             when :status
-              stdout, stderr, status = Open3.capture3(*cmdline)
-              Log.log.debug("status=#{status}, stderr=#{stderr}")
-              items = stdout.split("\n").each_with_object({}){|l,m|i = l.split(/:  */);m[i.first.lstrip] = i.last.lstrip;}
-              return {type: :single_object,data: items}
-            else raise 'error'
-            end # command
-          else raise 'error'
+              return {type: :single_object, data: sync_admin.status}
+            end # command2
           end # command
         end # execute_action
       end # Sync