aspera-cli 4.15.0 → 4.17.0

data/bin/ascli

@@ -3,36 +3,18 @@
 
 Encoding.default_internal = Encoding::UTF_8
 Encoding.default_external = Encoding::UTF_8
-# coverage for tests
-if ENV.key?('ENABLE_COVERAGE')
-  require 'simplecov'
-  require 'securerandom'
-  # compute gem source root based on this script location, assuming it is in bin/
-  # use dirname instead of gsub, in case folder separator is not /
-  development_root = File.dirname(File.dirname(File.realpath(__FILE__)))
-  SimpleCov.root(development_root)
-  SimpleCov.enable_for_subprocesses if SimpleCov.respond_to?(:enable_for_subprocesses)
-  # keep cache data for 1 day (must be longer that time to run the whole test suite)
-  SimpleCov.merge_timeout(86400)
-  SimpleCov.command_name(SecureRandom.uuid)
-  SimpleCov.at_exit do
-    original_file_descriptor = $stdout
-    $stdout.reopen(File.join(development_root, 'simplecov.log'))
-    SimpleCov.result.format!
-    $stdout.reopen(original_file_descriptor)
-  end
-  SimpleCov.start
-end
-# if in development, add path to gem
-#
+
 begin
-  require 'aspera/cli/main'
-rescue LoadError
-  # development environment
-  development_root = File.dirname(File.dirname(File.realpath(__FILE__)))
-  $LOAD_PATH.unshift(File.join(development_root, 'lib'))
-  require 'aspera/cli/main'
+  gem_lib_folder = File.join(File.dirname(File.dirname(File.realpath(__FILE__))), 'lib')
+  Kernel.load(File.join(gem_lib_folder, 'aspera/coverage.rb'))
+  begin
+    require 'aspera/cli/main'
+  rescue LoadError
+    # if in development, add path toward gem
+    $LOAD_PATH.unshift(gem_lib_folder)
+    require 'aspera/cli/main'
+  end
+  require 'aspera/environment'
+  Aspera::Environment.fix_home
+  Aspera::Cli::Main.new(ARGV).process_command_line
 end
-require 'aspera/environment'
-Aspera::Environment.fix_home
-Aspera::Cli::Main.new(ARGV).process_command_line

data/bin/asession

@@ -3,16 +3,18 @@
 
 # Laurent Martin/2017
 $LOAD_PATH.unshift("#{File.dirname(__FILE__)}/../lib")
-require 'aspera/fasp/agent_direct'
+require 'aspera/agent/direct'
 require 'aspera/cli/extended_value'
+require 'aspera/ascp/installation'
 require 'aspera/log'
 require 'json'
 # extended transfer spec parameter (only used in asession)
 # Change log level
-TS_LOG_LEVEL = 'EX_loglevel'
+PARAM_LOG_LEVEL = 'loglevel'
 # by default go to /tmp/username.filelist
-TS_TMP_FILE_LIST_FOLDER = 'EX_file_list_folder'
-
+PARAM_TMP_FILE_LIST_FOLDER = 'file_list_folder'
+# place transfer spec in that
+PARAM_SPEC = 'spec'
 SAMPLE_DEMO = '"remote_host":"demo.asperasoft.com","remote_user":"asperaweb","ssh_port":33001,"remote_password":"demoaspera"'
 SAMPLE_DEMO2 = '"direction":"receive","destination_root":"./test.dir"'
 def assert_usage(assertion, error_message)
@@ -21,25 +23,27 @@ def assert_usage(assertion, error_message)
   $stderr.puts('USAGE')
   $stderr.puts('    asession')
   $stderr.puts('    asession -h|--help')
-  $stderr.puts('    asession <transfer spec extended value>')
+  $stderr.puts('    asession <session spec extended value>')
   $stderr.puts('    ')
   $stderr.puts('    If no argument is provided, default will be used: @json:@stdin')
   $stderr.puts('    -h, --help display this message')
-  $stderr.puts('    <transfer spec extended value> a JSON value for transfer_spec, using the prefix: @json:')
+  $stderr.puts('    <session spec extended value> a dictionary value (Hash)')
   $stderr.puts('    The value can be either:')
   $stderr.puts("       the JSON description itself, e.g. @json:'{\"xx\":\"yy\",...}'")
   $stderr.puts('       @json:@stdin, if the JSON is provided from stdin')
   $stderr.puts('       @json:@file:<path>, if the JSON is provided from a file')
+  $stderr.puts("    Parameter #{PARAM_SPEC} is mandatory, it contains the transfer spec")
   $stderr.puts('    Asynchronous commands can be provided on STDIN, examples:')
   $stderr.puts('       {"type":"START","source":"/aspera-test-dir-tiny/200KB.2"}')
   $stderr.puts('       {"type":"START","source":"xx","destination":"yy"}')
   $stderr.puts('       {"type":"DONE"}')
-  $stderr.puts(%Q(Note: debug information can be placed on STDERR, using the "#{TS_LOG_LEVEL}" parameter in transfer spec (debug=0)))
+  $stderr.puts(%Q(Note: debug information can be placed on STDERR, using the "#{PARAM_LOG_LEVEL}" parameter in session spec (debug=0)))
   $stderr.puts('EXAMPLES')
-  $stderr.puts(%Q(    asession @json:'{#{SAMPLE_DEMO},#{SAMPLE_DEMO2},"paths":[{"source":"/aspera-test-dir-tiny/200KB.1"}]}'))
-  $stderr.puts(%q(    echo '{"remote_host":...}'|asession @json:@stdin))
+  $stderr.puts(%Q(    asession @json:'{"#{PARAM_SPEC}":{#{SAMPLE_DEMO},#{SAMPLE_DEMO2},"paths":[{"source":"/aspera-test-dir-tiny/200KB.1"}]}}'))
+  $stderr.puts(%Q(    echo '{"#{PARAM_SPEC}":{"remote_host":...}}'|asession @json:@stdin))
   Process.exit(1)
 end
+Aspera::Ascp::Installation.instance.sdk_folder = File.join(Dir.home, '.aspera', 'sdk')
 
 parameter_source_err_msg = ' (argument), did you specify: "@json:" ?'
 # by default assume JSON input on stdin if no argument
@@ -52,33 +56,39 @@ assert_usage(ARGV.length.eql?(1), 'exactly one argument is expected')
 assert_usage(!['-h', '--help'].include?(ARGV.first), nil)
 # parse transfer spec
 begin
-  transfer_spec_arg = ARGV.pop
-  transfer_spec = Aspera::Cli::ExtendedValue.instance.evaluate(transfer_spec_arg)
+  session_argument = ARGV.pop
+  session_spec = Aspera::Cli::ExtendedValue.instance.evaluate(session_argument)
 rescue
-  assert_usage(false, "Cannot extract transfer spec from: #{transfer_spec_arg}")
+  assert_usage(false, "Cannot parse argument: #{session_argument}")
 end
 # ensure right type
-assert_usage(transfer_spec.is_a?(Hash), "the value must be a hash table#{parameter_source_err_msg}")
+assert_usage(session_spec.is_a?(Hash), "The value must be a hash table#{parameter_source_err_msg}")
+assert_usage(session_spec[PARAM_SPEC].is_a?(Hash), "the value must contain key #{PARAM_SPEC}")
 # additional debug capability
-if transfer_spec.key?(TS_LOG_LEVEL)
-  Aspera::Log.instance.level = transfer_spec[TS_LOG_LEVEL]
-  transfer_spec.delete(TS_LOG_LEVEL)
+if session_spec.key?(PARAM_LOG_LEVEL)
+  Aspera::Log.instance.level = session_spec[PARAM_LOG_LEVEL]
 end
 # possibly override temp folder
-if transfer_spec.key?(TS_TMP_FILE_LIST_FOLDER)
-  Aspera::Fasp::Parameters.file_list_folder = transfer_spec[TS_TMP_FILE_LIST_FOLDER]
-  transfer_spec.delete(TS_TMP_FILE_LIST_FOLDER)
+if session_spec.key?(PARAM_TMP_FILE_LIST_FOLDER)
+  Aspera::Transfer::Parameters.file_list_folder = session_spec[PARAM_TMP_FILE_LIST_FOLDER]
+end
+session_spec['agent'] = {} unless session_spec.key?('agent')
+session_spec['agent']['quiet'] = true
+session_spec['agent']['management_cb'] = ->(event) do
+  puts JSON.generate(Aspera::Ascp::Management.enhanced_event_format(event))
 end
 # get local agent (ascp), disable ascp output on stdout to not mix with JSON events
-client = Aspera::Fasp::AgentDirect.new({quiet: true})
+client = Aspera::Agent::Direct.new(session_spec['agent'])
 # start transfer (asynchronous)
-job_id = client.start_transfer(transfer_spec)
+job_id = client.start_transfer(session_spec[PARAM_SPEC])
 # async commands
 Thread.new do
+  # we assume here a single session
+  session_id = client.sessions_by_job(job_id).first
   begin # rubocop:disable Style/RedundantBegin
     loop do
       data = JSON.parse($stdin.gets)
-      client.send_command(job_id, 0, data)
+      client.send_command(session_id, data)
     end
   rescue
     Process.exit(1)

data/examples/dascli

@@ -7,7 +7,7 @@
 : "${imgtag=$image:$version}"
 # set env var `docker` to podman, to use podman
 : "${docker:=docker}"
-# set env var docker_args to add options to docker run (transform var into array) # spellcheck disable=SC2086
+# set env var `docker_args` to add options to docker run (then, transform this var into array) # spellcheck disable=SC2086
 read -a add_dock_args <<< $docker_args
 # set env var ASCLI_HOME to set the config folder on host
 : "${ASCLI_HOME:=$HOME/.aspera/ascli}"
@@ -24,7 +24,7 @@ exec $docker run \
 --interactive \
 --user root \
 --env ASCLI_HOME="$ascli_home_container" \
---volume "$ASCLI_HOME:$ascli_home_container" \
+--volume "$ASCLI_HOME:$ascli_home_container:z" \
 "${add_dock_args[@]}" \
 "$imgtag" \
 "$@"

data/lib/aspera/agent/alpha.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+require 'aspera/agent/base'
+require 'aspera/rest'
+require 'aspera/log'
+require 'aspera/json_rpc'
+require 'aspera/open_application'
+require 'securerandom'
+
+module Aspera
+  module Agent
+    class Alpha < Base
+      # try twice the main init url in sequence
+      START_URIS = ['aspera://', 'aspera://', 'aspera://']
+      # delay between each try to start the app
+      SLEEP_SEC_BETWEEN_RETRY = 5
+      APP_IDENTIFIER = 'com.ibm.software.aspera.desktop'
+      APP_NAME = 'Aspera Desktop Alpha Client'
+      private_constant :START_URIS, :SLEEP_SEC_BETWEEN_RETRY
+      def initialize(options)
+        @application_id = SecureRandom.uuid
+        super(options)
+        raise 'Using client requires a graphical environment' if !OpenApplication.default_gui_mode.eql?(:graphical)
+        method_index = 0
+        begin
+          # curl 'http://127.0.0.1:33024/' -X POST -H 'content-type: application/json' --data-raw '{"jsonrpc":"2.0","params":[],"id":999999,"method":"rpc.discover"}'
+          # https://playground.open-rpc.org/?schemaUrl=http://127.0.0.1:33024
+          @client_app_api = Aspera::JsonRpcClient.new(Aspera::Rest.new(base_url: aspera_client_api_url))
+          client_info = @client_app_api.get_info
+          Log.log.debug{Log.dump(:client_version, client_info)}
+          Log.log.info('Client was reached') if method_index > 0
+        rescue Errno::ECONNREFUSED => e
+          start_url = START_URIS[method_index]
+          method_index += 1
+          raise StandardError, "Unable to start #{APP_NAME} #{method_index} times" if start_url.nil?
+          Log.log.warn{"#{APP_NAME} is not started (#{e}). Trying to start it ##{method_index}..."}
+          if !OpenApplication.uri_graphical(start_url)
+            OpenApplication.uri_graphical('https://www.ibm.com/aspera/connect/')
+            raise StandardError, "#{APP_NAME} is not installed"
+          end
+          sleep(SLEEP_SEC_BETWEEN_RETRY)
+          retry
+        end
+      end
+
+      def sdk_log_file
+        File.join(Dir.home, 'Library', 'Logs', APP_IDENTIFIER, 'ibm-aspera-desktop.log')
+      end
+
+      def aspera_client_api_url
+        log_file = sdk_log_file
+        url = nil
+        File.open(log_file, 'r') do |file|
+          file.each_line do |line|
+            line = line.chomp
+            if (m = line.match(/JSON-RPC server listening on (.*)/))
+              url = "http://#{m[1]}"
+            end
+          end
+        end
+        url = 'http://127.0.0.1:33024' if url.nil?
+        raise StandardError, "Unable to find the JSON-RPC server URL in #{log_file}" if url.nil?
+        return url
+      end
+
+      def start_transfer(transfer_spec, token_regenerator: nil)
+        @request_id = SecureRandom.uuid
+        # if there is a token, we ask the client app to use well known ssh private keys
+        # instead of asking password
+        transfer_spec['authentication'] = 'token' if transfer_spec.key?('token')
+        result = @client_app_api.start_transfer(app_id: @application_id, desktop_spec: {}, transfer_spec: transfer_spec)
+        @xfer_id = result['uuid']
+      end
+
+      def wait_for_transfers_completion
+        started = false
+        pre_calc = false
+        begin
+          loop do
+            transfer = @client_app_api.get_transfer(app_id: @application_id, transfer_id: @xfer_id)
+            case transfer['status']
+            when 'initiating', 'queued'
+              notify_progress(session_id: nil, type: :pre_start, info: transfer['status'])
+            when 'running'
+              if !started
+                notify_progress(session_id: @xfer_id, type: :session_start)
+                started = true
+              end
+              if !pre_calc && (transfer['bytes_expected'] != 0)
+                notify_progress(type: :session_size, session_id: @xfer_id, info: transfer['bytes_expected'])
+                pre_calc = true
+              else
+                notify_progress(type: :transfer, session_id: @xfer_id, info: transfer['bytes_written'])
+              end
+            when 'completed'
+              notify_progress(type: :end, session_id: @xfer_id)
+              break
+            when 'failed'
+              notify_progress(type: :end, session_id: @xfer_id)
+              raise Transfer::Error, transfer['error_desc']
+            when 'cancelled'
+              notify_progress(type: :end, session_id: @xfer_id)
+              raise Transfer::Error, 'Transfer cancelled by user'
+            else
+              notify_progress(type: :end, session_id: @xfer_id)
+              raise Transfer::Error, "unknown status: #{transfer['status']}: #{transfer['error_desc']}"
+            end
+            sleep(1)
+          end
+        rescue StandardError => e
+          return [e]
+        end
+        return [:success]
+      end # wait
+    end # AgentAlpha
+  end
+end

data/lib/aspera/agent/base.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require 'aspera/log'
+require 'aspera/assert'
+module Aspera
+  module Agent
+    # Base class for transfer agents
+    class Base
+      RUBY_EXT = '.rb'
+      class << self
+        # compute options from user provided and default options
+        def options(default:, options:)
+          result = options.symbolize_keys
+          available = default.map{|k, v|"#{k}(#{v})"}.join(', ')
+          result.each_key do |k|
+            Aspera.assert_values(k, default.keys){"transfer agent parameter: #{k}"}
+            # check it is the expected type: too limiting, as we can have an Integer or Float, or symbol and string
+            # raise "Invalid value for transfer agent parameter: #{k}, expect #{default[k].class.name}" unless default[k].nil? || v.is_a?(default[k].class)
+          end
+          default.each do |k, v|
+            raise "Missing required agent parameter: #{k}. Parameters: #{available}" if v.eql?(:required) && !result.key?(k)
+            result[k] = v unless result.key?(k)
+          end
+          return result
+        end
+
+        # discover available agents
+        def agent_list
+          base_class = File.basename(__FILE__)
+          Dir.entries(File.dirname(File.expand_path(__FILE__))).select do |file|
+            file.end_with?(RUBY_EXT) && !file.eql?(base_class)
+          end.map{|file|file[0..(-1 - RUBY_EXT.length)].to_sym}
+        end
+      end
+      def wait_for_completion
+        # list of: :success or "error message string"
+        statuses = wait_for_transfers_completion
+        @progress&.reset
+        Aspera.assert_type(statuses, Array)
+        Aspera.assert(statuses.none?{|i|!i.eql?(:success) && !i.is_a?(StandardError)}){"bad statuses content: #{statuses}"}
+        return statuses
+      end
+
+      private
+
+      def initialize(options)
+        # method `shutdown` is optional
+        Aspera.assert(respond_to?(:start_transfer))
+        Aspera.assert(respond_to?(:wait_for_transfers_completion))
+        Aspera.assert_type(options, Hash){'transfer agent options'}
+        Log.log.debug{Log.dump(:agent_options, options)}
+        @progress = options[:progress]
+        options.delete(:progress)
+      end
+
+      def notify_progress(**parameters)
+        @progress&.event(**parameters)
+      end
+    end
+  end
+end

data/lib/aspera/{fasp/agent_connect.rb → agent/connect.rb}

@@ -1,17 +1,17 @@
 # frozen_string_literal: true
 
-require 'aspera/fasp/agent_base'
+require 'aspera/agent/base'
 require 'aspera/rest'
 require 'aspera/open_application'
 require 'securerandom'
 
 module Aspera
-  module Fasp
-    class AgentConnect < Aspera::Fasp::AgentBase
+  module Agent
+    class Connect < Base
       # try twice the main init url in sequence
       CONNECT_START_URIS = ['fasp://initialize', 'fasp://initialize', 'aspera-drive://initialize', 'https://test-connect.ibmaspera.com/']
       # delay between each try to start connect
-      SLEEP_SEC_BETWEEN_RETRY = 3
+      SLEEP_SEC_BETWEEN_RETRY = 5
       private_constant :CONNECT_START_URIS, :SLEEP_SEC_BETWEEN_RETRY
       def initialize(options)
         super(options)
@@ -21,9 +21,11 @@ module Aspera
         raise 'Using connect requires a graphical environment' if !OpenApplication.default_gui_mode.eql?(:graphical)
         method_index = 0
         begin
-          connect_url = Products.connect_uri
+          connect_url = Ascp::Products.connect_uri
           Log.log.debug{"found: #{connect_url}"}
-          @connect_api = Rest.new({base_url: "#{connect_url}/v5/connect", headers: {'Origin' => Rest.user_agent}}) # could use v6 also now
+          @connect_api = Rest.new(
+            base_url: "#{connect_url}/v5/connect", # could use v6 also now
+            headers: {'Origin' => Rest.user_agent})
           connect_info = @connect_api.read('info/version')[:data]
           Log.log.info('Connect was reached') if method_index > 0
           Log.log.debug{Log.dump(:connect_version, connect_info)}
@@ -33,7 +35,7 @@ module Aspera
           raise StandardError, "Unable to start connect #{method_index} times" if start_url.nil?
           Log.log.warn{"Aspera Connect is not started (#{e}). Trying to start it ##{method_index}..."}
           if !OpenApplication.uri_graphical(start_url)
-            OpenApplication.uri_graphical('https://downloads.asperasoft.com/connect2/')
+            OpenApplication.uri_graphical('https://www.ibm.com/aspera/connect/')
             raise StandardError, 'Connect is not installed'
           end
           sleep(SLEEP_SEC_BETWEEN_RETRY)
@@ -67,7 +69,7 @@ module Aspera
           }]}
         # asynchronous anyway
         res = @connect_api.create('transfers/start', connect_transfer_args)[:data]
-        @xfer_id = res['transfer_specs'].first['transfer_spec']['tags'][Fasp::TransferSpec::TAG_RESERVED]['xfer_id']
+        @xfer_id = res['transfer_specs'].first['transfer_spec']['tags'][Transfer::Spec::TAG_RESERVED]['xfer_id']
       end
 
       def wait_for_transfers_completion
@@ -105,13 +107,13 @@ module Aspera
                 break
               when 'failed'
                 notify_progress(type: :end, session_id: session_id)
-                raise Fasp::Error, transfer['error_desc']
+                raise Transfer::Error, transfer['error_desc']
               when 'cancelled'
                 notify_progress(type: :end, session_id: session_id)
-                raise Fasp::Error, 'Transfer cancelled by user'
+                raise Transfer::Error, 'Transfer cancelled by user'
               else
                 notify_progress(type: :end, session_id: session_id)
-                raise Fasp::Error, "unknown status: #{transfer['status']}: #{transfer['error_desc']}"
+                raise Transfer::Error, "unknown status: #{transfer['status']}: #{transfer['error_desc']}"
               end
             end
             sleep(1)