aspera-cli 4.18.1 → 4.20.0

data/bin/asession

@@ -1,20 +1,24 @@
 #!/usr/bin/env ruby
 # frozen_string_literal: true
 
-# Laurent Martin/2017
-$LOAD_PATH.unshift("#{File.dirname(__FILE__)}/../lib")
+gem_lib_folder = File.join(File.dirname(File.dirname(File.realpath(__FILE__))), 'lib')
+Kernel.load(File.join(gem_lib_folder, 'aspera/coverage.rb'))
+$LOAD_PATH.unshift(gem_lib_folder)
 require 'aspera/agent/direct'
 require 'aspera/cli/extended_value'
 require 'aspera/ascp/installation'
 require 'aspera/log'
 require 'json'
 # extended transfer spec parameter (only used in asession)
-# Change log level
+PARAM_SPEC = 'spec'
+# log level
 PARAM_LOG_LEVEL = 'loglevel'
+# transfer agent options
+PARAM_AGENT = 'agent'
 # by default go to /tmp/username.filelist
 PARAM_TMP_FILE_LIST_FOLDER = 'file_list_folder'
+PARAM_SDK = 'sdk'
 # place transfer spec in that
-PARAM_SPEC = 'spec'
 SAMPLE_DEMO = '"remote_host":"demo.asperasoft.com","remote_user":"asperaweb","ssh_port":33001,"remote_password":"demoaspera"'
 SAMPLE_DEMO2 = '"direction":"receive","destination_root":"./test.dir"'
 def assert_usage(assertion, error_message)
@@ -23,28 +27,30 @@ def assert_usage(assertion, error_message)
   $stderr.puts('USAGE')
   $stderr.puts('    asession')
   $stderr.puts('    asession -h|--help')
-  $stderr.puts('    asession <session spec extended value>')
+  $stderr.puts('    asession [<session spec extended value>]')
   $stderr.puts('    ')
   $stderr.puts('    If no argument is provided, default will be used: @json:@stdin')
   $stderr.puts('    -h, --help display this message')
-  $stderr.puts('    <session spec extended value> a dictionary value (Hash)')
+  $stderr.puts('    <session spec extended value> a dictionary (Hash)')
   $stderr.puts('    The value can be either:')
   $stderr.puts("       the JSON description itself, e.g. @json:'{\"xx\":\"yy\",...}'")
   $stderr.puts('       @json:@stdin, if the JSON is provided from stdin')
   $stderr.puts('       @json:@file:<path>, if the JSON is provided from a file')
-  $stderr.puts("    Parameter #{PARAM_SPEC} is mandatory, it contains the transfer spec")
+  $stderr.puts('    The following keys are recognized in session spec:')
+  $stderr.puts("       #{PARAM_SPEC} : mandatory, contains the transfer spec")
+  $stderr.puts("       #{PARAM_LOG_LEVEL} : modify log level (to stderr)")
+  $stderr.puts("       #{PARAM_AGENT} : modify transfer agent parameters, e.g. ascp_args")
+  $stderr.puts("       #{PARAM_TMP_FILE_LIST_FOLDER} : location of temporary files")
+  $stderr.puts("       #{PARAM_SDK} : location of SDK (ascp)")
   $stderr.puts('    Asynchronous commands can be provided on STDIN, examples:')
   $stderr.puts('       {"type":"START","source":"/aspera-test-dir-tiny/200KB.2"}')
   $stderr.puts('       {"type":"START","source":"xx","destination":"yy"}')
   $stderr.puts('       {"type":"DONE"}')
-  $stderr.puts(%Q(Note: debug information can be placed on STDERR, using the "#{PARAM_LOG_LEVEL}" parameter in session spec (debug=0)))
   $stderr.puts('EXAMPLES')
   $stderr.puts(%Q(    asession @json:'{"#{PARAM_SPEC}":{#{SAMPLE_DEMO},#{SAMPLE_DEMO2},"paths":[{"source":"/aspera-test-dir-tiny/200KB.1"}]}}'))
   $stderr.puts(%Q(    echo '{"#{PARAM_SPEC}":{"remote_host":...}}'|asession @json:@stdin))
   Process.exit(0)
 end
-Aspera::Ascp::Installation.instance.sdk_folder = File.join(Dir.home, '.aspera', 'sdk')
-
 parameter_source_err_msg = ' (argument), did you specify: "@json:" ?'
 # by default assume JSON input on stdin if no argument
 if ARGV.empty?
@@ -62,23 +68,24 @@ rescue
   assert_usage(false, "Cannot parse argument: #{session_argument}")
 end
 # ensure right type
-assert_usage(session_spec.is_a?(Hash), "The value must be a hash table#{parameter_source_err_msg}")
-assert_usage(session_spec[PARAM_SPEC].is_a?(Hash), "the value must contain key #{PARAM_SPEC}")
+assert_usage(session_spec.is_a?(Hash), "The value must be a Hash#{parameter_source_err_msg}")
+assert_usage(session_spec[PARAM_SPEC].is_a?(Hash), "The value must contain key #{PARAM_SPEC} with Hash value")
 # additional debug capability
-if session_spec.key?(PARAM_LOG_LEVEL)
-  Aspera::Log.instance.level = session_spec[PARAM_LOG_LEVEL]
-end
+Aspera::Log.instance.level = session_spec[PARAM_LOG_LEVEL] if session_spec.key?(PARAM_LOG_LEVEL)
 # possibly override temp folder
 if session_spec.key?(PARAM_TMP_FILE_LIST_FOLDER)
   Aspera::Transfer::Parameters.file_list_folder = session_spec[PARAM_TMP_FILE_LIST_FOLDER]
 end
-session_spec['agent'] = {} unless session_spec.key?('agent')
-session_spec['agent']['quiet'] = true
-session_spec['agent']['management_cb'] = ->(event) do
+session_spec[PARAM_SDK] = File.join(Dir.home, '.aspera', 'sdk') unless session_spec.key?(PARAM_SDK)
+Aspera::Ascp::Installation.instance.sdk_folder = session_spec[PARAM_SDK]
+session_spec[PARAM_AGENT] = {} unless session_spec.key?(PARAM_AGENT)
+agent_params = session_spec[PARAM_AGENT]
+agent_params['quiet'] = true
+agent_params['management_cb'] = ->(event) do
   puts JSON.generate(Aspera::Ascp::Management.enhanced_event_format(event))
 end
 # get local agent (ascp), disable ascp output on stdout to not mix with JSON events
-client = Aspera::Agent::Direct.new(**session_spec['agent'].symbolize_keys)
+client = Aspera::Agent::Direct.new(**agent_params.symbolize_keys)
 # start transfer (asynchronous)
 job_id = client.start_transfer(session_spec[PARAM_SPEC])
 # async commands

data/examples/build_exec

@@ -0,0 +1,74 @@
+#!/bin/bash
+# Glibc for various OS: https://gist.github.com/wagenet/35adca1a032cec2999d47b6c40aa45b1
+# https://distrowatch.com/table.php?distribution=redhat
+# RHEL 8 : glibc 2.28
+# RHEL 9 : glibc 2.34
+# https://distrowatch.com/table.php?distribution=ubuntu
+# Ubuntu 18.10 cosmic : glibc 2.28
+# Ubuntu 21.10 impish : gblic 2.34
+# https://www.ibm.com/docs/en/ahts/4.4.x?topic=release-notes
+# ascp 4.4.5 : gblic 2.28
+# dwarfs requires boost >= 1.65
+# boost requires cmake >= 3.30
+set -e
+
+if test $# -ne 5; then
+    echo "Usage: $0 <CLI_EXECUTABLE> <CLI_PATH> <GEM_NAME> <GEM_VERSION> <DIR_TMP>" 1>&2
+    exit 1
+fi
+
+CLI_EXECUTABLE=$1
+CLI_PATH=$(realpath $2)
+GEM_NAME=$3
+GEM_VERSION=$4
+DIR_TMP=$(realpath $5)
+
+echo "Building: $(basename $CLI_EXECUTABLE)"
+
+# tebako compilation
+tebako_env=$DIR_TMP/tebako-env
+# project files (gem)
+tebako_root=$DIR_TMP/tebako-root
+# only provide the name, not path
+tebako_entry=$(basename $CLI_PATH)
+tebako_output=$CLI_EXECUTABLE
+
+container_tag=alpine-3.17
+#container_tag=ubuntu-20.04
+
+gem install tebako
+mkdir -p $tebako_env $tebako_root
+(cd $tebako_root && gem fetch $GEM_NAME:$GEM_VERSION)
+case $(uname -s) in
+Darwin)
+    brew update
+    # install tools
+    brew install bash binutils bison flex gnu-sed lz4 pkg-config xz
+    # install libs
+    brew install boost double-conversion fmt gdbm glog jemalloc libevent libffi libsodium libyaml ncurses openssl@3 zlib
+    # make sure tools are reachable
+    export PATH=/opt/homebrew/opt/flex/bin:/opt/homebrew/opt/bison/bin:$PATH
+    # remove binutils from path so that macos `ar` is used
+    export PATH=$(echo $PATH|tr : \\n|grep -v /binutils/|tr \\n :)
+    ;;
+Linux)
+    tebako_root=/mnt/w
+    tebako_prefix="podman run -it --rm -v $PWD:$tebako_root ghcr.io/tamatebako/tebako-${container_tag}:0.8.6"
+    tebako_opts=--patchelf
+    tebako_output=$tebako_root/$(basename $CLI_EXECUTABLE)
+    ;;
+LinuxLocal)
+    dnf module reset ruby
+    dnf module install ruby:3.3
+    dnf install -y cmake git bash sudo autoconf boost-devel flex bison make clang binutils-devel libevent-devel libacl-devel sed python3 pkgconfig curl lz4-devel openssl-devel zlib-devel xz zip unzip tar xz-devel elfutils-devel libffi-devel gdbm-devel ncurses-devel readline-devel ruby-devel gettext-devel brotli-devel clang libxslt-devel
+    echo TODO
+    exit 1
+    dnf install 
+    ;;
+*)
+    echo "This OS is not supported." >&2
+    exit 1
+    ;;
+esac
+set -x
+exec $tebako_prefix tebako press --root=$tebako_root --entry-point=$tebako_entry --output=$tebako_output --prefix=$tebako_env $tebako_opts

data/examples/{rubyc → build_exec_rubyc}

@@ -1,6 +1,22 @@
 #!/bin/bash
+
+set -e
+
+if test $# -ne 5; then
+    echo "Usage: $0 <CLI_EXECUTABLE> <CLI_PATH> <GEM_NAME> <GEM_VERSION> <DIR_TMP>" 1>&2
+    exit 1
+fi
+
+CLI_EXECUTABLE=$1
+CLI_PATH=$(realpath $2)
+GEM_NAME=$3
+GEM_VERSION=$4
+DIR_TMP=$(realpath $5)
+
+echo "Building: $(basename $CLI_EXECUTABLE)"
+
 # https://github.com/you54f/ruby-packer
-# https://github.com/YOU54F/ruby-packer/releases
+# https://github.com/you54f/ruby-packer/releases
 set -e
 FOLDER="$(dirname $0)/../tmp"
 RUBYC="$FOLDER/rubyc"
@@ -21,4 +37,4 @@ if test ! -e "$RUBYC"; then
             ;;
     esac
 fi
-exec "$RUBYC" "$@"
+exec "$RUBYC" -o "$CLI_EXECUTABLE" "$CLI_PATH"

data/examples/get_proto_file.rb

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Retrieve `transfer.proto` from the web
+$LOAD_PATH.unshift(File.join(File.dirname(File.dirname(File.realpath(__FILE__))), 'lib'))
+require 'aspera/ascp/installation'
+Aspera::Ascp::Installation.instance.install_sdk(folder: ARGV.first, backup: false, with_exe: false) {|name| '/' if name.end_with?('transfer.proto')}

data/lib/aspera/agent/alpha.rb

@@ -82,29 +82,29 @@ module Aspera
             transfer = @client_app_api.get_transfer(app_id: @application_id, transfer_id: @xfer_id)
             case transfer['status']
             when 'initiating', 'queued'
-              notify_progress(session_id: nil, type: :pre_start, info: transfer['status'])
+              notify_progress(:pre_start, session_id: nil, info: transfer['status'])
             when 'running'
               if !started
-                notify_progress(session_id: @xfer_id, type: :session_start)
+                notify_progress(:session_start, session_id: @xfer_id)
                 started = true
               end
               if !pre_calc && (transfer['bytes_expected'] != 0)
-                notify_progress(type: :session_size, session_id: @xfer_id, info: transfer['bytes_expected'])
+                notify_progress(:session_size, session_id: @xfer_id, info: transfer['bytes_expected'])
                 pre_calc = true
               else
-                notify_progress(type: :transfer, session_id: @xfer_id, info: transfer['bytes_written'])
+                notify_progress(:transfer, session_id: @xfer_id, info: transfer['bytes_written'])
               end
             when 'completed'
-              notify_progress(type: :end, session_id: @xfer_id)
+              notify_progress(:end, session_id: @xfer_id)
               break
             when 'failed'
-              notify_progress(type: :end, session_id: @xfer_id)
+              notify_progress(:end, session_id: @xfer_id)
               raise Transfer::Error, transfer['error_desc']
             when 'cancelled'
-              notify_progress(type: :end, session_id: @xfer_id)
+              notify_progress(:end, session_id: @xfer_id)
               raise Transfer::Error, 'Transfer cancelled by user'
             else
-              notify_progress(type: :end, session_id: @xfer_id)
+              notify_progress(:end, session_id: @xfer_id)
               raise Transfer::Error, "unknown status: #{transfer['status']}: #{transfer['error_desc']}"
             end
             sleep(1)

data/lib/aspera/agent/base.rb

@@ -14,22 +14,6 @@ module Aspera
           Aspera::Agent.const_get(agent.to_s.capitalize).new(**options)
         end
 
-        # compute options from user provided and default options
-        def to_move_options(default:, options:)
-          result = options.symbolize_keys
-          available = default.map{|k, v|"#{k}(#{v})"}.join(', ')
-          result.each_key do |k|
-            Aspera.assert_values(k, default.keys){"transfer agent parameter: #{k}"}
-            # check it is the expected type: too limiting, as we can have an Integer or Float, or symbol and string
-            # raise "Invalid value for transfer agent parameter: #{k}, expect #{default[k].class.name}" unless default[k].nil? || v.is_a?(default[k].class)
-          end
-          default.each do |k, v|
-            raise "Missing required agent parameter: #{k}. Parameters: #{available}" if v.eql?(:required) && !result.key?(k)
-            result[k] = v unless result.key?(k)
-          end
-          return result
-        end
-
         # discover available agents
         def agent_list
           base_class = File.basename(__FILE__)
@@ -38,6 +22,8 @@ module Aspera
           end.map{|file|file[0..(-1 - RUBY_EXT.length)].to_sym}
         end
       end
+
+      # Wait for all sessions to terminate and return the status of each session
       def wait_for_completion
         # list of: :success or "error message string"
         statuses = wait_for_transfers_completion
@@ -56,8 +42,8 @@ module Aspera
         @progress = progress
       end
 
-      def notify_progress(**parameters)
-        @progress&.event(**parameters)
+      def notify_progress(*pos_args, **kw_args)
+        @progress&.event(*pos_args, **kw_args)
       end
     end
   end

data/lib/aspera/agent/connect.rb

@@ -24,11 +24,12 @@ module Aspera
           Log.log.debug{"found: #{connect_url}"}
           @connect_api = Rest.new(
             base_url: "#{connect_url}/v5/connect", # could use v6 also now
-            headers: {'Origin' => Rest.user_agent})
-          connect_info = @connect_api.read('info/version')[:data]
+            headers: {'Origin' => RestParameters.instance.user_agent})
+          connect_info = @connect_api.read('info/version')
           Log.log.info('Connect was reached') if method_index > 0
           Log.log.debug{Log.dump(:connect_version, connect_info)}
         rescue StandardError => e # Errno::ECONNREFUSED
+          Log.log.debug{"Exception: #{e}"}
           start_url = CONNECT_START_URIS[method_index]
           method_index += 1
           raise StandardError, "Unable to start connect #{method_index} times" if start_url.nil?
@@ -51,7 +52,7 @@ module Aspera
             'title'                   => 'Select Files',
             'suggestedName'           => '',
             'allowMultipleSelection'  => true,
-            'allowedFileTypes'        => ''})[:data]
+            'allowedFileTypes'        => ''})
           transfer_spec['paths'] = selection['dataTransfer']['files'].map { |i| {'source' => i['name']}}
         end
         @request_id = SecureRandom.uuid
@@ -67,7 +68,7 @@ module Aspera
             'transfer_spec' => transfer_spec
           }]}
         # asynchronous anyway
-        res = @connect_api.create('transfers/start', connect_transfer_args)[:data]
+        res = @connect_api.create('transfers/start', connect_transfer_args)
         @xfer_id = res['transfer_specs'].first['transfer_spec']['tags'][Transfer::Spec::TAG_RESERVED]['xfer_id']
       end
 
@@ -78,7 +79,7 @@ module Aspera
         session_id = @xfer_id
         begin
           loop do
-            tr_info = @connect_api.create("transfers/info/#{@xfer_id}", connect_activity_args)[:data]
+            tr_info = @connect_api.create("transfers/info/#{@xfer_id}", connect_activity_args)
             Log.log.trace1{Log.dump(:tr_info, tr_info)}
             if tr_info['transfer_info'].is_a?(Hash)
               transfer = tr_info['transfer_info']
@@ -89,29 +90,29 @@ module Aspera
               # TODO: get session id
               case transfer['status']
               when 'initiating', 'queued'
-                notify_progress(session_id: nil, type: :pre_start, info: transfer['status'])
+                notify_progress(:pre_start, session_id: nil, info: transfer['status'])
               when 'running'
                 if !started
-                  notify_progress(session_id: session_id, type: :session_start)
+                  notify_progress(:session_start, session_id: session_id)
                   started = true
                 end
                 if !pre_calc && (transfer['bytes_expected'] != 0)
-                  notify_progress(type: :session_size, session_id: session_id, info: transfer['bytes_expected'])
+                  notify_progress(:session_size, session_id: session_id, info: transfer['bytes_expected'])
                   pre_calc = true
                 else
-                  notify_progress(type: :transfer, session_id: session_id, info: transfer['bytes_written'])
+                  notify_progress(:transfer, session_id: session_id, info: transfer['bytes_written'])
                 end
               when 'completed'
-                notify_progress(type: :end, session_id: session_id)
+                notify_progress(:end, session_id: session_id)
                 break
               when 'failed'
-                notify_progress(type: :end, session_id: session_id)
+                notify_progress(:end, session_id: session_id)
                 raise Transfer::Error, transfer['error_desc']
               when 'cancelled'
-                notify_progress(type: :end, session_id: session_id)
+                notify_progress(:end, session_id: session_id)
                 raise Transfer::Error, 'Transfer cancelled by user'
               else
-                notify_progress(type: :end, session_id: session_id)
+                notify_progress(:end, session_id: session_id)
                 raise Transfer::Error, "unknown status: #{transfer['status']}: #{transfer['error_desc']}"
               end
             end