aspera-cli 4.7.0 → 4.8.0

data/bin/ascli

@@ -1,7 +1,26 @@
 #!/usr/bin/env ruby
 # frozen_string_literal: true
+
 require 'rubygems'
-$LOAD_PATH.unshift(File.realpath(File.dirname(File.realpath(__FILE__))+'/../lib'))
+require 'securerandom'
+GEM_ROOT=File.realpath(File.join(File.dirname(File.realpath(__FILE__)),'..'))
+# coverage for tests
+if ENV.has_key?('ENABLE_COVERAGE')
+  require 'simplecov'
+  SimpleCov.root(GEM_ROOT)
+  SimpleCov.enable_for_subprocesses if SimpleCov.respond_to?(:enable_for_subprocesses)
+  # keep cache data for 1 day (must be longer that time to run the whole test suite)
+  SimpleCov.merge_timeout(86400)
+  SimpleCov.command_name(SecureRandom.uuid)
+  SimpleCov.at_exit do
+    original_file_descriptor = $stdout
+    $stdout.reopen(File.join(GEM_ROOT,'simplecov.log'))
+    SimpleCov.result.format!
+    $stdout.reopen(original_file_descriptor)
+  end
+  SimpleCov.start
+end
+$LOAD_PATH.unshift(File.join(GEM_ROOT,'lib'))
 require 'aspera/cli/main'
 require 'aspera/environment'
 Encoding.default_internal = Encoding::UTF_8

data/bin/asession

@@ -1,82 +1,85 @@
 #!/usr/bin/env ruby
 # frozen_string_literal: true
+
 # Laurent Martin/2017
-$LOAD_PATH.unshift(File.dirname(__FILE__)+"/../lib")
+$LOAD_PATH.unshift(File.dirname(__FILE__) + '/../lib')
 require 'aspera/fasp/agent_direct'
 require 'aspera/cli/listener/line_dump'
 require 'aspera/cli/extended_value'
 require 'aspera/log'
 require 'json'
 # extended transfer spec parameter to change log level
-TS_LOGLEVEL='EX_loglevel'
+TS_LOGLEVEL = 'EX_loglevel'
 # by default go to /tmp/username.filelist
-TS_TMP_FILELIST_FOLDER='EX_file_list_folder'
+TS_TMP_FILELIST_FOLDER = 'EX_file_list_folder'
 
+SAMPLE_DEMO='"remote_host":"demo.asperasoft.com","remote_user":"asperaweb","ssh_port":33001,"remote_password":"demoaspera"'
+SAMPLE_DEMO2='"direction":"receive","destination_root":"./test.dir"'
 def assert_usage(assertion,errmsg)
   return if assertion
-  STDERR.puts('ERROR: '.red.blink+errmsg) if errmsg
-  STDERR.puts('USAGE')
-  STDERR.puts('    asession')
-  STDERR.puts('    asession -h|--help')
-  STDERR.puts('    asession <transfer spec extended value>')
-  STDERR.puts('    ')
-  STDERR.puts('    If no argument is provided, default will be used: @json:@stdin')
-  STDERR.puts('    -h, --help display this message')
-  STDERR.puts('    <transfer spec extended value> a JSON value for transfer_spec, using the prefix: @json:')
-  STDERR.puts('    The value can be either:')
-  STDERR.puts("       the JSON description itself, e.g. @json:'{\"xx\":\"yy\",...}'")
-  STDERR.puts('       @json:@stdin, if the JSON is provided from stdin')
-  STDERR.puts('       @json:@file:<path>, if the JSON is provided from a file')
-  STDERR.puts('    Asynchronous commands can be provided on STDIN, examples:')
-  STDERR.puts('       {"type":"START","source":"/aspera-test-dir-tiny/200KB.2"}')
-  STDERR.puts('       {"type":"START","source":"xx","destination":"yy"}')
-  STDERR.puts('       {"type":"DONE"}')
-  STDERR.puts('Note: debug information can be placed on STDERR, using the "'+TS_LOGLEVEL+'" parameter in transfer spec (debug=0)')
-  STDERR.puts('EXAMPLES')
-  STDERR.puts(%q(    asession @json:'{"remote_host":"demo.asperasoft.com","remote_user":"asperaweb","ssh_port":33001,"remote_password":"demoaspera","direction":"receive","destination_root":"./test.dir","paths":[{"source":"/aspera-test-dir-tiny/200KB.1"}]}'))
-  STDERR.puts(%q(    echo '{"remote_host":...}'|asession @json:@stdin))
+  $stderr.puts('ERROR: '.red.blink + errmsg) if errmsg
+  $stderr.puts('USAGE')
+  $stderr.puts('    asession')
+  $stderr.puts('    asession -h|--help')
+  $stderr.puts('    asession <transfer spec extended value>')
+  $stderr.puts('    ')
+  $stderr.puts('    If no argument is provided, default will be used: @json:@stdin')
+  $stderr.puts('    -h, --help display this message')
+  $stderr.puts('    <transfer spec extended value> a JSON value for transfer_spec, using the prefix: @json:')
+  $stderr.puts('    The value can be either:')
+  $stderr.puts("       the JSON description itself, e.g. @json:'{\"xx\":\"yy\",...}'")
+  $stderr.puts('       @json:@stdin, if the JSON is provided from stdin')
+  $stderr.puts('       @json:@file:<path>, if the JSON is provided from a file')
+  $stderr.puts('    Asynchronous commands can be provided on STDIN, examples:')
+  $stderr.puts('       {"type":"START","source":"/aspera-test-dir-tiny/200KB.2"}')
+  $stderr.puts('       {"type":"START","source":"xx","destination":"yy"}')
+  $stderr.puts('       {"type":"DONE"}')
+  $stderr.puts(%Q(Note: debug information can be placed on STDERR, using the "#{TS_LOGLEVEL}" parameter in transfer spec (debug=0)))
+  $stderr.puts('EXAMPLES')
+  $stderr.puts(%Q(    asession @json:'{#{SAMPLE_DEMO},#{SAMPLE_DEMO2},"paths":[{"source":"/aspera-test-dir-tiny/200KB.1"}]}'))
+  $stderr.puts(%q(    echo '{"remote_host":...}'|asession @json:@stdin))
   Process.exit(1)
 end
 
-parameter_source_err_msg=' (argument), did you specify: "@json:" ?'
+parameter_source_err_msg = ' (argument), did you specify: "@json:" ?'
 # by default assume JSON input on stdin if no argument
 if ARGV.empty?
   ARGV.push('@json:@stdin')
-  parameter_source_err_msg=' (JSON on stdin)'
+  parameter_source_err_msg = ' (JSON on stdin)'
 end
 # anyway expect only one argument: session information
 assert_usage(ARGV.length.eql?(1),'exactly one argument is expected')
 assert_usage(!['-h','--help'].include?(ARGV.first),nil)
 # parse transfer spec
 begin
-  transfer_spec_arg=ARGV.pop
-  transfer_spec=Aspera::Cli::ExtendedValue.instance.evaluate(transfer_spec_arg)
-rescue => e
+  transfer_spec_arg = ARGV.pop
+  transfer_spec = Aspera::Cli::ExtendedValue.instance.evaluate(transfer_spec_arg)
+rescue
   assert_usage(false,"Cannot extract transfer spec from: #{transfer_spec_arg}")
 end
 # ensure right type
 assert_usage(transfer_spec.is_a?(Hash),"the value must be a hash table#{parameter_source_err_msg}")
 # additional debug capability
 if transfer_spec.has_key?(TS_LOGLEVEL)
-  Aspera::Log.instance.level=transfer_spec[TS_LOGLEVEL]
+  Aspera::Log.instance.level = transfer_spec[TS_LOGLEVEL]
   transfer_spec.delete(TS_LOGLEVEL)
 end
 # possibly override temp folder
 if transfer_spec.has_key?(TS_TMP_FILELIST_FOLDER)
-  Aspera::Fasp::Parameters.file_list_folder=transfer_spec[TS_TMP_FILELIST_FOLDER]
+  Aspera::Fasp::Parameters.file_list_folder = transfer_spec[TS_TMP_FILELIST_FOLDER]
   transfer_spec.delete(TS_TMP_FILELIST_FOLDER)
 end
 # get local agent (ascp), disable ascp output on stdout to not mix with JSON events
-client=Aspera::Fasp::AgentDirect.new({quiet: true})
+client = Aspera::Fasp::AgentDirect.new({quiet: true})
 # display JSON instead of legacy Lines
 client.add_listener(Aspera::Cli::Listener::LineDump.new)
 # start transfer (asynchronous)
-job_id=client.start_transfer(transfer_spec)
+job_id = client.start_transfer(transfer_spec)
 # async commands
 Thread.new do
   begin
     loop do
-      data=JSON.parse(STDIN.gets)
+      data = JSON.parse($stdin.gets)
       client.send_command(job_id,0,data)
     end
   rescue

data/docs/test_env.conf

@@ -19,6 +19,7 @@ default:
 cli_default:
   interactive: your value here
   smtp: your value here
+  secrets: your value here
 local_user:
   ssh_keys: your value here
 smtp_config:
@@ -60,12 +61,12 @@ tst_faspex5_web:
   client_secret: your value here
 tst_faspex5:
   url: your value here
+  insecure: your value here
   auth: your value here
   client_id: your value here
   client_secret: your value here
   private_key: your value here
   username: your value here
-  password: your value here
 tst_shares:
   url: your value here
   username: your value here
@@ -97,6 +98,7 @@ tst_orch:
   url: your value here
   username: your value here
   password: your value here
+  insecure: your value here
 tst_ats:
   ibm_api_key: your value here
   ats_key: your value here
@@ -106,6 +108,7 @@ tst_bss:
   password: your value here
 tst_ak_preview:
   url: your value here
+  insecure: your value here
   username: your value here
   password: your value here
   mimemagic: your value here
@@ -113,6 +116,7 @@ tst_node_preview:
   url: your value here
   username: your value here
   password: your value here
+  insecure: your value here
 tst_cos:
   apikey: your value here
   crn: your value here
@@ -126,6 +130,8 @@ misc:
   faspex_publink_send_to_dropbox: your value here
   faspex_dbx: your value here
   faspex_wkg: your value here
+  faspex_src: your value here
+  faspex5_meta: your value here
   shares_upload: your value here
   console_smart_id: your value here
   console_smart_file: your value here
@@ -146,8 +152,8 @@ misc:
   aoc_shbx_ws: your value here
   aoc_shbx_name: your value here
   aoc_shbx_meta: your value here
-  aoc_node1_name: your value here
-  aoc_node1_secret: your value here
+  aoc_ak_name: your value here
+  aoc_ak_secret: your value here
   icos_bucket_key: your value here
   icos_bucket_secret: your value here
   icos_bucket_name: your value here
@@ -161,3 +167,5 @@ misc:
   aoc_user_email: your value here
   aoc_workspace2: your value here
   http_gw_fqdn_port: your value here
+tst_secrets:
+  st1: your value here

data/examples/aoc.rb

@@ -1,9 +1,10 @@
 #!/usr/bin/env ruby
 # frozen_string_literal: true
+
 require 'aspera/aoc'
 require 'aspera/log'
 
-Aspera::Log.instance.level=:debug
+Aspera::Log.instance.level = :debug
 
 if !ARGV.length.eql?(3)
   Aspera::Log.log.error("wrong number of args: #{ARGV.length}")
@@ -12,18 +13,18 @@ if !ARGV.length.eql?(3)
   Process.exit(1)
 end
 
-aoc_url=ARGV[0]
-aoc_user=ARGV[1]
-aoc_key_value=ARGV[2]
+aoc_url = ARGV[0]
+aoc_user = ARGV[1]
+aoc_key_value = ARGV[2]
 
-aocapi=Aspera::AoC.new(
-url: aoc_url,
-auth: :jwt,
-private_key: aoc_key_value,
-username: aoc_user,
-scope: 'user:all',
-subpath: 'api/v1')
+aocapi = Aspera::AoC.new(
+  url: aoc_url,
+  auth: :jwt,
+  private_key: aoc_key_value,
+  username: aoc_user,
+  scope: 'user:all',
+  subpath: 'api/v1')
 
-self_user_data=aocapi.read('self')
+self_user_data = aocapi.read('self')
 
 Aspera::Log.dump('self',self_user_data)

data/examples/dascli

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+: ${image:=martinlaurent/ascli}
+# by default take latest version
+: ${version:=latest}
+imgtag=$image:$version
+# same location as in Dockerfile: generic top folder for apps
+appdir=/usr/src/app
+# same location as in Dockerfile: main config folder for ascli in container
+ascli_home_container=${appdir}/config
+# convenience: special argument to install the image
+case "$1" in install) docker pull $imgtag; exit 0; esac
+# set default location for config folder on host if necessary
+: ${ASCLI_HOME:=$HOME/.aspera/ascli}
+if test ! -d $ASCLI_HOME;then
+	echo "creating foder: $ASCLI_HOME"
+	# create it if necessary to allow mounting the volume in container
+	mkdir -p "$ASCLI_HOME"
+fi
+exec docker run \
+--rm \
+--tty \
+--interactive \
+--env ASCLI_HOME="$ascli_home_container" \
+--volume "$ASCLI_HOME:$ascli_home_container" \
+$imgtag \
+ascli "$@"

data/examples/faspex4.rb

@@ -1,5 +1,6 @@
 #!/usr/bin/env ruby
 # frozen_string_literal: true
+
 # find Faspex API here: https://developer.ibm.com/apis/catalog/?search=faspex
 # this example makes use of class Aspera::Rest for REST calls, alternatively class RestClient of gem rest-client could be used
 # this example makes use of class Aspera::Fasp::AgentDirect for transfers, alternatively the official "Transfer SDK" could be used
@@ -8,10 +9,10 @@ require 'aspera/rest'
 require 'aspera/log'
 require 'aspera/fasp/agent_direct'
 
-tmpdir=ENV['tmp']||Dir.tmpdir || '.'
+tmpdir = ENV['tmp'] || Dir.tmpdir || '.'
 
 # Set high log level for the example, decrease to :warn usually
-Aspera::Log.instance.level=:debug
+Aspera::Log.instance.level = :debug
 
 # Set folder where SDK is installed (mandatory)
 # (if ascp is not there, the lib will try to find in usual locations)
@@ -25,23 +26,23 @@ if !ARGV.length.eql?(3)
   Process.exit(1)
 end
 
-faspex_url=ARGV[0] # typically: https://faspex.example.com/aspera/faspex
-faspex_user=ARGV[1]
-faspex_pass=ARGV[2]
+faspex_url = ARGV[0] # typically: https://faspex.example.com/aspera/faspex
+faspex_user = ARGV[1]
+faspex_pass = ARGV[2]
 
-# comment out this if certificate is valid, keep line to ignore certificate
-Aspera::Rest.insecure=true
+# uncomment this if certificate is self signed
+# Rest.session_cb = lambda{|http|http.verify_mode = OpenSSL::SSL::VERIFY_NONE}
 
 # 1: Faspex 4 API v3
 #---------------
 
 # create REST API object
-api_v3=Aspera::Rest.new({
+api_v3 = Aspera::Rest.new({
   base_url: faspex_url,
-  auth: {
-  type:     :basic,
-  username: faspex_user,
-  password: faspex_pass
+  auth:     {
+    type:     :basic,
+    username: faspex_user,
+    password: faspex_pass
   }})
 
 # very simple api call
@@ -51,22 +52,26 @@ api_v3.read('me')
 #---------------
 
 # create a sample file to send
-file_to_send=File.join(tmpdir,'myfile.bin')
+file_to_send = File.join(tmpdir,'myfile.bin')
 File.write(file_to_send, 'sample data')
 # package creation parameters
-package_create_params={'delivery'=>{'title'=>'test package','recipients'=>['aspera.user1@gmail.com'],'sources'=>[{'paths'=>[file_to_send]}]}}
-pkg_created=api_v3.create('send',package_create_params)[:data]
+package_create_params = {'delivery' => {
+  'title'      => 'test package',
+  'recipients' => ['aspera.user1@gmail.com'],
+  'sources'    => [{'paths' => [file_to_send]}]
+}}
+pkg_created = api_v3.create('send',package_create_params)[:data]
 # get transfer specification (normally: only one)
-transfer_spec=pkg_created['xfer_sessions'].first
+transfer_spec = pkg_created['xfer_sessions'].first
 # set paths of files to send
-transfer_spec['paths']=[{'source'=>file_to_send}]
+transfer_spec['paths'] = [{'source' => file_to_send}]
 # get local agent (ascp), disable ascp output on stdout to not mix with JSON events
-transfer_client=Aspera::Fasp::AgentDirect.new({quiet: true})
+transfer_client = Aspera::Fasp::AgentDirect.new({quiet: true})
 # start transfer (asynchronous)
-job_id=transfer_client.start_transfer(transfer_spec)
+job_id = transfer_client.start_transfer(transfer_spec)
 Aspera::Log.dump('job_id',job_id)
 # wait for all transfer completion (for the example)
-result=transfer_client.wait_for_transfers_completion
+result = transfer_client.wait_for_transfers_completion
 #  notify of any transfer error
 result.reject{|i|i.eql?(:success)}.each do |e|
   Aspera::Log.log.error("A transfer error occured: #{e.message}")
@@ -74,16 +79,16 @@ end
 
 # 3: Faspex 4 API v4
 #---------------
-api_v4=Aspera::Rest.new({
-  base_url: faspex_url+'/api',
+api_v4 = Aspera::Rest.new({
+  base_url: faspex_url + '/api',
   auth:     {
-    type:         :oauth2,
-    base_url:     faspex_url+'/auth/oauth2',
-    auth:         {type: :basic, username: faspex_user, password: faspex_pass},
-    crtype:       :generic,
-    generic:       {grant_type: 'password'},
-    scope:        'admin'
-    }})
+    type:     :oauth2,
+    base_url: faspex_url + '/auth/oauth2',
+    auth:     {type: :basic, username: faspex_user, password: faspex_pass},
+    crtype:   :generic,
+    generic:  {grant_type: 'password'},
+    scope:    'admin'
+  }})
 
 # Use it. Note that Faspex 4 API v4 is totally different from Faspex 4 v3 APIs, see ref on line 2
 Aspera::Log.dump('users',api_v4.read('users')[:data])

data/examples/transfer.rb

@@ -1,5 +1,6 @@
 #!/usr/bin/env ruby
 # frozen_string_literal: true
+
 # Example: transfer a file using one of the provided transfer agents
 # location of ascp can be specified with env var "ascp"
 # temp folder can be specified with env var "tmp"
@@ -12,19 +13,19 @@ require 'aspera/rest_errors_aspera'
 require 'json'
 require 'tmpdir'
 
-tmpdir=ENV['tmp']||Dir.tmpdir || '.'
+tmpdir = ENV['tmp'] || Dir.tmpdir || '.'
 
-DEMO_CONFIG=[
+DEMO_CONFIG = [
   'ssh://asperaweb@eudemo.asperademo.com:33001',
   'https://node_asperaweb@eudemo.asperademo.com:9092',
   'demoaspera'
-]
+].freeze
 
 ##############################################################
 # generic initialisation : configuration of FaspManager
 
 # set trace level for sample, set to :debug to see complete list of debug information
-Aspera::Log.instance.level=:debug
+Aspera::Log.instance.level = :debug
 
 # register aspera REST call error handlers
 Aspera::RestErrorsAspera.register_handlers
@@ -32,7 +33,7 @@ Aspera::RestErrorsAspera.register_handlers
 # some required files are generated here (keys, certs)
 Aspera::Fasp::Installation.instance.folder = tmpdir
 # set path to your copy of ascp binary (else, let the system find)
-Aspera::Fasp::Installation.instance.ascp_path=ENV['ascp'] if ENV.has_key?('ascp')
+Aspera::Fasp::Installation.instance.ascp_path = ENV['ascp'] if ENV.has_key?('ascp')
 # another way is to detect installed products and use one of them
 #Aspera::Fasp::Installation.instance.installed_products.each{|p|puts("found: #{p[:name]}")}
 #Aspera::Fasp::Installation.instance.use_ascp_from_product('Aspera Connect')
@@ -40,7 +41,7 @@ Aspera::Fasp::Installation.instance.ascp_path=ENV['ascp'] if ENV.has_key?('ascp'
 #
 
 # get FASP Manager singleton based on above ascp location
-fasp_manager=Aspera::Fasp::AgentDirect.new
+fasp_manager = Aspera::Fasp::AgentDirect.new
 
 # Note that it would also be possible to start transfers using other agents
 #require 'aspera/fasp/connect'
@@ -65,15 +66,15 @@ fasp_manager.add_listener(MyListener.new)
 # first example: download by SSH credentials
 
 # manually build teansfer spec
-transfer_spec={
+transfer_spec = {
   #'remote_host'     =>'demo.asperasoft.com',
-  'remote_host'     =>URI.parse(DEMO_CONFIG[0]).host,
-  'ssh_port'        =>URI.parse(DEMO_CONFIG[0]).port,
-  'remote_user'     =>URI.parse(DEMO_CONFIG[0]).user,
-  'remote_password' =>DEMO_CONFIG[2],
-  'direction'       =>'receive',
-  'destination_root'=>tmpdir,
-  'paths'           =>[{'source'=>'aspera-test-dir-tiny/200KB.1'}]
+  'remote_host'      => URI.parse(DEMO_CONFIG[0]).host,
+  'ssh_port'         => URI.parse(DEMO_CONFIG[0]).port,
+  'remote_user'      => URI.parse(DEMO_CONFIG[0]).user,
+  'remote_password'  => DEMO_CONFIG[2],
+  'direction'        => 'receive',
+  'destination_root' => tmpdir,
+  'paths'            => [{'source' => 'aspera-test-dir-tiny/200KB.1'}]
 }
 # start transfer in separate thread
 # method returns as soon as transfer thread is created
@@ -84,10 +85,10 @@ fasp_manager.start_transfer(transfer_spec)
 # here we started a single transfer session (no multisession parameter)
 # get array of status, one for each session (so, a single value array)
 # each status is either :success or "error message"
-transfer_result=fasp_manager.wait_for_transfers_completion
+transfer_result = fasp_manager.wait_for_transfers_completion
 $stdout.puts(JSON.generate(transfer_result))
 # get list of errors only
-errors=transfer_result.reject{|i|i.eql?(:success)}
+errors = transfer_result.reject{|i|i.eql?(:success)}
 # the transfer was not success, as there is at least one error
 raise "Error(s) occured: #{errors.join(',')}" if !errors.empty?
 
@@ -95,30 +96,30 @@ raise "Error(s) occured: #{errors.join(',')}" if !errors.empty?
 # second example: upload with node authorization
 
 # create rest client for Node API on a public demo system, using public demo credentials
-node_api=Aspera::Rest.new({
+node_api = Aspera::Rest.new({
   base_url: DEMO_CONFIG[1],
-  auth: {
-  type:     :basic,
-  username: URI.parse(DEMO_CONFIG[1]).user,
-  password: DEMO_CONFIG[2]
+  auth:     {
+    type:     :basic,
+    username: URI.parse(DEMO_CONFIG[1]).user,
+    password: DEMO_CONFIG[2]
   }})
 # define sample file(s) and destination folder
-sources=["#{tmpdir}/sample_file.txt"]
-destination='/Upload'
+sources = ["#{tmpdir}/sample_file.txt"]
+destination = '/Upload'
 # create sample file(s)
 sources.each{|p|File.write(p,'Hello World!')}
 # request transfer authorization to node for a single transfer (This is a node api v3 call)
-send_result=node_api.create('files/upload_setup',{ transfer_requests: [{ transfer_request: { paths: [{ destination: destination }] } }] })[:data]
+send_result = node_api.create('files/upload_setup',{ transfer_requests: [{ transfer_request: { paths: [{ destination: destination }] } }] })[:data]
 # we normally have only one transfer spec in list, so just get the first transfer_spec
-transfer_spec=send_result['transfer_specs'].first['transfer_spec']
+transfer_spec = send_result['transfer_specs'].first['transfer_spec']
 # add list of files to upload
-transfer_spec['paths']=sources.map{|p|{'source'=>p}}
+transfer_spec['paths'] = sources.map{|p|{'source' => p}}
 # set authentication type to "token" (will trigger use of bypass SSH key)
-transfer_spec['authentication']='token'
+transfer_spec['authentication'] = 'token'
 # from here : same as example 1
 fasp_manager.start_transfer(transfer_spec)
 # optional: wait for transfer completion helper function to get events
-transfer_result=fasp_manager.wait_for_transfers_completion
-errors=transfer_result.reject{|i|i.eql?(:success)}
+transfer_result = fasp_manager.wait_for_transfers_completion
+errors = transfer_result.reject{|i|i.eql?(:success)}
 # the transfer was not success, as there is at least one error
 raise "Error(s) occured: #{errors.join(',')}" if !errors.empty?