aspera-cli 4.1.0 → 4.2.0

data/lib/aspera/cli/plugins/faspex.rb

@@ -72,7 +72,7 @@ module Aspera
         def self.get_source_id(source_list,source_name)
           source_ids=source_list.select { |i| i['name'].eql?(source_name) }
           if source_ids.empty?
-            raise CliError,"No such Faspex source #{field_sym.to_s}: #{field_value} in [#{source_list.map{|i| i[field_sym.to_s]}.join(', ')}]"
+            raise CliError,%Q{No such Faspex source "#{source_name}" in [#{source_list.map{|i| %Q{"#{i['name']}"}}.join(', ')}]}
           end
           return source_ids.first['id']
         end

data/lib/aspera/cli/plugins/faspex5.rb

@@ -46,39 +46,22 @@ module Aspera
             })
           when :jwt
             #raise "JWT to be implemented"
+            app_client_id=options.get_option(:client_id,:mandatory)
             @api_v5=Rest.new({
-              :base_url => faxpex5_api_base_url,
+              :base_url => faxpex5_api_v5_url,
               :auth     => {
               :type                => :oauth2,
               :base_url            => faxpex5_api_auth_url,
               :grant               => :jwt,
-              :client_id           => options.get_option(:client_id,:mandatory),
+              :client_id           => app_client_id,
               :client_secret       => options.get_option(:client_secret,:mandatory),
               #:redirect_uri   => options.get_option(:redirect_uri,:mandatory),
-              :jwt_subject         => "client:#{options.get_option(:client_id,:mandatory)}", # TODO Mmmm
+              :jwt_subject         => "client:#{app_client_id}", # TODO Mmmm
+              :jwt_audience        => app_client_id, # TODO Mmmm
               :jwt_private_key_obj => OpenSSL::PKey::RSA.new(options.get_option(:private_key,:mandatory)),
-              :jwt_audience =>options.get_option(:client_id,:mandatory), # TODO Mmmm
-              #:token_field    =>'auth_token',
-              #:path_token     => 'authenticate',
-              #:path_authorize => :unused,
-              #:userpass_body  => {name: options.get_option(:username,:mandatory),password: options.get_option(:password,:mandatory)}
+              :jwt_is_f5           => true,
+              :jwt_headers         => {typ: 'JWT'}
               }})
-            #  former version
-            #            # get parameters
-            #            faxpex5_username=options.get_option(:username,:mandatory)
-            #            faxpex5_password=options.get_option(:password,:mandatory)
-            #            # create object for REST calls to Shares2
-            #            @api_v5=Rest.new({
-            #              :base_url => faxpex5_api_base_url,
-            #              :auth     => {
-            #              :type           => :oauth2,
-            #              :base_url       => faxpex5_api_base_url,
-            #              :grant          => :body_data,
-            #              :token_field    =>'auth_token',
-            #              :path_token     => 'authenticate',
-            #              :path_authorize => :unused,
-            #              :userpass_body  => {name: faxpex5_username,password: faxpex5_password}
-            #              }})
           end
         end
 

data/lib/aspera/cli/plugins/preview.rb

@@ -4,6 +4,7 @@ require 'aspera/preview/options'
 require 'aspera/preview/utils'
 require 'aspera/preview/file_types'
 require 'aspera/persistency_action_once'
+require 'aspera/node'
 require 'aspera/hash_ext'
 require 'date'
 require 'securerandom'
@@ -322,12 +323,14 @@ module Aspera
         end # generate_preview
 
         # scan all files in provided folder entry
+        # @param scan_start subpath to start folder scan inside
         def scan_folder_files(top_entry,scan_start=nil)
           if !scan_start.nil?
             # canonical path: start with / and ends with /
             scan_start='/'+scan_start.split('/').select{|i|!i.empty?}.join('/')
             scan_start="#{scan_start}/" #unless scan_start.end_with?('/')
           end
+          filter_block=Aspera::Node.file_matcher(options.get_option(:value,:optional))
           Log.log.debug("scan: #{top_entry} : #{scan_start}".green)
           # don't use recursive call, use list instead
           entries_to_process=[top_entry]
@@ -343,7 +346,11 @@ module Aspera
             Log.log.debug("item:#{entry}")
             case entry['type']
             when 'file'
-              generate_preview(entry)
+              if filter_block.call(entry)
+                generate_preview(entry)
+              else
+                Log.log.debug('skip by filter')
+              end
             when 'link'
               Log.log.debug('Ignoring link.')
             when 'folder'
@@ -438,6 +445,8 @@ module Aspera
               data: iteration_data,
               ids: ["preview_iteration_#{command}",self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory)])
             end
+
+            # call method specified
             iteration_data[0]=send("process_#{command}",iteration_data[0])
             iteration_persistency.save unless iteration_persistency.nil?
             return Main.result_status("#{command} finished")

data/lib/aspera/cli/transfer_agent.rb

@@ -10,7 +10,7 @@ module Aspera
   module Cli
     # The Transfer agent is a common interface to start a transfer using
     # one of the supported transfer agents
-    # provides CLI options to select one of the transfer agents (fasp client)
+    # provides CLI options to select one of the transfer agents (FASP/ascp client)
     class TransferAgent
       # special value for --sources : read file list from arguments
       FILE_LIST_FROM_ARGS='@args'
@@ -45,6 +45,7 @@ module Aspera
 
       def option_transfer_spec; @transfer_spec_cmdline; end
 
+      # multiple option are merged
       def option_transfer_spec=(value); @transfer_spec_cmdline.merge!(value); end
 
       def option_transfer_spec_deep_merge(ts); @transfer_spec_cmdline.deep_merge!(ts); end
@@ -75,7 +76,7 @@ module Aspera
         when :connect
           new_agent=Fasp::Connect.new
         when :node
-          # way for code to setup alternate node api in avance
+          # way for code to setup alternate node api in advance
           # support: @preset:<name>
           # support extended values
           node_config=@opt_mgr.get_option(:transfer_info,:optional)
@@ -118,7 +119,7 @@ module Aspera
           aoc_config[:private_key]=ExtendedValue.instance.evaluate(aoc_config[:private_key])
           new_agent=Fasp::Aoc.new(aoc_config)
         else
-          raise "INTERNAL ERROR"
+          raise "Unexpected transfer agent type: #{agent_type}"
         end
         set_agent_instance(new_agent)
         return nil
@@ -178,9 +179,9 @@ module Aspera
           # when providing a list, just specify source
           @transfer_paths=file_list.map{|i|{'source'=>i}}
         when :pair
-          raise CliBadArgument,"whe using pair, provide even number of paths: #{file_list.length}" unless file_list.length.even?
+          raise CliBadArgument,"When using pair, provide an even number of paths: #{file_list.length}" unless file_list.length.even?
           @transfer_paths=file_list.each_slice(2).to_a.map{|s,d|{'source'=>s,'destination'=>d}}
-        else raise "ERROR"
+        else raise "Unsupported src_type"
         end
         Log.log.debug("paths=#{@transfer_paths}")
         return @transfer_paths

data/lib/aspera/cli/version.rb

@@ -1,5 +1,5 @@
 module Aspera
   module Cli
-    VERSION = "4.1.0"
+    VERSION = "4.2.0"
   end
 end

data/lib/aspera/cos_node.rb

@@ -5,46 +5,51 @@ require 'xmlsimple'
 module Aspera
   class CosNode < Rest
     attr_reader :add_ts
-    def initialize(bucket_name,storage_endpoint,instance_id,api_key,auth_url='https://iam.cloud.ibm.com/identity')
+    IBM_CLOUD_TOKEN_URL='https://iam.cloud.ibm.com/identity'
+    def initialize(bucket_name,storage_endpoint,instance_id,api_key,auth_url=IBM_CLOUD_TOKEN_URL)
+      @auth_url=auth_url
+      @api_key=api_key
       s3_api=Aspera::Rest.new({
-        :base_url => storage_endpoint,
-        :not_auth_codes => ['401','403'],
-        :headers  => {'ibm-service-instance-id' => instance_id},
-        :auth     => {
-        :type       => :oauth2,
-        :base_url   => auth_url,
-        :grant      => :ibm_apikey,
-        :api_key    => api_key
+        :base_url       => storage_endpoint,
+        :not_auth_codes => ['401','403'], # error codes when not authorized
+        :headers        => {'ibm-service-instance-id' => instance_id},
+        :auth           => {
+        :type      => :oauth2,
+        :base_url  => @auth_url,
+        :grant     => :ibm_apikey,
+        :api_key   => @api_key
         }})
       # read FASP connection information for bucket
       xml_result_text=s3_api.call({:operation=>'GET',:subpath=>bucket_name,:headers=>{'Accept'=>'application/xml'},:url_params=>{'faspConnectionInfo'=>nil}})[:http].body
       ats_info=XmlSimple.xml_in(xml_result_text, {'ForceArray' => false})
       Aspera::Log.dump('ats_info',ats_info)
-      # get delegated token
-      delegated_oauth=Oauth.new({
-        :type       => :oauth2,
-        :base_url   => auth_url,
-        :grant      => :delegated_refresh,
-        :api_key    => api_key,
-        :token_field=> 'delegated_refresh_token'
-      })
-      # to be placed in rest call header and in transfer tags
-      aspera_storage_credentials={
-        'type'  => 'token',
-        'token' => {'delegated_refresh_token'=>delegated_oauth.get_authorization().gsub(/^Bearer /,'')}
-      }
-      # transfer spec addition
-      @add_ts={'tags'=>{'aspera'=>{'node'=>{'storage_credentials'=>aspera_storage_credentials}}}}
-      # set a general addon to transfer spec
-      # here we choose to use the add_request_param
-      #self.transfer.option_transfer_spec_deep_merge(@add_ts)
       super({
         :base_url => ats_info['ATSEndpoint'],
-        :headers  => {'X-Aspera-Storage-Credentials'=>JSON.generate(aspera_storage_credentials)},
         :auth     => {
         :type     => :basic,
         :username => ats_info['AccessKey']['Id'],
         :password => ats_info['AccessKey']['Secret']}})
+      # prepare transfer spec addition
+      @add_ts={'tags'=>{'aspera'=>{'node'=>{'storage_credentials'=>{
+        'type'  => 'token',
+        'token' => {'delegated_refresh_token'=>nil}
+        }}}}}
+      generate_token
+    end
+
+    # potentially call this if delegated token is expired
+    def generate_token
+      # OAuth API to get delegated token
+      delegated_oauth=Oauth.new({
+        :type       => :oauth2,
+        :base_url   => @auth_url,
+        :grant      => :delegated_refresh,
+        :api_key    => @api_key,
+        :token_field=> 'delegated_refresh_token'
+      })
+      # get delagated token to be placed in rest call header and in transfer tags
+      @add_ts['tags']['aspera']['node']['storage_credentials']['token']['delegated_refresh_token']=delegated_oauth.get_authorization().gsub(/^Bearer /,'')
+      @params[:headers]={'X-Aspera-Storage-Credentials'=>JSON.generate(@add_ts['tags']['aspera']['node']['storage_credentials'])}
     end
   end
 end

data/lib/aspera/environment.rb

@@ -2,7 +2,7 @@ require 'aspera/log'
 require 'rbconfig'
 
 module Aspera
-  # a simple binary data repository
+  # detect OS, architecture, and OS specific stuff
   class Environment
     OS_WINDOWS = :windows
     OS_X = :osx
@@ -53,7 +53,7 @@ module Aspera
       return ''
     end
 
-    # on Windows, the env var %USERPROFILE% provides the path to user's home more reliably then %HOMEDRIVE%%HOMEPATH%
+    # on Windows, the env var %USERPROFILE% provides the path to user's home more reliably than %HOMEDRIVE%%HOMEPATH%
     def self.fix_home
       if os.eql?(OS_WINDOWS)
         if ENV.has_key?('USERPROFILE') and Dir.exist?(ENV['USERPROFILE'])

data/lib/aspera/fasp/installation.rb

@@ -22,11 +22,18 @@ module Aspera
       PRODUCT_CLI_V1='Aspera CLI'
       PRODUCT_DRIVE='Aspera Drive'
       PRODUCT_ENTSRV='Enterprise Server'
-      # currently used ascp executable
+      MAX_REDIRECT_SDK=2
+      private_constant :MAX_REDIRECT_SDK
+      # set ascp executable path
       def ascp_path=(v)
         @path_to_ascp=v
       end
 
+      # filename for ascp with optional extension (Windows)
+      def ascp_filename
+        return 'ascp'+Environment.exe_extension
+      end
+
       # location of SDK files
       def folder=(v)
         @sdk_folder=v
@@ -50,26 +57,30 @@ module Aspera
       # @return the list of installed products in format of product_locations
       def installed_products
         if @found_products.nil?
-          @found_products=product_locations
-          # add sdk as first search path
-          @found_products.unshift({# SDK
+          scan_locations=product_locations.clone
+          # add SDK as first search path
+          scan_locations.unshift({
             :expected =>'SDK',
             :app_root =>folder_path,
             :sub_bin =>''
           })
-          @found_products.select! do |pl|
-            next false unless Dir.exist?(pl[:app_root])
-            Log.log.debug("found #{pl[:app_root]}")
-            sub_bin = pl[:sub_bin] || BIN_SUBFOLDER
-            pl[:ascp_path]=File.join(pl[:app_root],sub_bin,'ascp'+Environment.exe_extension)
-            next false unless File.exist?(pl[:ascp_path])
-            product_info_file="#{pl[:app_root]}/#{PRODUCT_INFO}"
+          # search installed products: with ascp
+          @found_products=scan_locations.select! do |item|
+            # skip if not main folder
+            next false unless Dir.exist?(item[:app_root])
+            Log.log.debug("Found #{item[:app_root]}")
+            sub_bin = item[:sub_bin] || BIN_SUBFOLDER
+            item[:ascp_path]=File.join(item[:app_root],sub_bin,ascp_filename)
+            # skip if no ascp
+            next false unless File.exist?(item[:ascp_path])
+            # read info from product info file if present
+            product_info_file="#{item[:app_root]}/#{PRODUCT_INFO}"
             if File.exist?(product_info_file)
-              res_s=XmlSimple.xml_in(File.read(product_info_file),{"ForceArray"=>false})
-              pl[:name]=res_s['name']
-              pl[:version]=res_s['version']
+              res_s=XmlSimple.xml_in(File.read(product_info_file),{'ForceArray'=>false})
+              item[:name]=res_s['name']
+              item[:version]=res_s['version']
             else
-              pl[:name]=pl[:expected]
+              item[:name]=item[:expected]
             end
             true # select this version
           end
@@ -77,6 +88,7 @@ module Aspera
         return @found_products
       end
 
+      # all ascp files (in SDK)
       FILES=[:ascp,:ascp4,:ssh_bypass_key_dsa,:ssh_bypass_key_rsa,:aspera_license,:aspera_conf,:fallback_cert,:fallback_key]
 
       # get path of one resource file of currently activated product
@@ -106,14 +118,8 @@ module Aspera
 <CONF version="2">
 <default>
     <file_system>
-        <storage_rc>
-            <adaptive>
-                true
-            </adaptive>
-        </storage_rc>
         <resume_suffix>.aspera-ckpt</resume_suffix>
         <partial_file_suffix>.partial</partial_file_suffix>
-        <replace_illegal_chars>_</replace_illegal_chars>
     </file_system>
 </default>
 </CONF>
@@ -147,7 +153,7 @@ module Aspera
         return file
       end
 
-      # @returns the file path of local connect where API's URI can be read
+      # @return the file path of local connect where API's URI can be read
       def connect_uri
         connect=get_product_folders(PRODUCT_CONNECT)
         folder=File.join(connect[:run_root],VARRUN_SUBFOLDER)
@@ -176,8 +182,21 @@ module Aspera
         return [:ssh_bypass_key_dsa,:ssh_bypass_key_rsa].map{|i|Installation.instance.path(i)}
       end
 
+      # Check that specified path is ascp and get version
+      def get_ascp_version(ascp_path)
+        raise "File basename of #{ascp_path} must be #{ascp_filename}" unless File.basename(ascp_path).eql?(ascp_filename)
+        ascp_version='n/a'
+        raise "error in sdk: no ascp included" if ascp_path.nil?
+        cmd_out=%x{"#{ascp_path}" -A}
+        raise "An error occured when testing #{ascp_filename}: #{cmd_out}" unless $? == 0
+        # get version from ascp, only after full extract, as windows requires DLLs (SSL/TLS/etc...)
+        m=cmd_out.match(/ascp version (.*)/)
+        ascp_version=m[1] unless m.nil?
+      end
+
       # download aspera SDK or use local file
-      # extract only ascp binary for current system architecture
+      # extracts ascp binary for current system architecture
+      # @return ascp version (from execution)
       def install_sdk(sdk_url)
         require 'zip'
         sdk_zip_path=File.join(Dir.tmpdir,'sdk.zip')
@@ -186,7 +205,7 @@ module Aspera
           raise 'use format: file:///<path>' unless sdk_url.start_with?('file:///')
           sdk_zip_path=sdk_url.gsub(%r{^file:///},'')
         else
-          redirect_remain=2
+          redirect_remain=MAX_REDIRECT_SDK
           begin
             Aspera::Rest.new(base_url: sdk_url).call(operation: 'GET',save_to_file: sdk_zip_path)
           rescue Aspera::RestCallError => e
@@ -196,7 +215,7 @@ module Aspera
                 sdk_url=e.response['location']
                 retry
               else
-                raise "too meny redirect"
+                raise "Too many redirect"
               end
             else
               raise e
@@ -206,6 +225,12 @@ module Aspera
         # SDK is organized by architecture
         filter="/#{Environment.architecture}/"
         ascp_path=nil
+        sdk_path=folder_path
+        # rename old install
+        if File.exist?(File.join(sdk_path,ascp_filename))
+          Log.log.warn("Previous install exists, renaming.")
+          File.rename(sdk_path,"#{sdk_path}.#{Time.now.strftime("%Y%m%d%H%M%S")}")
+        end
         # first ensure license file is here so that ascp invokation for version works
         self.path(:aspera_license)
         self.path(:aspera_conf)
@@ -213,11 +238,11 @@ module Aspera
           zip_file.each do |entry|
             # get only specified arch, but not folder, only files
             if entry.name.include?(filter) and !entry.name.end_with?('/')
-              archive_file=File.join(folder_path,File.basename(entry.name))
+              archive_file=File.join(sdk_path,File.basename(entry.name))
               File.open(archive_file, 'wb') do |output_stream|
                 IO.copy_stream(entry.get_input_stream, output_stream)
               end
-              if entry.name.include?('ascp')
+              if File.basename(entry.name).eql?(ascp_filename)
                 FileUtils.chmod(0755,archive_file)
                 ascp_path=archive_file
               end
@@ -225,13 +250,7 @@ module Aspera
           end
         end
         File.unlink(sdk_zip_path) rescue nil # Windows may give error
-        ascp_version='n/a'
-        raise "error in sdk: no ascp included" if ascp_path.nil?
-        cmd_out=%x{#{ascp_path} -A}
-        raise "An error occured when testing ascp: #{cmd_out}" unless $? == 0
-        # get version from ascp, only after full extract, as windows requires DLLs (SSL/TLS/etc...)
-        m=cmd_out.match(/ascp version (.*)/)
-        ascp_version=m[1] unless m.nil?
+        ascp_version=get_ascp_version(ascp_path)
         File.write(File.join(folder_path,PRODUCT_INFO),"<product><name>IBM Aspera SDK</name><version>#{ascp_version}</version></product>")
         return ascp_version
       end
@@ -248,27 +267,28 @@ module Aspera
 
       private_constant :BIN_SUBFOLDER,:ETC_SUBFOLDER,:VARRUN_SUBFOLDER,:PRODUCT_INFO
 
-      # get some specific folder from specific applications: Connect or CLI
-      def get_product_folders(name)
-        found=installed_products.select{|i|i[:expected].eql?(name) or i[:name].eql?(name)}
-        raise "Product: #{name} not found, please install." if found.empty?
-        return found.first
-      end
-
       def initialize
         @path_to_ascp=nil
         @sdk_folder=nil
         @found_products=nil
       end
 
+      # @return folder paths for specified applications
+      # @param name Connect or CLI
+      def get_product_folders(name)
+        found=installed_products.select{|i|i[:expected].eql?(name) or i[:name].eql?(name)}
+        raise "Product: #{name} not found, please install." if found.empty?
+        return found.first
+      end
+
+      # @return the path to folder where SDK is installed
       def folder_path
-        raise "undefined path to SDK" if @sdk_folder.nil?
+        raise "Undefined path to SDK" if @sdk_folder.nil?
         FileUtils.mkdir_p(@sdk_folder) unless Dir.exist?(@sdk_folder)
         @sdk_folder
       end
 
-      # returns product folders depending on OS
-      # fields
+      # @return product folders depending on OS fields
       # :expected  M app name is taken from the manifest if present, else defaults to this value
       # :app_root  M main folder for the application
       # :log_root  O location of log files (Linux uses syslog)
@@ -310,7 +330,7 @@ module Aspera
             :log_root =>File.join(Dir.home,'Library','Logs','Aspera_Drive'),
             :sub_bin  =>File.join('Contents','Resources'),
             }]
-        else; return [{  # other: Linux and unix family
+        else; return [{  # other: Linux and Unix family
             :expected =>PRODUCT_CONNECT,
             :app_root =>File.join(Dir.home,'.aspera','connect'),
             :run_root =>File.join(Dir.home,'.aspera','connect')
@@ -324,6 +344,9 @@ module Aspera
         end
       end
 
+      # @return a standard bypass key
+      # @param type rsa or dsa
+      # @param id in repository 1 for dsa, 2 for rsa
       def get_key(type,id)
         hf=['begin','end'].map{|t|"-----#{t} #{type} private key-----".upcase}
         bin=Base64.strict_encode64(DataRepository.instance.get_bin(id))