aspera-cli 4.0.0 → 4.2.2

data/lib/aspera/command_line_builder.rb

@@ -4,21 +4,46 @@ module Aspera
   # process_param is called repeatedly with all known parameters
   # add_env_args is called to get resulting param list and env var (also checks that all params were used)
   class CommandLineBuilder
+    # transform yes/no to trye/false
+    def self.yes_to_true(value)
+      case value
+      when 'yes'; return true
+      when 'no'; return false
+      end
+      raise "unsupported value: #{value}"
+    end
 
-    private
-    # default value for command line based on option name
-    def switch_name(param_name,options)
-      return options[:option_switch] if options.has_key?(:option_switch)
-      return '--'+param_name.to_s.gsub('_','-')
+    # Called by provider of definition before constructor of this class so that params_definition has all mandatory fields
+    def self.normalize_description(d)
+      d.each do |param_name,options|
+        Log.log.debug("def: #{param_name}")
+        raise "Expecting Hash, but have #{options.class} in #{param_name}" unless options.is_a?(Hash)
+        #options[:accepted_types]=:bool if options[:cltype].eql?(:envvar) and !options.has_key?(:accepted_types)
+        # by default : not mandatory
+        options[:mandatory]||=false
+        options[:desc]||=''
+        # by default : string, unless it's without arg
+        if ! options.has_key?(:accepted_types)
+          options[:accepted_types]=options[:cltype].eql?(:opt_without_arg) ? :bool : :string
+        end
+        # single type is placed in array
+        options[:accepted_types]=[options[:accepted_types]] unless options[:accepted_types].is_a?(Array)
+        if !options.has_key?(:option_switch) and options.has_key?(:cltype) and [:opt_without_arg,:opt_with_arg].include?(options[:cltype])
+          options[:option_switch]='--'+param_name.to_s.gsub('_','-')
+        end
+      end
     end
 
+    private
+
+    # clvarname : command line variable name
     def env_name(param_name,options)
-      return options[:variable]
+      return options[:clvarname]
     end
 
     public
 
-    BOOLEAN_CLASSES=[TrueClass,FalseClass]
+    attr_reader :params_definition
 
     # @param param_hash
     def initialize(param_hash,params_definition)
@@ -44,15 +69,6 @@ module Aspera
       return nil
     end
 
-    # transform yes/no to trye/false
-    def self.yes_to_true(value)
-      case value
-      when 'yes'; return true
-      when 'no'; return false
-      end
-      raise "unsupported value: #{value}"
-    end
-
     # add options directly to ascp command line
     def add_command_line_options(options)
       return if options.nil?
@@ -71,24 +87,25 @@ module Aspera
     # @param options : options for type
     def process_param(param_name,action=nil)
       options=@params_definition[param_name]
-      action=options[:type] if action.nil?
+      action=options[:cltype] if action.nil?
       # should not happen
       raise "Internal error: ask processing of param #{param_name}" if options.nil?
-      # by default : not mandatory
-      options[:mandatory]||=false
-      if options.has_key?(:accepted_types)
-        # single type is placed in array
-        options[:accepted_types]=[options[:accepted_types]] unless options[:accepted_types].is_a?(Array)
-      else
-        # by default : string, unless it's without arg
-        options[:accepted_types]=action.eql?(:opt_without_arg) ? BOOLEAN_CLASSES : [String]
-      end
       # check mandatory parameter (nil is valid value)
       raise Fasp::Error.new("mandatory parameter: #{param_name}") if options[:mandatory] and !@param_hash.has_key?(param_name)
       parameter_value=@param_hash[param_name]
-      parameter_value=options[:default] if parameter_value.nil? and options.has_key?(:default)
+      #parameter_value=options[:default] if parameter_value.nil? and options.has_key?(:default)
+      expected_classes=options[:accepted_types].map do |s|
+        case s
+        when :string; String
+        when :array; Array
+        when :hash; Hash
+        when :int; Integer
+        when :bool; [TrueClass,FalseClass]
+        else raise "INTERNAL: unexpected value: #{s}"
+        end
+      end.flatten
       # check provided type
-      raise Fasp::Error.new("#{param_name} is : #{parameter_value.class} (#{parameter_value}), shall be #{options[:accepted_types]}, ") unless parameter_value.nil? or options[:accepted_types].inject(false){|m,v|m or parameter_value.is_a?(v)}
+      raise Fasp::Error.new("#{param_name} is : #{parameter_value.class} (#{parameter_value}), shall be #{options[:accepted_types]}, ") unless parameter_value.nil? or expected_classes.include?(parameter_value.class)
       @used_param_names.push(param_name) unless action.eql?(:defer)
 
       # process only non-nil values
@@ -102,7 +119,8 @@ module Aspera
       end
       raise "unsupported value: #{parameter_value}" unless options[:accepted_values].nil? or options[:accepted_values].include?(parameter_value)
       if options[:encode]
-        newvalue=options[:encode].call(parameter_value)
+        # :encode has name of class with encoding method
+        newvalue=Kernel.const_get(options[:encode]).send("encode_#{param_name}",parameter_value)
         raise Fasp::Error.new("unsupported #{param_name}: #{parameter_value}") if newvalue.nil?
         parameter_value=newvalue
       end
@@ -123,12 +141,12 @@ module Aspera
         else raise Fasp::Error.new("unsupported #{param_name}: #{parameter_value}")
         end
         add_param=!add_param if options[:add_on_false]
-        add_command_line_options([switch_name(param_name,options)]) if add_param
+        add_command_line_options([options[:option_switch]]) if add_param
       when :opt_with_arg # transform into command line option with value
         #parameter_value=parameter_value.to_s if parameter_value.is_a?(Integer)
         parameter_value=[parameter_value] unless parameter_value.is_a?(Array)
         # if transfer_spec value is an array, applies option many times
-        parameter_value.each{|v|add_command_line_options([switch_name(param_name,options),v])}
+        parameter_value.each{|v|add_command_line_options([options[:option_switch],v])}
       else
         raise "Error"
       end

data/lib/aspera/cos_node.rb

@@ -5,46 +5,51 @@ require 'xmlsimple'
 module Aspera
   class CosNode < Rest
     attr_reader :add_ts
-    def initialize(bucket_name,storage_endpoint,instance_id,api_key,auth_url='https://iam.cloud.ibm.com/identity')
+    IBM_CLOUD_TOKEN_URL='https://iam.cloud.ibm.com/identity'
+    def initialize(bucket_name,storage_endpoint,instance_id,api_key,auth_url=IBM_CLOUD_TOKEN_URL)
+      @auth_url=auth_url
+      @api_key=api_key
       s3_api=Aspera::Rest.new({
-        :base_url => storage_endpoint,
-        :not_auth_codes => ['401','403'],
-        :headers  => {'ibm-service-instance-id' => instance_id},
-        :auth     => {
-        :type       => :oauth2,
-        :base_url   => auth_url,
-        :grant      => :ibm_apikey,
-        :api_key    => api_key
+        :base_url       => storage_endpoint,
+        :not_auth_codes => ['401','403'], # error codes when not authorized
+        :headers        => {'ibm-service-instance-id' => instance_id},
+        :auth           => {
+        :type      => :oauth2,
+        :base_url  => @auth_url,
+        :grant     => :ibm_apikey,
+        :api_key   => @api_key
         }})
       # read FASP connection information for bucket
       xml_result_text=s3_api.call({:operation=>'GET',:subpath=>bucket_name,:headers=>{'Accept'=>'application/xml'},:url_params=>{'faspConnectionInfo'=>nil}})[:http].body
       ats_info=XmlSimple.xml_in(xml_result_text, {'ForceArray' => false})
       Aspera::Log.dump('ats_info',ats_info)
-      # get delegated token
-      delegated_oauth=Oauth.new({
-        :type       => :oauth2,
-        :base_url   => auth_url,
-        :grant      => :delegated_refresh,
-        :api_key    => api_key,
-        :token_field=> 'delegated_refresh_token'
-      })
-      # to be placed in rest call header and in transfer tags
-      aspera_storage_credentials={
-        'type'  => 'token',
-        'token' => {'delegated_refresh_token'=>delegated_oauth.get_authorization().gsub(/^Bearer /,'')}
-      }
-      # transfer spec addition
-      @add_ts={'tags'=>{'aspera'=>{'node'=>{'storage_credentials'=>aspera_storage_credentials}}}}
-      # set a general addon to transfer spec
-      # here we choose to use the add_request_param
-      #self.transfer.option_transfer_spec_deep_merge(@add_ts)
       super({
         :base_url => ats_info['ATSEndpoint'],
-        :headers  => {'X-Aspera-Storage-Credentials'=>JSON.generate(aspera_storage_credentials)},
         :auth     => {
         :type     => :basic,
         :username => ats_info['AccessKey']['Id'],
         :password => ats_info['AccessKey']['Secret']}})
+      # prepare transfer spec addition
+      @add_ts={'tags'=>{'aspera'=>{'node'=>{'storage_credentials'=>{
+        'type'  => 'token',
+        'token' => {'delegated_refresh_token'=>nil}
+        }}}}}
+      generate_token
+    end
+
+    # potentially call this if delegated token is expired
+    def generate_token
+      # OAuth API to get delegated token
+      delegated_oauth=Oauth.new({
+        :type       => :oauth2,
+        :base_url   => @auth_url,
+        :grant      => :delegated_refresh,
+        :api_key    => @api_key,
+        :token_field=> 'delegated_refresh_token'
+      })
+      # get delagated token to be placed in rest call header and in transfer tags
+      @add_ts['tags']['aspera']['node']['storage_credentials']['token']['delegated_refresh_token']=delegated_oauth.get_authorization().gsub(/^Bearer /,'')
+      @params[:headers]={'X-Aspera-Storage-Credentials'=>JSON.generate(@add_ts['tags']['aspera']['node']['storage_credentials'])}
     end
   end
 end

data/lib/aspera/environment.rb

@@ -2,7 +2,7 @@ require 'aspera/log'
 require 'rbconfig'
 
 module Aspera
-  # a simple binary data repository
+  # detect OS, architecture, and OS specific stuff
   class Environment
     OS_WINDOWS = :windows
     OS_X = :osx
@@ -53,7 +53,7 @@ module Aspera
       return ''
     end
 
-    # on Windows, the env var %USERPROFILE% provides the path to user's home more reliably then %HOMEDRIVE%%HOMEPATH%
+    # on Windows, the env var %USERPROFILE% provides the path to user's home more reliably than %HOMEDRIVE%%HOMEPATH%
     def self.fix_home
       if os.eql?(OS_WINDOWS)
         if ENV.has_key?('USERPROFILE') and Dir.exist?(ENV['USERPROFILE'])

data/lib/aspera/fasp/connect.rb

@@ -12,7 +12,9 @@ module Aspera
       private_constant :MAX_CONNECT_START_RETRY,:SLEEP_SEC_BETWEEN_RETRY
       def initialize
         super
-        @connect_app_id=SecureRandom.uuid
+        @connect_settings={
+          'app_id' => SecureRandom.uuid
+        }
         # TODO: start here and create monitor
       end
 
@@ -32,46 +34,51 @@ module Aspera
             OpenApplication.uri_graphical('https://downloads.asperasoft.com/connect2/')
             raise StandardError,'Connect is not installed'
           end
-          sleep SLEEP_SEC_BETWEEN_RETRY
+          sleep(SLEEP_SEC_BETWEEN_RETRY)
           retry
         end
         if transfer_spec['direction'] == 'send'
           Log.log.warn("Connect requires upload selection using GUI, ignoring #{transfer_spec['paths']}".red)
           transfer_spec.delete('paths')
-          resdata=@connect_api.create('windows/select-open-file-dialog/',{'title'=>'Select Files','suggestedName'=>'','allowMultipleSelection'=>true,'allowedFileTypes'=>'','aspera_connect_settings'=>{'app_id'=>@connect_app_id}})[:data]
+          resdata=@connect_api.create('windows/select-open-file-dialog/',{'aspera_connect_settings'=>@connect_settings,'title'=>'Select Files','suggestedName'=>'','allowMultipleSelection'=>true,'allowedFileTypes'=>''})[:data]
           transfer_spec['paths']=resdata['dataTransfer']['files'].map { |i| {'source'=>i['name']}}
         end
         @request_id=SecureRandom.uuid
         # if there is a token, we ask connect client to use well known ssh private keys
         # instead of asking password
         transfer_spec['authentication']='token' if transfer_spec.has_key?('token')
+        connect_settings=
         connect_transfer_args={
-          'transfer_specs'=>[{
-          'transfer_spec'=>transfer_spec,
-          'aspera_connect_settings'=>{
-          'allow_dialogs'=>true,
-          'app_id'=>@connect_app_id,
-          'request_id'=>@request_id
-          }}]}
+          'aspera_connect_settings'=>@connect_settings.merge({
+          'request_id'               =>@request_id,
+          'allow_dialogs'            =>true,
+          }),
+          'transfer_specs'         =>[{
+          'transfer_spec'            =>transfer_spec,
+          }]}
         # asynchronous anyway
-        @connect_api.create('transfers/start',connect_transfer_args)
+        res=@connect_api.create('transfers/start',connect_transfer_args)[:data]
+        @xfer_id=res['transfer_specs'].first['transfer_spec']['tags']['aspera']['xfer_id']
       end
 
       def wait_for_transfers_completion
-        connect_activity_args={'aspera_connect_settings'=>{'app_id'=>@connect_app_id}}
+        connect_activity_args={'aspera_connect_settings'=>@connect_settings}
         started=false
         spinner=nil
         loop do
-          result=@connect_api.create('transfers/activity',connect_activity_args)[:data]
-          if result['transfers']
-            trdata=result['transfers'].select{|i| i['aspera_connect_settings'] and i['aspera_connect_settings']['request_id'].eql?(@request_id)}.first
-            raise 'problem with connect, please kill it' unless trdata
+          tr_info=@connect_api.create("transfers/info/#{@xfer_id}",connect_activity_args)[:data]
+          if tr_info['transfer_info'].is_a?(Hash)
+            trdata=tr_info['transfer_info']
+            if trdata.nil?
+              Log.log.warn("no session in Connect")
+              break
+            end
             # TODO: get session id
             case trdata['status']
             when 'completed'
-              notify_listeners('emulated',{Manager::LISTENER_SESSION_ID_B=>@connect_app_id,'Type'=>'DONE'})
+              notify_end(@connect_settings['app_id'])
               break
-            when 'initiating'
+            when 'initiating','queued'
               if spinner.nil?
                 spinner = TTY::Spinner.new('[:spinner] :title', format: :classic)
                 spinner.start
@@ -81,13 +88,13 @@ module Aspera
             when 'running'
               #puts "running: sessions:#{trdata['sessions'].length}, #{trdata['sessions'].map{|i| i['bytes_transferred']}.join(',')}"
               if !started and trdata['bytes_expected'] != 0
-                notify_listeners('emulated',{Manager::LISTENER_SESSION_ID_B=>@connect_app_id,'Type'=>'NOTIFICATION','PreTransferBytes'=>trdata['bytes_expected']})
+                notify_begin(@connect_settings['app_id'],trdata['bytes_expected'])
                 started=true
               else
-                notify_listeners('emulated',{Manager::LISTENER_SESSION_ID_B=>@connect_app_id,'Type'=>'STATS','Bytescont'=>trdata['bytes_written']})
+                notify_progress(@connect_settings['app_id'],trdata['bytes_written'])
               end
             else
-              raise Fasp::Error.new("#{trdata['status']}: #{trdata['error_desc']}")
+              raise Fasp::Error.new("unknown status: #{trdata['status']}: #{trdata['error_desc']}")
             end
           end
           sleep 1

data/lib/aspera/fasp/http_gw.rb

@@ -2,12 +2,147 @@
 require 'aspera/fasp/manager'
 require 'aspera/log'
 require 'aspera/rest'
+require 'websocket-client-simple'
+require 'securerandom'
+require 'openssl'
+require 'base64'
+require 'json'
+require 'uri'
 
 # ref: https://api.ibm.com/explorer/catalog/aspera/product/ibm-aspera/api/http-gateway-api/doc/guides-toc
 module Aspera
   module Fasp
     # executes a local "ascp", connects mgt port, equivalent of "Fasp Manager"
     class HttpGW < Manager
+      # message returned by HTTP GW in case of success
+      OK_MESSAGE='end upload'
+      # refresh rate for progress
+      UPLOAD_REFRESH_SEC=0.5
+      private_constant :OK_MESSAGE,:UPLOAD_REFRESH_SEC
+      # send message on http gw web socket
+      def ws_send(ws,type,data)
+        ws.send(JSON.generate({type => data}))
+      end
+
+      def upload(transfer_spec)
+        # precalculate size
+        total_size=0
+        # currently, files are sent flat
+        source_path=[]
+        transfer_spec['paths'].each do |item|
+          filepath=item['source']
+          item['source']=item['destination']=File.basename(filepath)
+          total_size+=item['file_size']=File.size(filepath)
+          source_path.push(filepath)
+        end
+        session_id=SecureRandom.uuid
+        ws=::WebSocket::Client::Simple::Client.new
+        error=nil
+        received=0
+        ws.on :message do |msg|
+          Log.log.info("ws: message: #{msg.data}")
+          message=msg.data
+          if message.eql?(OK_MESSAGE)
+            received+=1
+          else
+            message.chomp!
+            if message[0].eql?('"') and message[-1].eql?('"')
+              error=JSON.parse(Base64.strict_decode64(message.chomp[1..-2]))['message']
+            else
+              error="expecting quotes in [#{message}]"
+            end
+          end
+        end
+        ws.on :error do |e|
+          error=e
+        end
+        ws.on :open do
+          Log.log.info("ws: open")
+        end
+        ws.on :close do
+          Log.log.info("ws: close")
+        end
+        # open web socket to end point
+        ws.connect("#{@gw_api.params[:base_url]}/upload")
+        while !ws.open? and error.nil? do
+          Log.log.info("ws: wait")
+          sleep(0.2)
+        end
+        notify_begin(session_id,total_size)
+        ws_send(ws,:transfer_spec,transfer_spec)
+        # current file index
+        filenum=0
+        # aggregate size sent
+        sent_bytes=0
+        # last progress event
+        lastevent=Time.now-1
+        transfer_spec['paths'].each do |item|
+          # TODO: on destination write same path?
+          destination_path=item['source']
+          # TODO: get mime type?
+          file_mime_type=''
+          total=item['file_size']
+          # compute total number of slices
+          numslices=1+(total-1)/@upload_chunksize
+          # current slice index
+          slicenum=0
+          File.open(source_path[filenum]) do |file|
+            while !file.eof? do
+              data=file.read(@upload_chunksize)
+              slice_data={
+                name: destination_path,
+                type: file_mime_type,
+                size: total,
+                data: Base64.strict_encode64(data),
+                slice: slicenum,
+                total_slices: numslices,
+                fileIndex: filenum
+              }
+              ws_send(ws,:slice_upload, slice_data)
+              sent_bytes+=data.length
+              currenttime=Time.now
+              if (currenttime-lastevent)>UPLOAD_REFRESH_SEC
+                notify_progress(session_id,sent_bytes)
+                lastevent=currenttime
+              end
+              slicenum+=1
+              raise error unless error.nil?
+            end
+          end
+          filenum+=1
+        end
+        ws.close
+        notify_end(session_id)
+      end
+
+      def download(transfer_spec)
+        transfer_spec['zip_required']||=false
+        transfer_spec['source_root']||='/'
+        # is normally provided by application, like package name
+        if !transfer_spec.has_key?('download_name')
+          # by default it is the name of first file
+          dname=File.basename(transfer_spec['paths'].first['source'])
+          # we remove extension
+          dname=dname.gsub(/\.@gw_api.*$/,'')
+          # ands add indication of number of files if there is more than one
+          if transfer_spec['paths'].length > 1
+            dname=dname+" #{transfer_spec['paths'].length} Files"
+          end
+          transfer_spec['download_name']=dname
+        end
+        creation=@gw_api.create('download',{'transfer_spec'=>transfer_spec})[:data]
+        transfer_uuid=creation['url'].split('/').last
+        if transfer_spec['zip_required'] or transfer_spec['paths'].length > 1
+          # it is a zip file if zip is required or there is more than 1 file
+          file_dest=transfer_spec['download_name']+'.zip'
+        else
+          # it is a plain file if we don't require zip and there is only one file
+          file_dest=File.basename(transfer_spec['paths'].first['source'])
+        end
+        file_dest=File.join(transfer_spec['destination_root'],file_dest)
+        @gw_api.call({:operation=>'GET',:subpath=>"download/#{transfer_uuid}",:save_to_file=>file_dest})
+      end
+
       # start FASP transfer based on transfer spec (hash table)
       # note that it is asynchronous
       # HTTP download only supports file list
@@ -15,37 +150,13 @@ module Aspera
         raise "GW URL must be set" unless !@gw_api.nil?
         raise "option: must be hash (or nil)" unless options.is_a?(Hash)
         raise "paths: must be Array" unless transfer_spec['paths'].is_a?(Array)
+        raise "on token based transfer is supported in GW" unless transfer_spec['token'].is_a?(String)
+        transfer_spec['authentication']||='token'
         case transfer_spec['direction']
         when 'send'
-          # this is a websocket
-          raise "error, not implemented"
+          upload(transfer_spec)
         when 'receive'
-          transfer_spec['zip_required']||=false
-          transfer_spec['authentication']||='token'
-          transfer_spec['source_root']||='/'
-          # is normally provided by application, like package name
-          if !transfer_spec.has_key?('download_name')
-            # by default it is the name of first file
-            dname=File.basename(transfer_spec['paths'].first['source'])
-            # we remove extension
-            dname=dname.gsub(/\.@gw_api.*$/,'')
-            # ands add indication of number of files if there is more than one
-            if transfer_spec['paths'].length > 1
-              dname=dname+" #{transfer_spec['paths'].length} Files"
-            end
-            transfer_spec['download_name']=dname
-          end
-          creation=@gw_api.create('download',{'transfer_spec'=>transfer_spec})[:data]
-          transfer_uuid=creation['url'].split('/').last
-          if transfer_spec['zip_required'] or transfer_spec['paths'].length > 1
-            # it is a zip file if zip is required or there is more than 1 file
-            file_dest=transfer_spec['download_name']+'.zip'
-          else
-            # it is a plain file if we don't require zip and there is only one file
-            file_dest=File.basename(transfer_spec['paths'].first['source'])
-          end
-          file_dest=File.join(transfer_spec['destination_root'],file_dest)
-          @gw_api.call({:operation=>'GET',:subpath=>"download/#{transfer_uuid}",:save_to_file=>file_dest})
+          download(transfer_spec)
         else
           raise "error"
         end
@@ -74,6 +185,7 @@ module Aspera
         @gw_api=Rest.new({:base_url => params[:url]})
         api_info = @gw_api.read('info')[:data]
         Log.log.info("#{api_info}")
+        @upload_chunksize=128000 # TODO: configurable ?
       end
 
     end # LocalHttp