aspera-cli 4.2.1 → 4.5.0

data/lib/aspera/fasp/{http_gw.rb → agent_httpgw.rb}

@@ -1,19 +1,18 @@
 #!/bin/echo this is a ruby class:
-require 'aspera/fasp/manager'
+require 'aspera/fasp/agent_base'
 require 'aspera/log'
 require 'aspera/rest'
 require 'websocket-client-simple'
 require 'securerandom'
-require 'openssl'
 require 'base64'
 require 'json'
-require 'uri'
 
 # ref: https://api.ibm.com/explorer/catalog/aspera/product/ibm-aspera/api/http-gateway-api/doc/guides-toc
+# https://developer.ibm.com/apis/catalog?search=%22aspera%20http%22
 module Aspera
   module Fasp
-    # executes a local "ascp", connects mgt port, equivalent of "Fasp Manager"
-    class HttpGW < Manager
+    # start a transfer using Aspera HTTP Gateway, using web socket session
+    class AgentHttpgw < AgentBase
       # message returned by HTTP GW in case of success
       OK_MESSAGE='end upload'
       # refresh rate for progress
@@ -25,20 +24,36 @@ module Aspera
       end
 
       def upload(transfer_spec)
-        # precalculate size
+        # total size of all files
         total_size=0
-        # currently, files are sent flat
-        source_path=[]
+        # we need to keep track of actual file path because transfer spec is modified to be sent in web socket
+        source_paths=[]
+        # get source root or nil
+        source_root = (transfer_spec.has_key?('source_root') and !transfer_spec['source_root'].empty?) ? transfer_spec['source_root'] : nil
+        # source root is ignored by GW, used only here
+        transfer_spec.delete('source_root')
+        # compute total size of files to upload (for progress)
+        # modify transfer spec to be suitable for GW
         transfer_spec['paths'].each do |item|
-          filepath=item['source']
-          item['source']=item['destination']=File.basename(filepath)
-          total_size+=item['file_size']=File.size(filepath)
-          source_path.push(filepath)
+          # save actual file location to be able read contents later
+          full_src_filepath=item['source']
+          # add source root if needed
+          full_src_filepath=File.join(source_root,full_src_filepath) unless source_root.nil?
+          # GW expects a simple file name in 'source' but if user wants to change the name, we take it
+          item['source']=File.basename(item['destination'].nil? ? item['source'] : item['destination'])
+          item['file_size']=File.size(full_src_filepath)
+          total_size+=item['file_size']
+          # save so that we can actually read the file later
+          source_paths.push(full_src_filepath)
         end
+
         session_id=SecureRandom.uuid
         ws=::WebSocket::Client::Simple::Client.new
+        # error message if any in callback
         error=nil
+        # number of files totally sent
         received=0
+        # setup callbacks on websocket
         ws.on :message do |msg|
           Log.log.info("ws: message: #{msg.data}")
           message=msg.data
@@ -64,44 +79,49 @@ module Aspera
         end
         # open web socket to end point
         ws.connect("#{@gw_api.params[:base_url]}/upload")
+        # async wait ready
         while !ws.open? and error.nil? do
           Log.log.info("ws: wait")
           sleep(0.2)
         end
+        # notify progress bar
         notify_begin(session_id,total_size)
+        # first step send transfer spec
+        Log.dump(:ws_spec,transfer_spec)
         ws_send(ws,:transfer_spec,transfer_spec)
         # current file index
-        filenum=0
+        file_index=0
         # aggregate size sent
         sent_bytes=0
         # last progress event
-        lastevent=Time.now-1
+        lastevent=nil
         transfer_spec['paths'].each do |item|
-          # TODO: on destination write same path?
-          destination_path=item['source']
           # TODO: get mime type?
           file_mime_type=''
-          total=item['file_size']
+          file_size=item['file_size']
+          file_name=File.basename(item[item['destination'].nil? ? 'source' : 'destination'])
           # compute total number of slices
-          numslices=1+(total-1)/@upload_chunksize
-          # current slice index
-          slicenum=0
-          File.open(source_path[filenum]) do |file|
+          numslices=1+(file_size-1)/@upload_chunksize
+          File.open(source_paths[file_index]) do |file|
+            # current slice index
+            slicenum=0
             while !file.eof? do
               data=file.read(@upload_chunksize)
               slice_data={
-                name: destination_path,
+                name: file_name,
                 type: file_mime_type,
-                size: total,
+                size: file_size,
                 data: Base64.strict_encode64(data),
                 slice: slicenum,
                 total_slices: numslices,
-                fileIndex: filenum
+                fileIndex: file_index
               }
+              # log without data
+              Log.dump(:slide_data,slice_data.keys.inject({}){|m,i|m[i]=i.eql?(:data)?'base64 data':slice_data[i];m}) if slicenum.eql?(0)
               ws_send(ws,:slice_upload, slice_data)
               sent_bytes+=data.length
               currenttime=Time.now
-              if (currenttime-lastevent)>UPLOAD_REFRESH_SEC
+              if lastevent.nil? or (currenttime-lastevent)>UPLOAD_REFRESH_SEC
                 notify_progress(session_id,sent_bytes)
                 lastevent=currenttime
               end
@@ -109,7 +129,7 @@ module Aspera
               raise error unless error.nil?
             end
           end
-          filenum+=1
+          file_index+=1
         end
         ws.close
         notify_end(session_id)
@@ -150,7 +170,8 @@ module Aspera
         raise "GW URL must be set" unless !@gw_api.nil?
         raise "option: must be hash (or nil)" unless options.is_a?(Hash)
         raise "paths: must be Array" unless transfer_spec['paths'].is_a?(Array)
-        raise "on token based transfer is supported in GW" unless transfer_spec['token'].is_a?(String)
+        raise "only token based transfer is supported in GW" unless transfer_spec['token'].is_a?(String)
+        Log.dump(:user_spec,transfer_spec)
         transfer_spec['authentication']||='token'
         case transfer_spec['direction']
         when 'send'
@@ -158,7 +179,7 @@ module Aspera
         when 'receive'
           download(transfer_spec)
         else
-          raise "error"
+          raise "unexpected direction: [#{transfer_spec['direction']}]"
         end
       end # start_transfer
 
@@ -188,6 +209,6 @@ module Aspera
         @upload_chunksize=128000 # TODO: configurable ?
       end
 
-    end # LocalHttp
+    end # AgentHttpgw
   end
 end

data/lib/aspera/fasp/{node.rb → agent_node.rb}

@@ -1,4 +1,4 @@
-require 'aspera/fasp/manager'
+require 'aspera/fasp/agent_base'
 require 'aspera/log'
 require 'tty-spinner'
 
@@ -6,10 +6,30 @@ module Aspera
   module Fasp
     # this singleton class is used by the CLI to provide a common interface to start a transfer
     # before using it, the use must set the `node_api` member.
-    class Node < Manager
-      def initialize(node_api)
+    class AgentNode < AgentBase
+      # option include: root_id if the node is an access key
+      attr_writer :options
+      def initialize(options)
+        raise "node specification must be Hash" unless options.is_a?(Hash)
+        [:url,:username,:password].each { |k| raise "missing parameter [#{k}] in node specification: #{options}" unless options.has_key?(k) }
         super()
-        @node_api=node_api
+        # root id is required for access key
+        @root_id=options[:root_id]
+        rest_params={ base_url: options[:url]}
+        if options[:password].match(/^Bearer /)
+          rest_params[:headers]={
+            'X-Aspera-AccessKey'=>options[:username],
+            'Authorization'     =>options[:password]
+          }
+          raise "root_id is required for access key" if @root_id.nil?
+        else
+          rest_params[:auth]={
+            type:     :basic,
+            username: options[:username],
+            password: options[:password]
+          }
+        end
+        @node_api=Rest.new(rest_params)
         # TODO: currently only supports one transfer. This is bad shortcut. but ok for CLI.
         @transfer_id=nil
       end
@@ -32,6 +52,25 @@ module Aspera
 
       # generic method
       def start_transfer(transfer_spec,options=nil)
+        # add root id if access key
+        if ! @root_id.nil?
+          case transfer_spec['direction']
+          when 'send';transfer_spec['source_root_id']=@root_id
+          when 'receive';transfer_spec['destination_root_id']=@root_id
+          else raise "unexpected direction in ts: #{transfer_spec['direction']}"
+          end
+        end
+        # manage special additional parameter
+        if transfer_spec.has_key?('EX_ssh_key_paths') and transfer_spec['EX_ssh_key_paths'].is_a?(Array) and !transfer_spec['EX_ssh_key_paths'].empty?
+          # not standard, so place standard field
+          if transfer_spec.has_key?('ssh_private_key')
+            Log.log.warn('Both ssh_private_key and EX_ssh_key_paths are present, using ssh_private_key')
+          else
+            Log.log.warn('EX_ssh_key_paths has multiple keys, using first one only') unless transfer_spec['EX_ssh_key_paths'].length.eql?(1)
+            transfer_spec['ssh_private_key']=File.read(transfer_spec['EX_ssh_key_paths'].first)
+            transfer_spec.delete('EX_ssh_key_paths')
+          end
+        end
         if transfer_spec['tags'].is_a?(Hash) and transfer_spec['tags']['aspera'].is_a?(Hash)
           transfer_spec['tags']['aspera']['xfer_retry']||=150
         end

data/lib/aspera/fasp/agent_trsdk.rb

@@ -0,0 +1,106 @@
+require 'aspera/fasp/agent_base'
+require 'aspera/fasp/installation'
+require 'json'
+
+module Aspera
+  module Fasp
+    class AgentTrsdk < AgentBase
+      DEFAULT_OPTIONS = {
+        address: '127.0.0.1',
+        port:    55002,
+      }
+      private_constant :DEFAULT_OPTIONS
+
+      # options come from transfer_info
+      def initialize(user_opts)
+        raise "expecting Hash (or nil), but have #{user_opts.class}" unless user_opts.nil? or user_opts.is_a?(Hash)
+        # set default options and override if specified
+        options=DEFAULT_OPTIONS.clone
+        if !user_opts.nil?
+          user_opts.each do |k,v|
+            if DEFAULT_OPTIONS.has_key?(k)
+              options[k]=v
+            else
+              raise "Unknown local agent parameter: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map{|i|i.to_s}.join(",")}"
+            end
+          end
+        end
+        Log.log.debug("options= #{options}")
+        super()
+        # load and create SDK stub
+        $LOAD_PATH.unshift(Installation.instance.sdk_ruby_folder)
+        require 'transfer_services_pb'
+        @transfer_client = Transfersdk::TransferService::Stub.new("#{options[:address]}:#{options[:port]}",:this_channel_is_insecure)
+        begin
+          get_info_response = @transfer_client.get_info(Transfersdk::InstanceInfoRequest.new)
+          Log.log.debug("daemon info: #{get_info_response}")
+        rescue GRPC::Unavailable => e
+          Log.log.warn("no daemon present, starting daemon...")
+          # location of daemon binary
+          bin_folder=File.realpath(File.join(Installation.instance.sdk_ruby_folder,'..'))
+          # config file and logs are created in same folder
+          conf_file = File.join(bin_folder,'sdk.conf')
+          log_base = File.join(bin_folder,'transferd')
+          # create a config file for daemon
+          config = {
+            address: options[:address],
+            port:    options[:port],
+            fasp_runtime: {
+            use_embedded: false,
+            user_defined: {
+            bin: bin_folder,
+            etc: bin_folder,
+            }
+            }
+          }
+          File.write(conf_file,config.to_json)
+          trd_pid = Process.spawn(Installation.instance.path(:transferd),'--config' , conf_file, out: "#{log_base}.out", err: "#{log_base}.err")
+          Process.detach(trd_pid)
+          sleep(2.0)
+          retry
+        end
+      end
+
+      def start_transfer(transfer_spec,options=nil)
+        # create a transfer request
+        transfer_request = Transfersdk::TransferRequest.new(
+        transferType: Transfersdk::TransferType::FILE_REGULAR, # transfer type (file/stream)
+        config: Transfersdk::TransferConfig.new, # transfer configuration
+        transferSpec: transfer_spec.to_json) # transfer definition
+        # send start transfer request to the transfer manager daemon
+        start_transfer_response = @transfer_client.start_transfer(transfer_request)
+        Log.log.debug("start transfer response #{start_transfer_response}")
+        @transfer_id = start_transfer_response.transferId
+        Log.log.debug("transfer started with id #{@transfer_id}")
+      end
+
+      def wait_for_transfers_completion
+        started=false
+        # monitor transfer status
+        @transfer_client.monitor_transfers(Transfersdk::RegistrationRequest.new(transferId: [@transfer_id])) do |response|
+          Log.dump(:response, response.to_h)
+          #Log.log.debug("#{response.sessionInfo.preTransferBytes} #{response.transferInfo.bytesTransferred}")
+          case response.status
+          when :RUNNING
+            if !started and !response.sessionInfo.preTransferBytes.eql?(0)
+              notify_begin(@transfer_id,response.sessionInfo.preTransferBytes)
+              started=true
+            elsif started
+              notify_progress(@transfer_id,response.transferInfo.bytesTransferred)
+            end
+          when :FAILED, :COMPLETED, :CANCELED
+            notify_end(@transfer_id)
+            raise Fasp::Error.new(JSON.parse(response.message)['Description']) unless :COMPLETED.eql?(response.status)
+            break
+          when :QUEUED,:UNKNOWN_STATUS,:PAUSED,:ORPHANED
+            # ignore
+          else
+            Log.log.error("unknown status#{response.status}")
+          end
+        end
+        # TODO return status
+        return []
+      end
+    end
+  end
+end

data/lib/aspera/fasp/default.rb

@@ -0,0 +1,17 @@
+module Aspera
+  module Fasp
+    # default parameters
+    class Default
+      # (public) default transfer username for access key based transfers
+      ACCESS_KEY_TRANSFER_USER='xfer'
+      SSH_PORT=33001
+      UDP_PORT=33001
+      AK_TSPEC_BASE={
+        'remote_user' => ACCESS_KEY_TRANSFER_USER,
+        'ssh_port'    => SSH_PORT,
+        'fasp_port'   => UDP_PORT
+      }
+    end
+  end
+end
+

data/lib/aspera/fasp/installation.rb

@@ -10,7 +10,7 @@ require 'fileutils'
 module Aspera
   module Fasp
     # Singleton that tells where to find ascp and other local resources (keys..) , using the "path(symb)" method.
-    # It is used by object : Fasp::Local to find necessary resources
+    # It is used by object : AgentDirect to find necessary resources
     # By default it takes the first Aspera product found specified in product_locations
     # but the user can specify ascp location by calling:
     # Installation.instance.use_ascp_from_product(product_name)
@@ -22,6 +22,9 @@ module Aspera
       PRODUCT_CLI_V1='Aspera CLI'
       PRODUCT_DRIVE='Aspera Drive'
       PRODUCT_ENTSRV='Enterprise Server'
+      # protobuf generated files from sdk
+      EXT_RUBY_PROTOBUF='_pb.rb'
+      RB_SDK_FOLDER='lib'
       MAX_REDIRECT_SDK=2
       private_constant :MAX_REDIRECT_SDK
       # set ascp executable path
@@ -29,9 +32,10 @@ module Aspera
         @path_to_ascp=v
       end
 
-      # filename for ascp with optional extension (Windows)
-      def ascp_filename
-        return 'ascp'+Environment.exe_extension
+      def sdk_ruby_folder
+        ruby_pb_folder=File.join(folder_path,RB_SDK_FOLDER)
+        FileUtils.mkdir_p(ruby_pb_folder) unless Dir.exist?(ruby_pb_folder)
+        return ruby_pb_folder
       end
 
       # location of SDK files
@@ -100,6 +104,8 @@ module Aspera
           file=@path_to_ascp
           # note that there might be a .exe at the end
           file=file.gsub('ascp','ascp4') if k.eql?(:ascp4)
+        when :transferd
+          file=transferd_filepath
         when :ssh_bypass_key_dsa
           file=File.join(folder_path,'aspera_bypass_dsa.pem')
           File.write(file,get_key('dsa',1)) unless File.exist?(file)
@@ -182,16 +188,22 @@ module Aspera
         return [:ssh_bypass_key_dsa,:ssh_bypass_key_rsa].map{|i|Installation.instance.path(i)}
       end
 
+      # use in plugin `config`
+      def get_ascp_version(exe_path)
+        return get_exe_version(exe_path,'-A')
+      end
+
       # Check that specified path is ascp and get version
-      def get_ascp_version(ascp_path)
-        raise "File basename of #{ascp_path} must be #{ascp_filename}" unless File.basename(ascp_path).eql?(ascp_filename)
-        ascp_version='n/a'
-        raise "error in sdk: no ascp included" if ascp_path.nil?
-        cmd_out=%x{"#{ascp_path}" -A}
+      def get_exe_version(exe_path,vers_arg)
+        raise "ERROR: nil arg" if exe_path.nil?
+        return nil unless File.exist?(exe_path)
+        exe_version=nil
+        cmd_out=%x{"#{exe_path}" #{vers_arg}}
         raise "An error occured when testing #{ascp_filename}: #{cmd_out}" unless $? == 0
         # get version from ascp, only after full extract, as windows requires DLLs (SSL/TLS/etc...)
-        m=cmd_out.match(/ascp version (.*)/)
-        ascp_version=m[1] unless m.nil?
+        m=cmd_out.match(/ version ([0-9\.]+)/)
+        exe_version=m[1] unless m.nil?
+        return exe_version
       end
 
       # download aspera SDK or use local file
@@ -205,54 +217,49 @@ module Aspera
           raise 'use format: file:///<path>' unless sdk_url.start_with?('file:///')
           sdk_zip_path=sdk_url.gsub(%r{^file:///},'')
         else
-          redirect_remain=MAX_REDIRECT_SDK
-          begin
-            Aspera::Rest.new(base_url: sdk_url).call(operation: 'GET',save_to_file: sdk_zip_path)
-          rescue Aspera::RestCallError => e
-            if e.response.is_a?(Net::HTTPRedirection)
-              if redirect_remain > 0
-                redirect_remain-=1
-                sdk_url=e.response['location']
-                retry
-              else
-                raise "Too many redirect"
-              end
-            else
-              raise e
-            end
-          end
+          Aspera::Rest.new(base_url: sdk_url, redirect_max: 3).call(operation: 'GET',save_to_file: sdk_zip_path)
         end
-        # SDK is organized by architecture
-        filter="/#{Environment.architecture}/"
-        ascp_path=nil
-        sdk_path=folder_path
         # rename old install
-        if File.exist?(File.join(sdk_path,ascp_filename))
-          Log.log.warn("Previous install exists, renaming.")
-          File.rename(sdk_path,"#{sdk_path}.#{Time.now.strftime("%Y%m%d%H%M%S")}")
+        if ! Dir.empty?(folder_path)
+          Log.log.warn("Previous install exists, renaming folder.")
+          File.rename(folder_path,"#{folder_path}.#{Time.now.strftime("%Y%m%d%H%M%S")}")
+          # TODO: delete old archives ?
         end
-        # first ensure license file is here so that ascp invokation for version works
-        self.path(:aspera_license)
-        self.path(:aspera_conf)
+        # SDK is organized by architecture
+        arch_filter="#{Environment.architecture}/"
+        # extract files from archive
         Zip::File.open(sdk_zip_path) do |zip_file|
           zip_file.each do |entry|
-            # get only specified arch, but not folder, only files
-            if entry.name.include?(filter) and !entry.name.end_with?('/')
-              archive_file=File.join(sdk_path,File.basename(entry.name))
-              File.open(archive_file, 'wb') do |output_stream|
+            # skip folder entries
+            next if entry.name.end_with?('/')
+            dest_folder=nil
+            # binaries
+            dest_folder=folder_path if entry.name.include?(arch_filter)
+            # ruby adapters
+            dest_folder=sdk_ruby_folder if entry.name.end_with?(EXT_RUBY_PROTOBUF)
+            if !dest_folder.nil?
+              File.open(File.join(dest_folder,File.basename(entry.name)), 'wb') do |output_stream|
                 IO.copy_stream(entry.get_input_stream, output_stream)
               end
-              if File.basename(entry.name).eql?(ascp_filename)
-                FileUtils.chmod(0755,archive_file)
-                ascp_path=archive_file
-              end
             end
           end
         end
         File.unlink(sdk_zip_path) rescue nil # Windows may give error
-        ascp_version=get_ascp_version(ascp_path)
-        File.write(File.join(folder_path,PRODUCT_INFO),"<product><name>IBM Aspera SDK</name><version>#{ascp_version}</version></product>")
-        return ascp_version
+        # ensure license file are generated so that ascp invokation for version works
+        self.path(:aspera_license)
+        self.path(:aspera_conf)
+        ascp_path=File.join(folder_path,ascp_filename)
+        raise "No #{ascp_filename} found in SDK archive" unless File.exist?(ascp_path)
+        FileUtils.chmod(0755,ascp_path)
+        FileUtils.chmod(0755,ascp_path.gsub('ascp','ascp4'))
+        ascp_version=get_ascp_version(File.join(folder_path,ascp_filename))
+        trd_path=transferd_filepath
+        Log.log.warn("No #{trd_path} in SDK archive") unless File.exist?(trd_path)
+        FileUtils.chmod(0755,trd_path) if File.exist?(trd_path)
+        transferd_version=get_exe_version(trd_path,'version')
+        sdk_version = transferd_version||ascp_version
+        File.write(File.join(folder_path,PRODUCT_INFO),"<product><name>IBM Aspera SDK</name><version>#{sdk_version}</version></product>")
+        return sdk_version
       end
 
       private
@@ -288,6 +295,15 @@ module Aspera
         @sdk_folder
       end
 
+      # filename for ascp with optional extension (Windows)
+      def ascp_filename
+        return 'ascp'+Environment.exe_extension
+      end
+
+      def transferd_filepath
+        return File.join(folder_path,'asperatransferd'+Environment.exe_extension)
+      end
+
       # @return product folders depending on OS fields
       # :expected  M app name is taken from the manifest if present, else defaults to this value
       # :app_root  M main folder for the application