aspera-cli 4.3.0 → 4.4.0

data/lib/aspera/fasp/http_gw.rb

@@ -4,15 +4,14 @@ require 'aspera/log'
 require 'aspera/rest'
 require 'websocket-client-simple'
 require 'securerandom'
-require 'openssl'
 require 'base64'
 require 'json'
-require 'uri'
 
 # ref: https://api.ibm.com/explorer/catalog/aspera/product/ibm-aspera/api/http-gateway-api/doc/guides-toc
+# https://developer.ibm.com/apis/catalog?search=%22aspera%20http%22
 module Aspera
   module Fasp
-    # executes a local "ascp", connects mgt port, equivalent of "Fasp Manager"
+    # start a transfer using Aspera HTTP Gateway, using web socket session
     class HttpGW < Manager
       # message returned by HTTP GW in case of success
       OK_MESSAGE='end upload'
@@ -25,20 +24,36 @@ module Aspera
       end
 
       def upload(transfer_spec)
-        # precalculate size
+        # total size of all files
         total_size=0
-        # currently, files are sent flat
-        source_path=[]
+        # we need to keep track of actual file path because transfer spec is modified to be sent in web socket
+        source_paths=[]
+        # get source root or nil
+        source_root = (transfer_spec.has_key?('source_root') and !transfer_spec['source_root'].empty?) ? transfer_spec['source_root'] : nil
+        # source root is ignored by GW, used only here
+        transfer_spec.delete('source_root')
+        # compute total size of files to upload (for progress)
+        # modify transfer spec to be suitable for GW
         transfer_spec['paths'].each do |item|
-          filepath=item['source']
-          item['source']=item['destination']=File.basename(filepath)
-          total_size+=item['file_size']=File.size(filepath)
-          source_path.push(filepath)
+          # save actual file location to be able read contents later
+          full_src_filepath=item['source']
+          # add source root if needed
+          full_src_filepath=File.join(source_root,full_src_filepath) unless source_root.nil?
+          # GW expects a simple file name in 'source' but if user wants to change the name, we take it
+          item['source']=File.basename(item['destination'].nil? ? item['source'] : item['destination'])
+          item['file_size']=File.size(full_src_filepath)
+          total_size+=item['file_size']
+          # save so that we can actually read the file later
+          source_paths.push(full_src_filepath)
         end
+
         session_id=SecureRandom.uuid
         ws=::WebSocket::Client::Simple::Client.new
+        # error message if any in callback
         error=nil
+        # number of files totally sent
         received=0
+        # setup callbacks on websocket
         ws.on :message do |msg|
           Log.log.info("ws: message: #{msg.data}")
           message=msg.data
@@ -64,44 +79,49 @@ module Aspera
         end
         # open web socket to end point
         ws.connect("#{@gw_api.params[:base_url]}/upload")
+        # async wait ready
         while !ws.open? and error.nil? do
           Log.log.info("ws: wait")
           sleep(0.2)
         end
+        # notify progress bar
         notify_begin(session_id,total_size)
+        # first step send transfer spec
+        Log.dump(:ws_spec,transfer_spec)
         ws_send(ws,:transfer_spec,transfer_spec)
         # current file index
-        filenum=0
+        file_index=0
         # aggregate size sent
         sent_bytes=0
         # last progress event
-        lastevent=Time.now-1
+        lastevent=nil
         transfer_spec['paths'].each do |item|
-          # TODO: on destination write same path?
-          destination_path=item['source']
           # TODO: get mime type?
           file_mime_type=''
-          total=item['file_size']
+          file_size=item['file_size']
+          file_name=File.basename(item[item['destination'].nil? ? 'source' : 'destination'])
           # compute total number of slices
-          numslices=1+(total-1)/@upload_chunksize
-          # current slice index
-          slicenum=0
-          File.open(source_path[filenum]) do |file|
+          numslices=1+(file_size-1)/@upload_chunksize
+          File.open(source_paths[file_index]) do |file|
+            # current slice index
+            slicenum=0
             while !file.eof? do
               data=file.read(@upload_chunksize)
               slice_data={
-                name: destination_path,
+                name: file_name,
                 type: file_mime_type,
-                size: total,
+                size: file_size,
                 data: Base64.strict_encode64(data),
                 slice: slicenum,
                 total_slices: numslices,
-                fileIndex: filenum
+                fileIndex: file_index
               }
+              # log without data
+              Log.dump(:slide_data,slice_data.keys.inject({}){|m,i|m[i]=i.eql?(:data)?'base64 data':slice_data[i];m}) if slicenum.eql?(0)
               ws_send(ws,:slice_upload, slice_data)
               sent_bytes+=data.length
               currenttime=Time.now
-              if (currenttime-lastevent)>UPLOAD_REFRESH_SEC
+              if lastevent.nil? or (currenttime-lastevent)>UPLOAD_REFRESH_SEC
                 notify_progress(session_id,sent_bytes)
                 lastevent=currenttime
               end
@@ -109,7 +129,7 @@ module Aspera
               raise error unless error.nil?
             end
           end
-          filenum+=1
+          file_index+=1
         end
         ws.close
         notify_end(session_id)
@@ -150,7 +170,8 @@ module Aspera
         raise "GW URL must be set" unless !@gw_api.nil?
         raise "option: must be hash (or nil)" unless options.is_a?(Hash)
         raise "paths: must be Array" unless transfer_spec['paths'].is_a?(Array)
-        raise "on token based transfer is supported in GW" unless transfer_spec['token'].is_a?(String)
+        raise "only token based transfer is supported in GW" unless transfer_spec['token'].is_a?(String)
+        Log.dump(:user_spec,transfer_spec)
         transfer_spec['authentication']||='token'
         case transfer_spec['direction']
         when 'send'
@@ -188,6 +209,6 @@ module Aspera
         @upload_chunksize=128000 # TODO: configurable ?
       end
 
-    end # LocalHttp
+    end # HttpGW
   end
 end

data/lib/aspera/fasp/local.rb

@@ -17,8 +17,6 @@ require 'securerandom'
 
 module Aspera
   module Fasp
-    # (public) default transfer username for access key based transfers
-    ACCESS_KEY_TRANSFER_USER='xfer'
     # executes a local "ascp", connects mgt port, equivalent of "Fasp Manager"
     class Local < Manager
       # options for initialize (same as values in option transfer_info)

data/lib/aspera/fasp/parameters.rb

@@ -70,10 +70,15 @@ module Aspera
         Base64.strict_encode64(JSON.generate(v))
       end
 
+      def self.ts_has_file_list(ts)
+        ts.has_key?('EX_ascp_args') and ts['EX_ascp_args'].is_a?(Array) and ['--file-list','--file-pair-list'].any?{|i|ts['EX_ascp_args'].include?(i)}
+      end
+
       def initialize(job_spec,options)
         @job_spec=job_spec
         @options=options
         @builder=Aspera::CommandLineBuilder.new(@job_spec,self.class.description)
+        Log.log.debug("agent options: #{@options}")
       end
 
       public
@@ -97,7 +102,7 @@ module Aspera
         @job_spec.delete('source_root') if @job_spec.has_key?('source_root') and @job_spec['source_root'].empty?
 
         # use web socket session initiation ?
-        if @builder.process_param('wss_enabled',:get_value) and @options[:wss]
+        if @builder.process_param('wss_enabled',:get_value) and ( @options[:wss] or !@job_spec.has_key?('fasp_port') )
           # by default use web socket session if available, unless removed by user
           @builder.add_command_line_options(['--ws-connect'])
           # TODO: option to give order ssh,ws (legacy http is implied bu ssh)
@@ -122,9 +127,17 @@ module Aspera
           # destination will be base64 encoded, put before path arguments
           @builder.add_command_line_options(['--dest64'])
         end
-        @builder.params_definition['paths'][:mandatory]=!@job_spec.has_key?('keepalive')
+        # paths is mandatory, unless ...
+        file_list_provided=self.class.ts_has_file_list(@job_spec)
+        @builder.params_definition['paths'][:mandatory]=!@job_spec.has_key?('keepalive') and !file_list_provided
         paths_array=@builder.process_param('paths',:get_value)
-        unless paths_array.nil?
+        if file_list_provided and ! paths_array.nil?
+          Log.log.warn("file list provided both in transfer spec and ascp file list. Keeping file list only.")
+          paths_array=nil
+        end
+        if ! paths_array.nil?
+          # it's an array
+          raise "paths is empty in transfer spec" if paths_array.empty?
           # use file list if there is storage defined for it.
           if @@file_list_folder.nil?
             # not safe for special characters ? (maybe not, depends on OS)

data/lib/aspera/faspex_gw.rb

@@ -1,5 +1,6 @@
 require 'aspera/log'
 require 'aspera/aoc'
+require 'aspera/node'
 require 'aspera/cli/main'
 require 'webrick'
 require 'webrick/https'
@@ -73,16 +74,16 @@ module Aspera
           "xfer_retry" => 3600 } }
         # this transfer spec is for transfer to AoC
         faspex_transfer_spec={
-          'direction' => 'send',
-          'remote_user' => 'xfer',
+          'direction'   => 'send',
           'remote_host' => node_info['host'],
-          'ssh_port' => 33001,
-          'fasp_port' => 33001,
-          'tags' => ts_tags,
-          'token' => node_auth_bearer_token,
-          'paths' => [{'destination' => '/'}],
-          'cookie' => 'unused',
-          'create_dir' => true,
+          'remote_user' => Node::ACCESS_KEY_TRANSFER_USER,
+          'ssh_port'    => Node::SSH_PORT_DEFAULT,
+          'fasp_port'   => Node::UDP_PORT_DEFAULT
+          'tags'        => ts_tags,
+          'token'       => node_auth_bearer_token,
+          'paths'       => [{'destination' => '/'}],
+          'cookie'      => 'unused',
+          'create_dir'  => true,
           'rate_policy' => 'fair',
           'rate_policy_allowed' => 'fixed',
           'min_rate_cap_kbps' => nil,

data/lib/aspera/node.rb

@@ -1,4 +1,5 @@
 require 'aspera/rest'
+require 'aspera/oauth'
 require 'aspera/log'
 require 'zlib'
 require 'base64'
@@ -9,10 +10,19 @@ module Aspera
     # permissions
     ACCESS_LEVELS=['delete','list','mkdir','preview','read','rename','write']
     MATCH_EXEC_PREFIX='exec:'
+    # (public) default transfer username for access key based transfers
+    ACCESS_KEY_TRANSFER_USER='xfer'
+    SSH_PORT_DEFAULT=33001
+    UDP_PORT_DEFAULT=33001
 
     # register node special token decoder
     Oauth.register_decoder(lambda{|token|JSON.parse(Zlib::Inflate.inflate(Base64.decode64(token)).partition('==SIGNATURE==').first)})
 
+    def self.set_ak_basic_token(ts,ak,secret)
+      raise "ERROR: expected xfer" unless ts['remote_user'].eql?(ACCESS_KEY_TRANSFER_USER)
+      ts['token']="Basic #{Base64.strict_encode64("#{ak}:#{secret}")}"
+    end
+
     # for access keys: provide expression to match entry in folder
     # if no prefix: regex
     # if prefix: ruby code

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: aspera-cli
 version: !ruby/object:Gem::Version
-  version: 4.3.0
+  version: 4.4.0
 platform: ruby
 authors:
 - Laurent Martin
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-10-19 00:00:00.000000000 Z
+date: 2021-11-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: xml-simple
@@ -224,6 +224,7 @@ files:
 - docs/README.erb.md
 - docs/README.md
 - docs/diagrams.txt
+- docs/doc_tools.rb
 - docs/test_env.conf
 - examples/aoc.rb
 - examples/faspex4.rb
@@ -273,7 +274,6 @@ files:
 - lib/aspera/data/7
 - lib/aspera/data_repository.rb
 - lib/aspera/environment.rb
-- lib/aspera/fasp/aoc.rb
 - lib/aspera/fasp/connect.rb
 - lib/aspera/fasp/error.rb
 - lib/aspera/fasp/error_info.rb

data/lib/aspera/fasp/aoc.rb

@@ -1,24 +0,0 @@
-require 'aspera/fasp/node'
-require 'aspera/log'
-require 'aspera/aoc.rb'
-
-module Aspera
-  module Fasp
-    class Aoc < Node
-      def initialize(aoc_options)
-        @app=aoc_options[:app] || AoC::FILES_APP
-        @api_aoc=AoC.new(aoc_options)
-        Log.log.warn("Under Development")
-        server_node_file = @api_aoc.resolve_node_file(server_home_node_file,server_folder)
-        # force node as transfer agent
-        node_api=Fasp::Node.new(@api_aoc.get_node_api(client_node_file[:node_info],scope: AoC::SCOPE_NODE_USER))
-        super(node_api)
-        # additional node to node TS info
-        @add_ts={
-          'remote_access_key'   => server_node_file[:node_info]['access_key'],
-          'destination_root_id' => server_node_file[:file_id]
-        }
-      end
-    end
-  end
-end