aspera-cli 4.2.1 → 4.5.0

data/lib/aspera/rest.rb

@@ -35,8 +35,14 @@ module Aspera
     @@insecure=false
     @@user_agent='Ruby'
     @@download_partial_suffix='.http_partial'
+    # a lambda which takes the Net::HTTP as arg, use this to change parameters
+    @@session_cb=nil
 
     public
+    def self.session_cb=(v); @@session_cb=v;Log.log.debug("session_cb => #{@@session_cb}".red);end
+
+    def self.session_cb; @@session_cb;end
+
     def self.insecure=(v); @@insecure=v;Log.log.debug("insecure => #{@@insecure}".red);end
 
     def self.insecure; @@insecure;end
@@ -85,12 +91,9 @@ module Aspera
         # this honors http_proxy env var
         @http_session=Net::HTTP.new(uri.host, uri.port)
         @http_session.use_ssl = uri.scheme.eql?('https')
-        Log.log.debug("insecure=#{@@insecure}")
         @http_session.verify_mode = OpenSSL::SSL::VERIFY_NONE if @@insecure
         @http_session.set_debug_output($stdout) if @@debug
-        if @params.has_key?(:session_cb)
-          @params[:session_cb].call(@http_session)
-        end
+        @@session_cb.call(@http_session) unless @@session_cb.nil?
         # manually start session for keep alive (if supported by server, else, session is closed every time)
         @http_session.start
       end
@@ -107,7 +110,6 @@ module Aspera
     # :username   [:basic]
     # :password   [:basic]
     # :url_creds  [:url]
-    # :session_cb a lambda which takes @http_session as arg, use this to change parameters
     # :*          [:oauth2] see Oauth class
     def initialize(a_rest_params)
       raise "ERROR: expecting Hash" unless a_rest_params.is_a?(Hash)
@@ -125,51 +127,18 @@ module Aspera
     end
 
     def oauth_token(options={})
-      raise "ERROR: not Oauth" unless @oauth.is_a?(Oauth)
+      raise "ERROR: expecting Oauth, have #{@oauth.class}" unless @oauth.is_a?(Oauth)
       return @oauth.get_authorization(options)
     end
 
-    # HTTP/S REST call
-    # call_data has keys:
-    # :auth
-    # :operation
-    # :subpath
-    # :headers
-    # :json_params
-    # :url_params
-    # :www_body_params
-    # :text_body_params
-    # :save_to_file (filepath)
-    # :return_error (bool)
-    def call(call_data)
-      raise "Hash call parameter is required (#{call_data.class})" unless call_data.is_a?(Hash)
-      Log.log.debug("accessing #{call_data[:subpath]}".red.bold.bg_green)
-      call_data[:headers]||={}
-      call_data[:headers]['User-Agent'] ||= @@user_agent
-      # defaults from @params are overriden by call dataz
-      call_data=@params.deep_merge(call_data)
-      case call_data[:auth][:type]
-      when :none
-        # no auth
-      when :basic
-        Log.log.debug("using Basic auth")
-        basic_auth_data=[call_data[:auth][:username],call_data[:auth][:password]]
-      when :oauth2
-        call_data[:headers]['Authorization']=oauth_token unless call_data[:headers].has_key?('Authorization')
-      when :url
-        call_data[:url_params]||={}
-        call_data[:auth][:url_creds].each do |key, value|
-          call_data[:url_params][key]=value
-        end
-      else raise "unsupported auth type: [#{call_data[:auth][:type]}]"
-      end
+    def build_request(call_data)
       # TODO: shall we percent encode subpath (spaces) test with access key delete with space in id
       # URI.escape()
-      uri=self.class.build_uri("#{@params[:base_url]}#{call_data[:subpath].nil? ? '' : '/'}#{call_data[:subpath]}",call_data[:url_params])
+      uri=self.class.build_uri("#{call_data[:base_url]}#{['','/'].include?(call_data[:subpath]) ? '' : '/'}#{call_data[:subpath]}",call_data[:url_params])
       Log.log.debug("URI=#{uri}")
       begin
         # instanciate request object based on string name
-        req=Object::const_get('Net::HTTP::'+call_data[:operation].capitalize).new(uri.request_uri)
+        req=Object::const_get('Net::HTTP::'+call_data[:operation].capitalize).new(uri)#.request_uri
       rescue NameError => e
         raise "unsupported operation : #{call_data[:operation]}"
       end
@@ -196,20 +165,59 @@ module Aspera
         end
       end
       # :type = :basic
-      req.basic_auth(*basic_auth_data) unless basic_auth_data.nil?
+      req.basic_auth(call_data[:auth][:username],call_data[:auth][:password]) if call_data[:auth][:type].eql?(:basic)
+      return req
+    end
 
+    # HTTP/S REST call
+    # call_data has keys:
+    # :auth
+    # :operation
+    # :subpath
+    # :headers
+    # :json_params
+    # :url_params
+    # :www_body_params
+    # :text_body_params
+    # :save_to_file (filepath)
+    # :return_error (bool)
+    # :redirect_max (int)
+    def call(call_data)
+      raise "Hash call parameter is required (#{call_data.class})" unless call_data.is_a?(Hash)
+      call_data[:subpath]='' if call_data[:subpath].nil?
+      Log.log.debug("accessing #{call_data[:subpath]}".red.bold.bg_green)
+      call_data[:headers]||={}
+      call_data[:headers]['User-Agent'] ||= @@user_agent
+      # defaults from @params are overriden by call data
+      call_data=@params.deep_merge(call_data)
+      case call_data[:auth][:type]
+      when :none
+        # no auth
+      when :basic
+        Log.log.debug("using Basic auth")
+        # done in build_req
+      when :oauth2
+        call_data[:headers]['Authorization']=oauth_token unless call_data[:headers].has_key?('Authorization')
+      when :url
+        call_data[:url_params]||={}
+        call_data[:auth][:url_creds].each do |key, value|
+          call_data[:url_params][key]=value
+        end
+      else raise "unsupported auth type: [#{call_data[:auth][:type]}]"
+      end
+      req=build_request(call_data)
       Log.log.debug("call_data = #{call_data}")
       result={:http=>nil}
       # start a block to be able to retry the actual HTTP request
       begin
         # we try the call, and will retry only if oauth, as we can, first with refresh, and then re-auth if refresh is bad
         oauth_tries ||= 2
-        tries_remain_redirect||=4
+        tries_remain_redirect||=call_data[:redirect_max].nil? ? 0 : call_data[:redirect_max].to_i
         Log.log.debug("send request")
         # make http request (pipelined)
         http_session.request(req) do |response|
           result[:http] = response
-          if call_data.has_key?(:save_to_file)
+          if call_data.has_key?(:save_to_file) and result[:http].code.to_s.start_with?('2')
             total_size=result[:http]['Content-Length'].to_i
             progress=ProgressBar.create(
             :format     => '%a %B %p%% %r KB/sec %e',
@@ -238,7 +246,7 @@ module Aspera
             progress=nil
           end # save_to_file
         end
-        # sometimes there is a ITF8 char (e.g. (c) )
+        # sometimes there is a UTF8 char (e.g. (c) )
         result[:http].body.force_encoding("UTF-8") if result[:http].body.is_a?(String)
         Log.log.debug("result: body=#{result[:http].body}")
         result_mime=(result[:http]['Content-Type']||'text/plain').split(';').first
@@ -270,9 +278,18 @@ module Aspera
           if tries_remain_redirect > 0
             tries_remain_redirect-=1
             Log.log.info("URL is moved: #{e.response['location']}")
-            raise e
-            # TODO: rebuild request with new location
-            #retry
+            current_uri=URI.parse(call_data[:base_url])
+            redir_uri=URI.parse(e.response['location'])
+            call_data[:base_url]=e.response['location']
+            call_data[:subpath]=''
+            if current_uri.host.eql?(redir_uri.host) and current_uri.port.eql?(redir_uri.port)
+              req=build_request(call_data)
+              retry
+            else
+              # change host
+              Log.log.info("Redirect changes host: #{current_uri.host} -> #{redir_uri.host}")
+              return self.class.new(call_data).call(call_data)
+            end
           else
             raise "too many redirect"
           end
@@ -284,7 +301,6 @@ module Aspera
       end # begin request
       Log.log.debug("result=#{result}")
       return result
-
     end
 
     #

data/lib/aspera/sync.rb

@@ -1,48 +1,53 @@
+require 'aspera/command_line_builder'
+
 module Aspera
   # builds command line arg for async
   class Sync
-    private
     INSTANCE_PARAMS=
     {
-      'alt_logdir'           => { :type => :opt_with_arg, :accepted_types=>String},
-      'watchd'               => { :type => :opt_with_arg, :accepted_types=>String},
-      'apply_local_docroot'  => { :type => :opt_without_arg},
-      'quiet'                => { :type => :opt_without_arg},
+      'alt_logdir'           => { :cltype => :opt_with_arg, :accepted_types=>:string},
+      'watchd'               => { :cltype => :opt_with_arg, :accepted_types=>:string},
+      'apply_local_docroot'  => { :cltype => :opt_without_arg},
+      'quiet'                => { :cltype => :opt_without_arg},
     }
     SESSION_PARAMS=
     {
-      'name'                 => { :type => :opt_with_arg, :accepted_types=>String},
-      'local_dir'            => { :type => :opt_with_arg, :accepted_types=>String},
-      'remote_dir'           => { :type => :opt_with_arg, :accepted_types=>String},
-      'local_db_dir'         => { :type => :opt_with_arg, :accepted_types=>String},
-      'remote_db_dir'        => { :type => :opt_with_arg, :accepted_types=>String},
-      'host'                 => { :type => :opt_with_arg, :accepted_types=>String},
-      'user'                 => { :type => :opt_with_arg, :accepted_types=>String},
-      'private_key_path'     => { :type => :opt_with_arg, :accepted_types=>String},
-      'direction'            => { :type => :opt_with_arg, :accepted_types=>String},
-      'checksum'             => { :type => :opt_with_arg, :accepted_types=>String},
-      'tcp_port'             => { :type => :opt_with_arg, :accepted_types=>Integer},
-      'rate_policy'          => { :type => :opt_with_arg, :accepted_types=>String},
-      'target_rate'          => { :type => :opt_with_arg, :accepted_types=>String},
-      'cooloff'              => { :type => :opt_with_arg, :accepted_types=>Integer},
-      'pending_max'          => { :type => :opt_with_arg, :accepted_types=>Integer},
-      'scan_intensity'       => { :type => :opt_with_arg, :accepted_types=>String},
-      'cipher'               => { :type => :opt_with_arg, :accepted_types=>String},
-      'transfer_threads'     => { :type => :opt_with_arg, :accepted_types=>Integer},
-      'preserve_time'        => { :type => :opt_without_arg},
-      'preserve_access_time' => { :type => :opt_without_arg},
-      'preserve_modification_time' => { :type => :opt_without_arg},
-      'preserve_uid'         => { :type => :opt_without_arg},
-      'preserve_gid'         => { :type => :opt_without_arg},
-      'create_dir'           => { :type => :opt_without_arg},
-      'reset'                => { :type => :opt_without_arg},
+      'name'                 => { :cltype => :opt_with_arg, :accepted_types=>:string},
+      'local_dir'            => { :cltype => :opt_with_arg, :accepted_types=>:string},
+      'remote_dir'           => { :cltype => :opt_with_arg, :accepted_types=>:string},
+      'local_db_dir'         => { :cltype => :opt_with_arg, :accepted_types=>:string},
+      'remote_db_dir'        => { :cltype => :opt_with_arg, :accepted_types=>:string},
+      'host'                 => { :cltype => :opt_with_arg, :accepted_types=>:string},
+      'user'                 => { :cltype => :opt_with_arg, :accepted_types=>:string},
+      'private_key_path'     => { :cltype => :opt_with_arg, :accepted_types=>:string},
+      'direction'            => { :cltype => :opt_with_arg, :accepted_types=>:string},
+      'checksum'             => { :cltype => :opt_with_arg, :accepted_types=>:string},
+      'tcp_port'             => { :cltype => :opt_with_arg, :accepted_types=>:int},
+      'rate_policy'          => { :cltype => :opt_with_arg, :accepted_types=>:string},
+      'target_rate'          => { :cltype => :opt_with_arg, :accepted_types=>:string},
+      'cooloff'              => { :cltype => :opt_with_arg, :accepted_types=>:int},
+      'pending_max'          => { :cltype => :opt_with_arg, :accepted_types=>:int},
+      'scan_intensity'       => { :cltype => :opt_with_arg, :accepted_types=>:string},
+      'cipher'               => { :cltype => :opt_with_arg, :accepted_types=>:string},
+      'transfer_threads'     => { :cltype => :opt_with_arg, :accepted_types=>:int},
+      'preserve_time'        => { :cltype => :opt_without_arg},
+      'preserve_access_time' => { :cltype => :opt_without_arg},
+      'preserve_modification_time' => { :cltype => :opt_without_arg},
+      'preserve_uid'         => { :cltype => :opt_without_arg},
+      'preserve_gid'         => { :cltype => :opt_without_arg},
+      'create_dir'           => { :cltype => :opt_without_arg},
+      'reset'                => { :cltype => :opt_without_arg},
       # note: only one env var, but multiple sessions... may be a problem
-      'remote_password'      => { :type => :envvar, :variable=>'ASPERA_SCP_PASS'},
-      'cookie'               => { :type => :envvar, :variable=>'ASPERA_SCP_COOKIE'},
-      'token'                => { :type => :envvar, :variable=>'ASPERA_SCP_TOKEN'},
-      'license'              => { :type => :envvar, :variable=>'ASPERA_SCP_LICENSE'},
+      'remote_password'      => { :cltype => :envvar, :clvarname=>'ASPERA_SCP_PASS'},
+      'cookie'               => { :cltype => :envvar, :clvarname=>'ASPERA_SCP_COOKIE'},
+      'token'                => { :cltype => :envvar, :clvarname=>'ASPERA_SCP_TOKEN'},
+      'license'              => { :cltype => :envvar, :clvarname=>'ASPERA_SCP_LICENSE'},
     }
-    public
+
+    Aspera::CommandLineBuilder.normalize_description(INSTANCE_PARAMS)
+    Aspera::CommandLineBuilder.normalize_description(SESSION_PARAMS)
+
+    private_constant :INSTANCE_PARAMS,:SESSION_PARAMS
 
     def initialize(sync_params)
       @sync_params=sync_params

data/lib/aspera/timer_limiter.rb

@@ -0,0 +1,22 @@
+module Aspera
+  # used to throttle logs
+  class TimerLimiter
+    # @param delay in seconds (float)
+    def initialize(delay)
+      @delay=delay
+      @last_time=nil
+      @count=0
+    end
+
+    def trigger?
+      old_time=@last_time
+      @last_time=Time.now.to_f
+      @count+=1
+      if old_time.nil? or (@last_time-old_time)>@delay
+        @count=0
+        return true
+      end
+      return false
+    end
+  end
+end

metadata

@@ -1,17 +1,31 @@
 --- !ruby/object:Gem::Specification
 name: aspera-cli
 version: !ruby/object:Gem::Version
-  version: 4.2.1
+  version: 4.5.0
 platform: ruby
 authors:
 - Laurent Martin
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-09-01 00:00:00.000000000 Z
+date: 2021-12-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: xml-simple
+  name: execjs
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: grpc
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -39,19 +53,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.0'
 - !ruby/object:Gem::Dependency
-  name: ruby-progressbar
+  name: mimemagic
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '0.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '0.3'
 - !ruby/object:Gem::Dependency
   name: net-ssh
   requirement: !ruby/object:Gem::Requirement
@@ -67,7 +81,21 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '6.0'
 - !ruby/object:Gem::Dependency
-  name: execjs
+  name: ruby-progressbar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: rubyzip
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -81,47 +109,47 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.0'
 - !ruby/object:Gem::Dependency
-  name: terminal-table
+  name: security
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.8'
+        version: '0.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.8'
+        version: '0.0'
 - !ruby/object:Gem::Dependency
-  name: tty-spinner
+  name: terminal-table
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.9'
+        version: '1.8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.9'
+        version: '1.8'
 - !ruby/object:Gem::Dependency
-  name: rubyzip
+  name: tty-spinner
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '0.9'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '0.9'
 - !ruby/object:Gem::Dependency
   name: websocket
   requirement: !ruby/object:Gem::Requirement
@@ -150,6 +178,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.3'
+- !ruby/object:Gem::Dependency
+  name: webrick
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: xml-simple
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -195,10 +251,7 @@ dependencies:
 description: Command line interface for IBM Aspera products
 email:
 - laurent.martin.aspera@fr.ibm.com
-executables:
-- ascli
-- asession
-- dascli
+executables: []
 extensions: []
 extra_rdoc_files: []
 files:
@@ -210,14 +263,13 @@ files:
 - docs/README.erb.md
 - docs/README.md
 - docs/diagrams.txt
+- docs/doc_tools.rb
 - docs/test_env.conf
-- docs/transfer_spec.html
 - examples/aoc.rb
 - examples/faspex4.rb
 - examples/proxy.pac
 - examples/transfer.rb
 - lib/aspera/aoc.rb
-- lib/aspera/api_detector.rb
 - lib/aspera/ascmd.rb
 - lib/aspera/ats_api.rb
 - lib/aspera/cli/basic_auth_plugin.rb
@@ -260,21 +312,26 @@ files:
 - lib/aspera/data/7
 - lib/aspera/data_repository.rb
 - lib/aspera/environment.rb
-- lib/aspera/fasp/aoc.rb
-- lib/aspera/fasp/connect.rb
+- lib/aspera/fasp/agent_base.rb
+- lib/aspera/fasp/agent_connect.rb
+- lib/aspera/fasp/agent_direct.rb
+- lib/aspera/fasp/agent_httpgw.rb
+- lib/aspera/fasp/agent_node.rb
+- lib/aspera/fasp/agent_trsdk.rb
+- lib/aspera/fasp/default.rb
 - lib/aspera/fasp/error.rb
 - lib/aspera/fasp/error_info.rb
-- lib/aspera/fasp/http_gw.rb
 - lib/aspera/fasp/installation.rb
 - lib/aspera/fasp/listener.rb
-- lib/aspera/fasp/local.rb
-- lib/aspera/fasp/manager.rb
-- lib/aspera/fasp/node.rb
 - lib/aspera/fasp/parameters.rb
+- lib/aspera/fasp/parameters.yaml
 - lib/aspera/fasp/resume_policy.rb
 - lib/aspera/fasp/uri.rb
 - lib/aspera/faspex_gw.rb
 - lib/aspera/hash_ext.rb
+- lib/aspera/id_generator.rb
+- lib/aspera/keychain/encrypted_hash.rb
+- lib/aspera/keychain/macos_security.rb
 - lib/aspera/log.rb
 - lib/aspera/nagios.rb
 - lib/aspera/node.rb
@@ -294,10 +351,10 @@ files:
 - lib/aspera/rest_call_error.rb
 - lib/aspera/rest_error_analyzer.rb
 - lib/aspera/rest_errors_aspera.rb
-- lib/aspera/secrets.rb
 - lib/aspera/ssh.rb
 - lib/aspera/sync.rb
 - lib/aspera/temp_file_manager.rb
+- lib/aspera/timer_limiter.rb
 - lib/aspera/uri_reader.rb
 - lib/aspera/web_auth.rb
 homepage: https://github.com/IBM/aspera-cli