aspera-cli 4.7.0 → 4.9.0

data/lib/aspera/faspex_gw.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require 'aspera/log'
 require 'aspera/aoc'
 require 'aspera/fasp/transfer_spec'
@@ -13,14 +14,14 @@ module Aspera
   # this class answers the Faspex /send API and creates a package on Aspera on Cloud
   class FaspexGW
     class FxGwServlet < WEBrick::HTTPServlet::AbstractServlet
-      def initialize(_server,a_aoc_api_user,a_workspace_id)
+      def initialize(_server, a_aoc_api_user, a_workspace_id)
         super
-        @aoc_api_user=a_aoc_api_user
-        @aoc_workspace_id=a_workspace_id
+        @aoc_api_user = a_aoc_api_user
+        @aoc_workspace_id = a_workspace_id
       end
 
       # parameters from user to Faspex API call
-      #{"delivery":{"use_encryption_at_rest":false,"note":"note","sources":[{"paths":["file1"]}],"title":"my title","recipients":["email1"],"send_upload_result":true}}
+      # {"delivery":{"use_encryption_at_rest":false,"note":"note","sources":[{"paths":["file1"]}],"title":"my title","recipients":["email1"],"send_upload_result":true}}
       #    {
       #      "delivery"=>{
       #      "use_encryption_at_rest"=>false,
@@ -33,127 +34,137 @@ module Aspera
       #    }
       def process_faspex_send(request, response)
         raise 'no payload' if request.body.nil?
-        faspex_pkg_parameters=JSON.parse(request.body)
-        faspex_pkg_delivery=faspex_pkg_parameters['delivery']
+
+        faspex_pkg_parameters = JSON.parse(request.body)
+        faspex_pkg_delivery = faspex_pkg_parameters['delivery']
         Log.log.debug("faspex pkg create parameters=#{faspex_pkg_parameters}")
 
         # get recipient ids
-        files_pkg_recipients=[]
+        files_pkg_recipients = []
         faspex_pkg_delivery['recipients'].each do |recipient_email|
-          user_lookup=@aoc_api_user.read('contacts',{'current_workspace_id'=>@aoc_workspace_id,'q'=>recipient_email})[:data]
-          raise StandardError,"no such unique user: #{recipient_email} / #{user_lookup}" unless !user_lookup.nil? && user_lookup.length.eql?(1)
-          recipient_user_info=user_lookup.first
-          files_pkg_recipients.push({'id'=>recipient_user_info['source_id'],'type'=>recipient_user_info['source_type']})
+          user_lookup = @aoc_api_user.read('contacts',
+            { 'current_workspace_id' => @aoc_workspace_id, 'q' => recipient_email })[:data]
+          raise StandardError,
+            "no such unique user: #{recipient_email} / #{user_lookup}" unless !user_lookup.nil? && user_lookup.length.eql?(1)
+
+          recipient_user_info = user_lookup.first
+          files_pkg_recipients.push({
+            'id'   => recipient_user_info['source_id'],
+            'type' => recipient_user_info['source_type']
+          })
         end
 
         #  create a new package with one file
-        the_package=@aoc_api_user.create('packages',{
-          'file_names'  =>faspex_pkg_delivery['sources'][0]['paths'],
-          'name'        =>faspex_pkg_delivery['title'],
-          'note'        =>faspex_pkg_delivery['note'],
-          'recipients'  =>files_pkg_recipients,
-          'workspace_id'=>@aoc_workspace_id})[:data]
+        the_package = @aoc_api_user.create('packages', {
+          'file_names'   => faspex_pkg_delivery['sources'][0]['paths'],
+          'name'         => faspex_pkg_delivery['title'],
+          'note'         => faspex_pkg_delivery['note'],
+          'recipients'   => files_pkg_recipients,
+          'workspace_id' => @aoc_workspace_id
+        })[:data]
 
         #  get node information for the node on which package must be created
-        node_info=@aoc_api_user.read("nodes/#{the_package['node_id']}")[:data]
+        node_info = @aoc_api_user.read("nodes/#{the_package['node_id']}")[:data]
 
         #  get transfer token (for node)
-        node_auth_bearer_token=@aoc_api_user.oauth_token(scope: AoC.node_scope(node_info['access_key'],AoC::SCOPE_NODE_USER))
+        node_auth_bearer_token = @aoc_api_user.oauth_token(scope: AoC.node_scope(node_info['access_key'],
+          AoC::SCOPE_NODE_USER))
 
         # tell Files what to expect in package: 1 transfer (can also be done after transfer)
-        @aoc_api_user.update("packages/#{the_package['id']}",{'sent'=>true,'transfers_expected'=>1})
+        @aoc_api_user.update("packages/#{the_package['id']}", { 'sent' => true, 'transfers_expected' => 1 })
 
         # to return an error:
         # response.status=400
         # return 'ERROR HERE'
 
         # TODO: check about xfer_*
-        ts_tags={
+        ts_tags = {
           'aspera' => {
-          'files'      => { 'package_id' => the_package['id'], 'package_operation' => 'upload' },
-          'node'       => { 'access_key' => node_info['access_key'], 'file_id' => the_package['contents_file_id'] },
-          'xfer_id'    => SecureRandom.uuid,
-          'xfer_retry' => 3600 } }
+            'files'      => { 'package_id' => the_package['id'], 'package_operation' => 'upload' },
+            'node'       => { 'access_key' => node_info['access_key'], 'file_id' => the_package['contents_file_id'] },
+            'xfer_id'    => SecureRandom.uuid,
+            'xfer_retry' => 3600
+          }
+        }
         # this transfer spec is for transfer to AoC
-        faspex_transfer_spec={
-          'direction'   => 'send',
-          'remote_host' => node_info['host'],
-          'remote_user' => Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER,
-          'ssh_port'    => Fasp::TransferSpec::SSH_PORT,
-          'fasp_port'   => Fasp::TransferSpec::UDP_PORT,
-          'tags'        => ts_tags,
-          'token'       => node_auth_bearer_token,
-          'paths'       => [{'destination' => '/'}],
-          'cookie'      => 'unused',
-          'create_dir'  => true,
-          'rate_policy' => 'fair',
-          'rate_policy_allowed'  => 'fixed',
-          'min_rate_cap_kbps'    => nil,
-          'min_rate_kbps'        => 0,
+        faspex_transfer_spec = {
+          'direction'              => 'send',
+          'remote_host'            => node_info['host'],
+          'remote_user'            => Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER,
+          'ssh_port'               => Fasp::TransferSpec::SSH_PORT,
+          'fasp_port'              => Fasp::TransferSpec::UDP_PORT,
+          'tags'                   => ts_tags,
+          'token'                  => node_auth_bearer_token,
+          'paths'                  => [{ 'destination' => '/' }],
+          'cookie'                 => 'unused',
+          'create_dir'             => true,
+          'rate_policy'            => 'fair',
+          'rate_policy_allowed'    => 'fixed',
+          'min_rate_cap_kbps'      => nil,
+          'min_rate_kbps'          => 0,
           'target_rate_percentage' => nil,
-          'lock_target_rate'     => nil,
-          'fasp_url'             => 'unused',
-          'lock_min_rate'        => true,
-          'lock_rate_policy'     => true,
-          'source_root'          => '',
-          'content_protection'   => nil,
-          'target_rate_cap_kbps' => 20_000, # TODO: is this value useful ?
-          'target_rate_kbps'     => 10_000, # TODO: get from where?
-          'cipher'               => 'aes-128',
-          'cipher_allowed'       => nil,
-          'http_fallback'        => false,
-          'http_fallback_port'   => nil,
-          'https_fallback_port'  => nil,
-          'destination_root'     => '/'
+          'lock_target_rate'       => nil,
+          'fasp_url'               => 'unused',
+          'lock_min_rate'          => true,
+          'lock_rate_policy'       => true,
+          'source_root'            => '',
+          'content_protection'     => nil,
+          'target_rate_cap_kbps'   => 20_000, # TODO: is this value useful ?
+          'target_rate_kbps'       => 10_000, # TODO: get from where?
+          'cipher'                 => 'aes-128',
+          'cipher_allowed'         => nil,
+          'http_fallback'          => false,
+          'http_fallback_port'     => nil,
+          'https_fallback_port'    => nil,
+          'destination_root'       => '/'
         }
         # but we place it in a Faspex package creation response
-        faspex_package_create_result={
-          'links' => {'status' => 'unused'},
+        faspex_package_create_result = {
+          'links'         => { 'status' => 'unused' },
           'xfer_sessions' => [faspex_transfer_spec]
         }
         Log.log.info("faspex_package_create_result=#{faspex_package_create_result}")
-        response.status=200
+        response.status = 200
         response.content_type = 'application/json'
-        response.body=JSON.generate(faspex_package_create_result)
+        response.body = JSON.generate(faspex_package_create_result)
       end
 
-      def do_GET(request, response) # rubocop:disable Naming/MethodName
+      def do_GET(request, response)
         case request.path
         when '/aspera/faspex/send'
           process_faspex_send(request, response)
         else
-          response.status=400
-          return 'ERROR HERE'
+          response.status = 400
+          'ERROR HERE'
         end
       end
     end # FxGwServlet
 
     class NewUserServlet < WEBrick::HTTPServlet::AbstractServlet
-      def do_GET(request, response) # rubocop:disable Naming/MethodName
+      def do_GET(request, response)
         case request.path
         when '/newuser'
-          response.status=200
+          response.status = 200
           response.content_type = 'text/html'
-          response.body='<html><body>hello world</body></html>'
+          response.body = '<html><body>hello world</body></html>'
         else
           raise "unsupported path: [#{request.path}]"
         end
       end
     end
 
-    def initialize(a_aoc_api_user,a_workspace_id)
+    def initialize(a_aoc_api_user, a_workspace_id)
       webrick_options = {
-        Port:                9443,
-        Logger:              Log.log,
-        SSLEnable:           true,
-        SSLVerifyClient:     OpenSSL::SSL::VERIFY_NONE,
-        SSLCertName:         [['CN',WEBrick::Utils.getservername]]
+        Port:        9443,
+        Logger:      Log.log,
+        SSLEnable:   true,
+        SSLCertName: [['CN', WEBrick::Utils.getservername]]
       }
       Log.log.info("Server started on port #{webrick_options[:Port]}")
       @server = WEBrick::HTTPServer.new(webrick_options)
-      @server.mount('/aspera/faspex', FxGwServlet,a_aoc_api_user,a_workspace_id)
+      @server.mount('/aspera/faspex', FxGwServlet, a_aoc_api_user, a_workspace_id)
       @server.mount('/newuser', NewUserServlet)
-      trap('INT') {@server.shutdown}
+      trap('INT') { @server.shutdown }
     end
 
     def start_server

data/lib/aspera/hash_ext.rb

@@ -1,29 +1,39 @@
 # frozen_string_literal: true
-# for older rubies
-unless Hash.method_defined?(:dig)
-  class Hash
-    def dig(*path)
-      path.inject(self) do |location, key|
-        location.respond_to?(:keys) ? location[key] : nil
-      end
-    end
-  end
-end
 
 class ::Hash
   def deep_merge(second)
-    merge(second){|_key,v1,v2|Hash===v1&&Hash===v2 ? v1.deep_merge(v2) : v2}
+    merge(second){|_key,v1,v2|Hash === v1 && Hash === v2 ? v1.deep_merge(v2) : v2}
   end
 
   def deep_merge!(second)
-    merge!(second){|_key,v1,v2|Hash===v1&&Hash===v2 ? v1.deep_merge!(v2) : v2}
+    merge!(second){|_key,v1,v2|Hash === v1 && Hash === v2 ? v1.deep_merge!(v2) : v2}
+  end
+end
+
+# in 2.5
+unless Hash.method_defined?(:transform_keys)
+  class Hash
+    def transform_keys
+      return each_with_object({}){|(k,v),memo|memo[yield(k)]=v} if block_given?
+      raise 'missing block'
+    end
   end
 end
 
+# rails
 unless Hash.method_defined?(:symbolize_keys)
   class Hash
     def symbolize_keys
-      return each_with_object({}){|(k,v),memo| memo[k.to_sym] = v; }
+      return transform_keys(&:to_sym)
+    end
+  end
+end
+
+# rails
+unless Hash.method_defined?(:stringify_keys)
+  class Hash
+    def stringify_keys
+      return transform_keys(&:to_s)
     end
   end
 end

data/lib/aspera/id_generator.rb

@@ -1,23 +1,26 @@
 # frozen_string_literal: true
+
 require 'uri'
 
 module Aspera
   class IdGenerator
-    ID_SEPARATOR='_'
-    WINDOWS_PROTECTED_CHAR=%r{[/:"<>\\*?]}
-    PROTECTED_CHAR_REPLACE='_'
+    ID_SEPARATOR = '_'
+    WINDOWS_PROTECTED_CHAR = %r{[/:"<>\\*?]}.freeze
+    PROTECTED_CHAR_REPLACE = '_'
     private_constant :ID_SEPARATOR,:PROTECTED_CHAR_REPLACE,:WINDOWS_PROTECTED_CHAR
-    def self.from_list(object_id)
-      if object_id.is_a?(Array)
-        object_id=object_id.reject(&:nil?).map do |i|
-          i.is_a?(String) && i.start_with?('https://') ? URI.parse(i).host : i.to_s
-        end.join(ID_SEPARATOR)
+    class << self
+      def from_list(object_id)
+        if object_id.is_a?(Array)
+          object_id = object_id.compact.map do |i|
+            i.is_a?(String) && i.start_with?('https://') ? URI.parse(i).host : i.to_s
+          end.join(ID_SEPARATOR)
+        end
+        raise 'id must be a String' unless object_id.is_a?(String)
+        return object_id.
+            gsub(WINDOWS_PROTECTED_CHAR,PROTECTED_CHAR_REPLACE). # remove windows forbidden chars
+            gsub('.',PROTECTED_CHAR_REPLACE).  # keep dot for extension only (nicer)
+            downcase
       end
-      raise 'id must be a String' unless object_id.is_a?(String)
-      return object_id.
-        gsub(WINDOWS_PROTECTED_CHAR,PROTECTED_CHAR_REPLACE). # remove windows forbidden chars
-        gsub('.',PROTECTED_CHAR_REPLACE).  # keep dot for extension only (nicer)
-        downcase
     end
   end
 end

data/lib/aspera/keychain/encrypted_hash.rb

@@ -1,11 +1,13 @@
 # frozen_string_literal: true
+
+require 'aspera/hash_ext'
 require 'openssl'
 
 module Aspera
   module Keychain
     class SimpleCipher
       def initialize(key)
-        @key=Digest::SHA1.hexdigest(key)[0..23]
+        @key = Digest::SHA1.hexdigest(key+('*'*23))[0..23]
         @cipher = OpenSSL::Cipher.new('DES-EDE3-CBC')
       end
 
@@ -26,93 +28,106 @@ module Aspera
 
     # Manage secrets in a simple Hash
     class EncryptedHash
-      SEPARATOR='%'
+      SEPARATOR = '%'
+      ACCEPTED_KEYS = %i[username url secret description].freeze
       private_constant :SEPARATOR
+      attr_reader :legacy_detected
       def initialize(values)
         raise 'values shall be Hash' unless values.is_a?(Hash)
-        @all_secrets=values
+        @all_secrets = values
+      end
+
+      def identifier(options)
+        return options[:username] if options[:url].to_s.empty?
+        %i[url username].map{|s|options[s]}.join(SEPARATOR)
       end
 
       def set(options)
         raise 'options shall be Hash' unless options.is_a?(Hash)
-        unsupported=options.keys-[:username,:url,:secret,:description]
+        unsupported = options.keys - ACCEPTED_KEYS
         raise "unsupported options: #{unsupported}" unless unsupported.empty?
-        username=options[:username]
+        username = options[:username]
         raise 'options shall have username' if username.nil?
-        url=options[:url]
+        url = options[:url]
         raise 'options shall have username' if url.nil?
-        secret=options[:secret]
+        secret = options[:secret]
         raise 'options shall have secret' if secret.nil?
-        key=[url,username].join(SEPARATOR)
+        key = identifier(options)
         raise "secret #{key} already exist, delete first" if @all_secrets.has_key?(key)
-        obj={username: username, url: url, secret: SimpleCipher.new(key).encrypt(secret)}
-        obj[:description]=options[:description] if options.has_key?(:description)
-        @all_secrets[key]=obj
+        obj = {username: username, url: url, secret: SimpleCipher.new(key).encrypt(secret)}
+        obj[:description] = options[:description] if options.has_key?(:description)
+        @all_secrets[key] = obj.stringify_keys
         nil
       end
 
       def list
-        result=[]
-        @all_secrets.each do |k,v|
-          case v
-          when String
-            o={username: k, url: '', description: ''}
-          when Hash
-            o=v.clone
-            o.delete(:secret)
-            o[:description]||=''
-          else raise 'error'
-          end
-          o[:description]=v[:description] if v.is_a?(Hash) && v[:description].is_a?(String)
-          result.push(o)
+        result = []
+        legacy_detected=false
+        @all_secrets.each do |name,value|
+          normal = # normalized version
+            case value
+            when String
+              legacy_detected=true
+              {username: name, url: '', secret: value}
+            when Hash then value.symbolize_keys
+            else raise 'error secret must be String (legacy) or Hash (new)'
+            end
+          normal[:description] = '' unless normal.has_key?(:description)
+          extraneous_keys=normal.keys - ACCEPTED_KEYS
+          Log.log.error("wrongs keys in secret hash: #{extraneous_keys.map(&:to_s).join(',')}") unless extraneous_keys.empty?
+          result.push(normal)
         end
+        Log.log.warn('Legacy vault format detected in config file, please refer to documentation to convert to new format.') if legacy_detected
         return result
       end
 
       def delete(options)
         raise 'options shall be Hash' unless options.is_a?(Hash)
-        unsupported=options.keys-[:username,:url]
+        unsupported = options.keys - %i[username url]
         raise "unsupported options: #{unsupported}" unless unsupported.empty?
-        username=options[:username]
+        username = options[:username]
         raise 'options shall have username' if username.nil?
-        url=options[:url]
-        key=nil
+        url = options[:url]
+        key = nil
         if !url.nil?
-          extk=[url,username].join(SEPARATOR)
-          key=extk if @all_secrets.has_key?(extk)
+          extk = identifier(options)
+          key = extk if @all_secrets.has_key?(extk)
         end
         # backward compatibility: TODO: remove in future ? (make url mandatory ?)
-        key=username if key.nil? && @all_secrets.has_key?(username)
+        key = username if key.nil? && @all_secrets.has_key?(username)
         raise 'no such secret' if key.nil?
         @all_secrets.delete(key)
       end
 
       def get(options)
         raise 'options shall be Hash' unless options.is_a?(Hash)
-        unsupported=options.keys-[:username,:url]
+        unsupported = options.keys - %i[username url]
         raise "unsupported options: #{unsupported}" unless unsupported.empty?
-        username=options[:username]
+        username = options[:username]
         raise 'options shall have username' if username.nil?
-        url=options[:url]
-        val=nil
+        url = options[:url]
+        info = nil
         if !url.nil?
-          val=@all_secrets[[url,username].join(SEPARATOR)]
+          info = @all_secrets[identifier(options)]
         end
         # backward compatibility: TODO: remove in future ? (make url mandatory ?)
-        if val.nil?
-          val=@all_secrets[username]
+        if info.nil?
+          info = @all_secrets[username]
         end
-        result=options.clone
-        case val
+        result = options.clone
+        case info
         when NilClass
-          raise 'no such secret'
+          raise "no such secret: [#{url}|#{username}] in #{@all_secrets.keys.join(',')}"
         when String
-          result.merge!({secret: val, description: ''})
+          result[:secret] = info
+          result[:description] = ''
         when Hash
-          key=[url,username].join(SEPARATOR)
-          plain=SimpleCipher.new(key).decrypt(val[:secret])
-          result.merge!({secret: plain, description: val[:description]})
-        else raise 'error'
+          info=info.symbolize_keys
+          key = identifier(options)
+          plain = SimpleCipher.new(key).decrypt(info[:secret]) rescue info[:secret]
+          result[:secret] = plain
+          result[:description] = info[:description]
+        else raise "#{info.class} is not an expected type"
         end
         return result
       end

data/lib/aspera/keychain/macos_security.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require 'security'
 
 # enhance the gem to support other keychains
@@ -6,7 +7,7 @@ module Security
   class Keychain
     class << self
       def by_name(name)
-        keychains_from_output('security list-keychains').select{|kc|kc.filename.end_with?("/#{name}.keychain-db")}.first
+        keychains_from_output('security list-keychains').find{|kc|kc.filename.end_with?("/#{name}.keychain-db")}
       end
     end
   end
@@ -14,20 +15,20 @@ module Security
   class Password
     class << self
       # add some login to original method
-      alias orig_flags_for_options flags_for_options
+      alias_method :orig_flags_for_options, :flags_for_options
       def flags_for_options(options = {})
-        keychain=options.delete(:keychain)
-        url=options.delete(:url)
+        keychain = options.delete(:keychain)
+        url = options.delete(:url)
         if !url.nil?
-          uri=URI.parse(url)
+          uri = URI.parse(url)
           raise 'only https' unless uri.scheme.eql?('https')
-          options[:r]='htps'
+          options[:r] = 'htps'
           raise 'host required in URL' if uri.host.nil?
-          options[:s]=uri.host
-          options[:p]=uri.path unless ['','/'].include?(uri.path)
-          options[:P]=uri.port unless uri.port.eql?(443) && !url.include?(':443/')
+          options[:s] = uri.host
+          options[:p] = uri.path unless ['','/'].include?(uri.path)
+          options[:P] = uri.port unless uri.port.eql?(443) && !url.include?(':443/')
         end
-        flags=[orig_flags_for_options(options)]
+        flags = [orig_flags_for_options(options)]
         flags.push(keychain.filename) unless keychain.nil?
         flags.join(' ')
       end
@@ -40,35 +41,36 @@ module Aspera
     # keychain based on macOS keychain, using `security` cmmand line
     class MacosSecurity
       def initialize(name=nil)
-        @keychain=name.nil? ? Security::Keychain.default_keychain : Security::Keychain.by_name(name)
+        @keychain = name.nil? ? Security::Keychain.default_keychain : Security::Keychain.by_name(name)
         raise "no such keychain #{name}" if @keychain.nil?
       end
 
       def set(options)
         raise 'options shall be Hash' unless options.is_a?(Hash)
-        unsupported=options.keys-[:username,:url,:secret,:description]
+        unsupported = options.keys - %i[username url secret description]
         raise "unsupported options: #{unsupported}" unless unsupported.empty?
-        username=options[:username]
+        username = options[:username]
         raise 'options shall have username' if username.nil?
-        url=options[:url]
+        url = options[:url]
         raise 'options shall have url' if url.nil?
-        secret=options[:secret]
+        secret = options[:secret]
         raise 'options shall have secret' if secret.nil?
         raise 'set not implemented'
       end
 
       def get(options)
         raise 'options shall be Hash' unless options.is_a?(Hash)
-        unsupported=options.keys-[:username,:url]
+        unsupported = options.keys - %i[username url]
         raise "unsupported options: #{unsupported}" unless unsupported.empty?
-        username=options[:username]
+        username = options[:username]
         raise 'options shall have username' if username.nil?
-        url=options[:url]
+        url = options[:url]
         raise 'options shall have url' if url.nil?
-        info=Security::InternetPassword.find(keychain: @keychain, url: url, account: username)
+        info = Security::InternetPassword.find(keychain: @keychain, url: url, account: username)
         raise 'not found' if info.nil?
-        result=options.clone
-        result.merge!({secret: info.password, description: info.attributes['icmt']})
+        result = options.clone
+        result[:secret] = info.password
+        result[:description] = info.attributes['icmt']
         return result
       end
 
@@ -78,11 +80,11 @@ module Aspera
 
       def delete(options)
         raise 'options shall be Hash' unless options.is_a?(Hash)
-        unsupported=options.keys-[:username,:url]
+        unsupported = options.keys - %i[username url]
         raise "unsupported options: #{unsupported}" unless unsupported.empty?
-        username=options[:username]
+        username = options[:username]
         raise 'options shall have username' if username.nil?
-        url=options[:url]
+        url = options[:url]
         raise "delete not implemented #{url}"
       end
     end