aspera-cli 4.0.0 → 4.2.2

data/lib/aspera/fasp/parameters.yaml

@@ -0,0 +1,531 @@
+---
+cipher:
+  :cltype: :opt_with_arg
+  :option_switch: "-c"
+  :enum:
+  - aes128
+  - aes192
+  - aes256
+  - aes128cfb
+  - aes192cfb
+  - aes256cfb
+  - aes128gcm
+  - aes192gcm
+  - aes256gcm
+  :encode: Aspera::Fasp::Parameters
+  :default: none
+  :desc: "In transit encryption type.\nnone, aes-128, aes-256"
+content_protection:
+  :cltype: :ignore
+  :option_switch: "--file-crypt"
+  :enum:
+  - encrypt
+  - decrypt
+  :desc: Enable client-side content protection. (encryption at rest)
+content_protection_password:
+  :accepted_types: :string
+  :desc: "Specifies CSEAR password."
+cookie:
+  :cltype: :envvar
+  :clvarname: ASPERA_SCP_COOKIE
+  :desc: Metadata for transfer (older,string)
+create_dir:
+  :cltype: :opt_without_arg
+  :option_switch: "-d"
+  :accepted_types: :bool
+  :desc: Specifies whether to create new directories.
+delete_before_transfer:
+  :cltype: :opt_without_arg
+delete_source:
+  :accepted_types: :bool
+  :context:
+  - :node
+  :desc: Remove SRC files after transfer success
+destination_root_id:
+  :cltype: :ignore
+  :accepted_types: :string
+  :context:
+  - :sdk
+direction:
+  :cltype: :opt_with_arg
+  :option_switch: "--mode"
+  :enum:
+  - send
+  - receive
+  :translate_values:
+    receive: recv
+    send: send
+  :required: true
+  :desc: 'Direction of transfer (on client side)'
+exclude_newer_than:
+  :cltype: :opt_with_arg
+  :accepted_types: :int
+  :context:
+  - :direct
+  :desc: skip src files with mtime > arg
+exclude_older_than:
+  :cltype: :opt_with_arg
+  :accepted_types: :int
+  :context:
+  - :direct
+  :desc: skip src files with mtime < arg
+fasp_port:
+  :cltype: :opt_with_arg
+  :option_switch: "-O"
+  :accepted_types: :int
+  :default: 33001
+  :desc: Specifies fasp (UDP) port.
+http_fallback:
+  :cltype: :opt_with_arg
+  :option_switch: "-y"
+  :accepted_types:
+  - :string
+  - :bool
+  :translate_values:
+    force: F
+    true: 1
+    false: 0
+  :desc: When true(1), attempts to perform an HTTP transfer if a fasp transfer cannot
+    be performed.
+http_fallback_port:
+  :cltype: :opt_with_arg
+  :option_switch: "-t"
+  :accepted_types: :int
+  :context:
+  - :direct
+  :desc: Specifies http port.
+https_fallback_port:
+  :cltype: :ignore
+  :accepted_types: :int
+  :desc: Specifies https port.
+move_after_transfer:
+  :cltype: :opt_with_arg
+multi_session:
+  :cltype: :ignore
+  :option_switch: "-C"
+  :accepted_types: :int
+  :desc: "Use multi-session transfer. max 128.\n
+Each participant on one host needs an independent UDP (-O) port.\n
+Large files are split between sessions only when transferring with resume_policy=none."
+multi_session_threshold:
+  :cltype: :opt_with_arg
+  :accepted_types: :int
+  :context:
+  - :direct
+  - :node
+  :desc: in bytes
+overwrite:
+  :cltype: :opt_with_arg
+  :default: diff
+  :enum:
+  - never
+  - always
+  - diff
+  - older
+  - diff+older
+  :desc: "Overwrite destination files with the source files of the same name."
+paths:
+  :cltype: :defer
+  :accepted_types: :array
+  :desc: Required. Contains a path to the source (required) and a path to the destination.
+precalculate_job_size:
+  :cltype: :opt_without_arg
+  :accepted_types: :bool
+  :desc: Specifies whether to precalculate the job size.
+preserve_access_time:
+  :cltype: :opt_without_arg
+preserve_creation_time:
+  :cltype: :opt_without_arg
+preserve_modification_time:
+  :cltype: :opt_without_arg
+preserve_times:
+  :cltype: :opt_without_arg
+rate_policy:
+  :cltype: :opt_with_arg
+  :option_switch: "--policy"
+  :enum:
+  - low
+  - fair
+  - high
+  - fixed
+  :desc: The transfer rate policy to use when sharing bandwidth.
+remote_access_key:
+  :accepted_types: :string
+  :desc: Node only?
+remote_host:
+  :cltype: :opt_with_arg
+  :option_switch: "--host"
+  :required: true
+  :desc: IP or fully qualified domain name of the remote server
+remote_user:
+  :cltype: :opt_with_arg
+  :option_switch: "--user"
+  :required: true
+  :desc: Remote user. Default value is "xfer" on node or connect.
+remote_password:
+  :cltype: :envvar
+  :clvarname: ASPERA_SCP_PASS
+  :desc: SSH session password
+remove_after_transfer:
+  :cltype: :opt_without_arg
+  :accepted_types: :bool
+  :context:
+  - :direct
+  - :node
+  :desc: Remove SRC files after transfer success
+remove_empty_directories:
+  :cltype: :opt_without_arg
+  :accepted_types: :bool
+  :context:
+  - :direct
+  - :node
+  :desc: Specifies whether to remove empty directories.
+remove_skipped:
+  :cltype: :ignore
+  :accepted_types: :bool
+  :context:
+  - :sdk
+proxy:
+  :cltype: :opt_with_arg
+  :context:
+  - :direct
+  - :sdk
+  :desc: "Specify the address of the Aspera high-speed proxy server.\n
+dnat(s)://[user[:password]@]server:port\n
+Default ports for DNAT and DNATS protocols are 9091 and 9092.\n
+Password, if specified here, overrides the value of environment variable ASPERA_PROXY_PASS."
+resume_policy:
+  :cltype: :opt_with_arg
+  :option_switch: "-k"
+  :default: faspmgr:none;other:sparse_csum
+  :enum:
+  - none
+  - attrs
+  - sparse_csum
+  - full_csum
+  :translate_values:
+    none: 0
+    attrs: 1
+    sparse_csum: 2
+    full_csum: 3
+  :desc: If a transfer is interrupted or fails to finish, resume without re-transferring the whole files.
+retry_duration:
+  :accepted_types:
+  - :string
+  - :int
+  :context:
+  - :node
+  - :connect
+  - :sdk
+  :desc: Specifies how long to wait before retrying transfer. (e.g. "5min")
+source_root_id:
+  :cltype: :ignore
+  :accepted_types: :string
+  :context:
+  - :sdk
+ssh_port:
+  :cltype: :opt_with_arg
+  :option_switch: "-P"
+  :accepted_types: :int
+  :desc: "Specifies ssh (TCP) port. Default: local:22, other:33001"
+ssh_private_key:
+  :cltype: :envvar
+  :clvarname: ASPERA_SCP_KEY
+  :context:
+  - :direct
+  - :sdk
+  :desc: "Private key used for SSH authentication.\n
+   Shall look like: -----BEGIN RSA PRIVATE KEY-----\\nMII...\n
+   Note the JSON encoding: \\n for newlines."
+ssh_private_key_passphrase:
+  :cltype: :ignore
+  :accepted_types: :string
+  :context:
+  - :sdk
+ssh_args:
+  :cltype: :ignore
+  :accepted_types: :string
+  :context:
+  - :sdk
+symlink_policy:
+  :cltype: :opt_with_arg
+  :option_switch: "--symbolic-links"
+  :default: follow
+  :enum:
+  - follow
+  - copy
+  - copy+force
+  - skip
+  :desc: "Handle source side symbolic links by:\n
+following the link (follow),\n
+copying the link itself (copy),\n
+skipping (skip),\n
+or forcibly copying the link itself (copy+force).\n
+Default: follow"
+checksum_type:
+  :cltype: :ignore
+  :accepted_types: :string
+  :context:
+  - :sdk
+  :desc: "Enable checksum reporting for transferred files by specifying the hash to use.\n
+Allowable values: sha-512, sha-384, sha-256, sha1, md5. (Default: none)"
+file_checksum:
+  :cltype: :ignore
+  :accepted_types: :string
+  :context:
+  - :sdk
+tags:
+  :cltype: :opt_with_arg
+  :option_switch: "--tags64"
+  :accepted_types: :hash
+  :encode: Aspera::Fasp::Parameters
+  :desc: Metadata for transfer
+target_rate_cap_kbps:
+  :cltype: :ignore
+  :accepted_types: :int
+  :context:
+  - :connect
+  :desc: Returned by upload/download_setup node api.
+target_rate_kbps:
+  :cltype: :opt_with_arg
+  :option_switch: "-l"
+  :accepted_types: :int
+  :desc: Specifies desired speed for the transfer.
+title:
+  :accepted_types: :string
+  :context:
+  - :node
+  - :connect
+  :desc: Title of the transfer
+token:
+  :cltype: :envvar
+  :clvarname: ASPERA_SCP_TOKEN
+  :desc: 'Authorization token: Bearer, Basic or ATM (Also arg -W)'
+use_ascp4:
+  :cltype: :defer
+  :accepted_types: :bool
+  :default: false
+  :context:
+  - :direct
+  - :node
+  - :sdk
+  :desc: specify version of protocol
+use_system_ssh:
+  :cltype: :ignore
+  :option_switch: "-SSH"
+  :accepted_types: :string
+  :context:
+  - :sdk
+destination_root:
+  :cltype: :defer
+  :required: true
+  :desc: Destination root directory.
+source_root:
+  :cltype: :opt_with_arg
+  :option_switch: "--source-prefix64"
+  :encode: Aspera::Fasp::Parameters
+  :desc: "Path to be prepended to each source path.\n
+This is either a conventional path or it can be a URI but only if there is no root defined."
+min_rate_cap_kbps:
+  :cltype: :ignore
+  :accepted_types: :int
+lock_rate_policy:
+  :cltype: :ignore
+  :accepted_types: :bool
+lock_target_rate_kbps:
+  :cltype: :ignore
+  :accepted_types: :bool
+lock_min_rate_kbps:
+  :cltype: :ignore
+  :accepted_types: :bool
+apply_local_docroot:
+  :cltype: :opt_without_arg
+  :context:
+  - :direct
+  - :sdk
+preserve_acls:
+  :cltype: :opt_with_arg
+  :context:
+  - :direct
+  - :sdk
+  :desc: "Preserve access control lists.\nAllowable values: none, native, metafile"
+preserve_remote_acls:
+  :cltype: :ignore
+  :accepted_types: :string
+  :context:
+  - :sdk
+  :desc: "Preserve remote access control lists.\n
+Allowable values: none, native, metafile"
+preserve_file_owner_uid:
+  :cltype: :ignore
+  :accepted_types: :bool
+  :context:
+  - :sdk
+preserve_file_owner_gid:
+  :cltype: :ignore
+  :accepted_types: :bool
+  :context:
+  - :sdk
+preserve_extended_attrs:
+  :cltype: :ignore
+  :accepted_types: :string
+  :context:
+  - :sdk
+  :desc: "Preserve the extended attributes.\n
+Allowable values: none, native, metafile"
+preserve_source_access_time:
+  :cltype: :ignore
+  :accepted_types: :bool
+  :context:
+  - :sdk
+remove_empty_source_directory:
+  :cltype: :opt_without_arg
+  :context:
+  - :direct
+  :desc: "TODO: check node, sdk"
+EX_at_rest_password:
+  :cltype: :envvar
+  :clvarname: ASPERA_SCP_FILEPASS
+  :context:
+  - :direct
+  :desc: Passphrase used for at rest encryption or decryption
+EX_proxy_password:
+  :cltype: :envvar
+  :clvarname: ASPERA_PROXY_PASS
+  :context:
+  - :direct
+  :desc: "Password used for Aspera proxy server authentication.\n
+May be overridden by password in URL EX_fasp_proxy_url."
+EX_license_text:
+  :cltype: :envvar
+  :clvarname: ASPERA_SCP_LICENSE
+  :context:
+  - :direct
+  :desc: "License file text override.\nBy default ascp looks for license file near executable."
+keepalive:
+  :cltype: :opt_without_arg
+  :context:
+  - :none
+dgram_size:
+  :cltype: :opt_with_arg
+  :option_switch: "-Z"
+  :accepted_types: :int
+  :desc: in bytes
+min_rate_kbps:
+  :cltype: :opt_with_arg
+  :option_switch: "-m"
+  :accepted_types: :int
+  :default: 0
+  :desc: Set the minimum transfer rate in kilobits per second.
+sshfp:
+  :cltype: :opt_with_arg
+  :option_switch: "--check-sshfp"
+  :desc: Check it against server SSH host key fingerprint
+EX_http_proxy_url:
+  :cltype: :opt_with_arg
+  :option_switch: "-x"
+  :context:
+  - :direct
+  :desc: Specify the proxy server address used by HTTP Fallback
+EX_ssh_key_paths:
+  :cltype: :opt_with_arg
+  :option_switch: "-i"
+  :accepted_types: :array
+  :context:
+  - :direct
+  :desc: Use public key authentication for SSH and specify the private key file paths
+EX_http_transfer_jpeg:
+  :cltype: :opt_with_arg
+  :option_switch: "-j"
+  :accepted_types: :int
+  :default: '0'
+  :context:
+  - :direct
+  :desc: HTTP transfers as JPEG file
+EX_no_read:
+  :cltype: :opt_without_arg
+  :option_switch: "--no-read"
+  :context:
+  - :direct
+  :desc: no read source
+EX_no_write:
+  :cltype: :opt_without_arg
+  :option_switch: "--no-write"
+  :context:
+  - :direct
+  :desc: no write on destination
+target_rate_percentage:
+  :cltype: :ignore
+rate_policy_allowed:
+  :cltype: :ignore
+  :context:
+  - :connect
+  :enum:
+  - low
+  - fair
+  - high
+  - fixed
+  :desc: "Specifies most aggressive rate policy that is allowed.\n
+Returned by node API."
+fasp_url:
+  :cltype: :ignore
+  :context:
+  - :unknown
+lock_min_rate:
+  :cltype: :ignore
+  :accepted_types: :bool
+lock_target_rate:
+  :cltype: :ignore
+  :accepted_types: :bool
+authentication:
+  :cltype: :ignore
+  :context:
+  - :connect
+  :desc: 'value=token for SSH bypass keys, else password asked if not provided.'
+cipher_allowed:
+  :cltype: :ignore
+  :desc: returned by node API. Valid literals include "aes-128" and "none".
+obfuscate_file_names:
+  :cltype: :ignore
+  :accepted_types: :bool
+  :context:
+  - :unknown
+EX_file_list:
+  :cltype: :defer
+  :option_switch: "--file-list"
+  :context:
+  - :direct
+  :desc: source file list
+EX_file_pair_list:
+  :cltype: :defer
+  :option_switch: "--file-pair-list"
+  :context:
+  - :direct
+  :desc: source file pair list
+EX_ascp_args:
+  :cltype: :defer
+  :accepted_types: :array
+  :context:
+  - :direct
+  :desc: Add command line arguments to ascp
+wss_enabled:
+  :cltype: :defer
+  :accepted_types: :bool
+wss_port:
+  :cltype: :defer
+  :accepted_types: :int
+compression:
+  :accepted_types: :int
+  :context:
+  - :unknown
+  :desc: ascp4 only, 0 / 1?
+read_threads:
+  :accepted_types: :int
+  :context:
+  - :unknown
+  :desc: ascp4 only
+write_threads:
+  :accepted_types: :int
+  :context:
+  - :unknown
+  :desc: ascp4 only

data/lib/aspera/fasp/resume_policy.rb

@@ -14,17 +14,21 @@ module Aspera
         :sleep_max     => 60
       }
 
-      def initialize(params={})
+      # @param params see DEFAULTS
+      def initialize(params=nil)
         @parameters=DEFAULTS.clone
-        return if params.nil?
-        raise "expecting Hash (or nil), but have #{params.class}" unless params.is_a?(Hash)
-        params.each do |k,v|
-          if DEFAULTS.has_key?(k)
-            @parameters[k]=v
-          else
-            raise "unknown resume parameter: #{k}, expect one of #{DEFAULTS.keys.map{|i|i.to_s}.join(",")}"
+        if !params.nil?
+          raise "expecting Hash (or nil), but have #{params.class}" unless params.is_a?(Hash)
+          params.each do |k,v|
+            if DEFAULTS.has_key?(k)
+              raise "#{k} must be Integer" unless v.is_a?(Integer)
+              @parameters[k]=v
+            else
+              raise "unknown resume parameter: #{k}, expect one of #{DEFAULTS.keys.map{|i|i.to_s}.join(",")}"
+            end
           end
         end
+        Log.log.debug("resume params=#{@parameters}")
       end
 
       # calls block a number of times (resumes) until success or limit reached
@@ -45,10 +49,7 @@ module Aspera
             # failure in ascp
             if e.retryable? then
               # exit if we exceed the max number of retry
-              unless remaining_resumes > 0
-                Log.log.error "Maximum number of retry reached"
-                raise Fasp::Error,"max retry after: [#{status[:message]}]"
-              end
+              raise Fasp::Error,'Maximum number of retry reached' if remaining_resumes <= 0
             else
               # give one chance only to non retryable errors
               unless remaining_resumes.eql?(@parameters[:iter_max])

data/lib/aspera/fasp/uri.rb

@@ -6,7 +6,7 @@ module Aspera
     # translates a "faspe:" URI (used in Faspex) into transfer spec hash
     class Uri
       def initialize(fasplink)
-        @fasp_uri=URI.parse(fasplink)
+        @fasp_uri=URI.parse(fasplink.gsub(' ','%20'))
         # TODO: check scheme is faspe
       end
 

data/lib/aspera/log.rb

@@ -14,7 +14,7 @@ module Aspera
     attr_reader :logger
     attr_reader :logger_type
     # levels are :debug,:info,:warn,:error,fatal,:unknown
-    def self.levels; Logger::Severity.constants.map{|c| c.downcase.to_sym};end
+    def self.levels; Logger::Severity.constants.sort{|a,b|Logger::Severity.const_get(a)<=>Logger::Severity.const_get(b)}.map{|c|c.downcase.to_sym};end
 
     # where logs are sent to
     def self.logtypes; [:stderr,:stdout,:syslog];end

data/lib/aspera/node.rb

@@ -1,14 +1,74 @@
+require 'aspera/rest'
+require 'aspera/log'
 require 'zlib'
 require 'base64'
+
 module Aspera
   # Provides additional functions using node API.
   class Node < Rest
+    # permissions
+    ACCESS_LEVELS=['delete','list','mkdir','preview','read','rename','write']
+    MATCH_EXEC_PREFIX='exec:'
+
+    # for information only
     def self.decode_bearer_token(token)
       return JSON.parse(Zlib::Inflate.inflate(Base64.decode64(token)).partition('==SIGNATURE==').first)
     end
+
+    # for access keys: provide expression to match entry in folder
+    # if no prefix: regex
+    # if prefix: ruby code
+    # if filder is nil, then always match
+    def self.file_matcher(match_expression)
+      match_expression||="#{MATCH_EXEC_PREFIX}true"
+      if match_expression.start_with?(MATCH_EXEC_PREFIX)
+        return eval "lambda{|f|#{match_expression[MATCH_EXEC_PREFIX.length..-1]}}"
+      end
+      return lambda{|f|f['name'].match(/#{match_expression}/)}
+    end
+
     def initialize(rest_params)
       super(rest_params)
-      # specifics here
+    end
+
+    # recursively crawl in a folder.
+    # subfolders a processed if the processing method returns true
+    # @param processor must provide a method to process each entry
+    # @param opt options
+    # - top_file_id file id to start at (default = access key root file id)
+    # - top_file_path path of top folder (default = /)
+    # - method processing method (default= process_entry)
+    def crawl(processor,opt={})
+      Log.log.debug("crawl1 #{opt}")
+      # not possible with bearer token
+      opt[:top_file_id] ||= read('access_keys/self')[:data]['root_file_id']
+      opt[:top_file_path] ||= '/'
+      opt[:method] ||= :process_entry
+      raise "processor must have #{opt[:method]}" unless processor.respond_to?(opt[:method])
+      Log.log.debug("crawl #{opt}")
+      #top_info=read("files/#{opt[:top_file_id]}")[:data]
+      folders_to_explore=[{id: opt[:top_file_id], relpath: opt[:top_file_path]}]
+      Log.dump(:folders_to_explore,folders_to_explore)
+      while !folders_to_explore.empty? do
+        current_item = folders_to_explore.shift
+        Log.log.debug("searching #{current_item[:relpath]}".bg_green)
+        # get folder content
+        folder_contents = begin
+          read("files/#{current_item[:id]}/files")[:data]
+        rescue => e
+          Log.log.warn("#{current_item[:relpath]}: #{e.class} #{e.message}")
+          []
+        end
+        Log.dump(:folder_contents,folder_contents)
+        folder_contents.each do |entry|
+          relative_path=File.join(current_item[:relpath],entry['name'])
+          Log.log.debug("looking #{relative_path}".bg_green)
+          # entry type is file, folder or link
+          if processor.send(opt[:method],entry,relative_path) and entry['type'].eql?('folder')
+            folders_to_explore.push({:id=>entry['id'],:relpath=>relative_path})
+          end
+        end
+      end
     end
   end
 end