RubyGems - aspera-cli - Versions diffs - 4.2.1 → 4.5.0 - Mend

aspera-cli 4.2.1 → 4.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (59) hide show

checksums.yaml +4 -4
data/README.md +1580 -946
data/bin/ascli +1 -1
data/bin/asession +3 -5
data/docs/Makefile +8 -11
data/docs/README.erb.md +1521 -829
data/docs/doc_tools.rb +58 -0
data/docs/test_env.conf +3 -1
data/examples/faspex4.rb +28 -19
data/examples/transfer.rb +2 -2
data/lib/aspera/aoc.rb +157 -134
data/lib/aspera/cli/listener/progress_multi.rb +5 -5
data/lib/aspera/cli/main.rb +106 -48
data/lib/aspera/cli/manager.rb +19 -20
data/lib/aspera/cli/plugin.rb +22 -7
data/lib/aspera/cli/plugins/aoc.rb +260 -208
data/lib/aspera/cli/plugins/ats.rb +11 -10
data/lib/aspera/cli/plugins/bss.rb +2 -2
data/lib/aspera/cli/plugins/config.rb +360 -189
data/lib/aspera/cli/plugins/faspex.rb +119 -56
data/lib/aspera/cli/plugins/faspex5.rb +32 -17
data/lib/aspera/cli/plugins/node.rb +72 -31
data/lib/aspera/cli/plugins/orchestrator.rb +5 -3
data/lib/aspera/cli/plugins/preview.rb +94 -68
data/lib/aspera/cli/plugins/server.rb +16 -5
data/lib/aspera/cli/plugins/shares.rb +17 -0
data/lib/aspera/cli/transfer_agent.rb +64 -82
data/lib/aspera/cli/version.rb +1 -1
data/lib/aspera/command_line_builder.rb +48 -31
data/lib/aspera/cos_node.rb +4 -3
data/lib/aspera/environment.rb +4 -4
data/lib/aspera/fasp/{manager.rb → agent_base.rb} +7 -6
data/lib/aspera/fasp/{connect.rb → agent_connect.rb} +46 -39
data/lib/aspera/fasp/{local.rb → agent_direct.rb} +42 -38
data/lib/aspera/fasp/{http_gw.rb → agent_httpgw.rb} +50 -29
data/lib/aspera/fasp/{node.rb → agent_node.rb} +43 -4
data/lib/aspera/fasp/agent_trsdk.rb +106 -0
data/lib/aspera/fasp/default.rb +17 -0
data/lib/aspera/fasp/installation.rb +64 -48
data/lib/aspera/fasp/parameters.rb +78 -91
data/lib/aspera/fasp/parameters.yaml +531 -0
data/lib/aspera/fasp/uri.rb +1 -1
data/lib/aspera/faspex_gw.rb +12 -11
data/lib/aspera/id_generator.rb +22 -0
data/lib/aspera/keychain/encrypted_hash.rb +120 -0
data/lib/aspera/keychain/macos_security.rb +94 -0
data/lib/aspera/log.rb +45 -32
data/lib/aspera/node.rb +9 -4
data/lib/aspera/oauth.rb +116 -100
data/lib/aspera/persistency_action_once.rb +11 -7
data/lib/aspera/persistency_folder.rb +6 -26
data/lib/aspera/rest.rb +66 -50
data/lib/aspera/sync.rb +40 -35
data/lib/aspera/timer_limiter.rb +22 -0
metadata +86 -29
data/docs/transfer_spec.html +0 -99
data/lib/aspera/api_detector.rb +0 -60
data/lib/aspera/fasp/aoc.rb +0 -24
data/lib/aspera/secrets.rb +0 -20

data/docs/transfer_spec.html DELETED Viewed

@@ -1,99 +0,0 @@
-<html>
-<head>
-<title>Transfer Specification</title>
-<style type="text/css">
-table {border-collapse: collapse;}
-table, th, td {border: 1px solid black;}
-.yes {color:white;background-color:green;font-weight:bold;}
-.no  {color:white;background-color:red;font-weight:bold;}
-</style>
-</head>
-<body>
-<p>
-Columns:
-<ul>
-<li>F=Fasp Manager(local FASP execution)</li>
-<li>N=remote node(node API)</li>
-<li>C=Connect Client(web plugin)</li>
-</ul>
-</p>
-<p>
-Req/Def : Required or default value (- means emty)
-</p>
-<p>
-Fields with EX_ prefix are specific extensions to local mode.
-</p>
-<p>
-arg: related ascp argument or env var suffix (PASS for ASPERA_SCP_PASS)
-</p>
-<p>
-UNDER CONSTRUCTION<br/>
-<a href="https://developer.ibm.com/apis/catalog/?search=aspera">Aspera API Documentation</a>&rarr;Node API&rarr;/opt/transfers<br/>
-</p>
-<table>
-<tr><th>Field</th><th>Req/Def</th><th>Type</th><th>F</th><th>N</th><th>C</th><th>arg</th><th>Description</th></tr>
-<tr><td>direction</td><td>Required</td><td>string</td><td class="yes">Y</td><td class="yes">Y</td><td class="yes">Y</td><td>--mode</td><td>Direction: "send" or "receive"</td></tr>
-<tr><td>remote_host</td><td>Required</td><td>string</td><td class="yes">Y</td><td class="yes">Y</td><td class="yes">Y</td><td>--host</td><td>IP or fully qualified domain name of the remote server</td></tr>
-<tr><td>remote_user</td><td>Required</td><td>string</td></td><td class="yes">Y</td><td class="yes">Y</td><td class="yes">Y</td><td>--user</td><td>Remote user. Default value is "xfer" on node or connect.</td></tr>
-<tr><td>destination_root</td><td>Required</td><td>string</td><td class="yes">Y</td><td class="yes">Y</td><td class="yes">Y</td><td>last arg</td><td>Destination root directory.</td></tr>
-<tr><td>title</td><td>-</td><td>string</td><td class="no">N</td><td class="yes">Y</td><td class="yes">Y</td><td>-</td><td>Title of the transfer</td></tr>
-<tr><td>tags</td><td>-</td><td>hash</td><td class="yes">Y</td><td class="yes">Y</td><td class="yes">Y</td><td>--tags<br>--tags64</td><td>Metadata for transfer</td></tr>
-<tr><td>token</td><td>-</td><td>string</td><td class="yes">Y</td><td class="yes">Y</td><td class="yes">Y</td><td>TOKEN<br/>-W</td><td>Authorization token: Bearer, Basic or ATM</td></tr>
-<tr><td>cookie</td><td>-</td><td>string</td><td class="yes">Y</td><td class="yes">Y</td><td class="yes">Y</td><td>COOKIE</td><td>Metadata for transfer (older,string)</td></tr>
-<tr><td>remote_access_key</td><td>TODO</td><td>string</td><td></td><td></td><td></td><td>?</td><td>Node only?</td></tr>
-<tr><td>source_root</td><td>-</td><td>string</td><td></td><td></td><td></td><td>--source-prefix<br/>--source-prefix64</td><td>Source root directory.(TODO: verify option)</td></tr>
-<tr><td>fasp_port</td><td>33001</td><td>integer</td></td><td class="yes">Y</td><td class="yes">Y</td><td class="yes">Y</td><td>-O</td><td>Specifies fasp (UDP) port.</td></tr>
-<tr><td>ssh_port</td><td>22 or 33001</td><td>integer</td></td><td class="yes">Y</td><td class="yes">Y</td><td class="yes">Y</td><td>-P</td><td>Specifies ssh (TCP) port.</td></tr>
-<tr><td>rate_policy</td><td>server config</td><td>string</td><td class="yes">Y</td><td class="yes">Y</td><td class="yes">Y</td><td>--policy</td><td>Valid literals include "low","fair","high" and "fixed".</td></tr>
-<tr><td>symlink_policy</td><td>follow</td><td>string</td><td class="yes">Y</td><td class="yes">Y</td><td class="yes">Y</td><td>--symbolic-links</td><td>copy, follow, copy+force, skip.  Default is follow.  Handle source side symbolic links by following the link (follow), copying the link itself (copy),  skipping (skip), or forcibly copying the link itself (copy+force).</td></tr>
-<tr><td>target_rate_kbps</td><td>-</td><td>integer</td><td class="yes">Y</td><td class="yes">Y</td><td class="yes">Y</td><td>-l</td><td>Specifies desired speed for the transfer.</td></tr>
-<tr><td>min_rate_kbps</td><td>0</td><td>integer</td><td class="yes">Y</td><td class="yes">Y</td><td class="yes">Y</td><td>-m</td><td>Set the minimum transfer rate in kilobits per second.</td></tr>
-<tr><td>cipher</td><td>none</td><td>string</td><td class="yes">Y</td><td class="yes">Y</td><td class="yes">Y</td><td>-c</td><td>in transit encryption type.<br/>none, aes-128, aes-256</td></tr>
-<tr><td>content_protection</td><td>encrypt<br/>decrypt</td><td>string</td><td></td><td></td><td></td><td>--file-crypt=</td><td>encryption at rest</td></tr>
-<tr><td>content_protection_password</td><td>-</td><td>string</td><td></td><td></td><td></td><td>PASS</td><td>Specifies a string password.</td></tr>
-<tr><td>overwrite</td><td>diff</td><td>string</td><td class="yes">Y</td><td class="yes">Y</td><td class="yes">Y</td><td>--overwrite</td><td>Overwrite destination files with the source files of the same name.<br/>never, always, diff, older, or diff+older</td></tr>
-<tr><td>retry_duration</td><td></td><td>string</td><td></td><td></td><td></td><td>TODO</td><td>Specifies how long to wait before retrying transfer. (e.g. "5min")</td></tr>
-<tr><td>http_fallback</td><td></td><td>bool (node), integer</td><td></td><td></td><td></td><td>-y<br/>TODO</td><td>When true(1), attempts to perform an HTTP transfer if a fasp transfer cannot be performed.</td></tr>
-<tr><td>create_dir</td><td></td><td>boolean</td><td class="yes">Y</td><td class="yes">Y</td><td class="yes">Y</td><td>-d</td><td>Specifies whether to create new directories.</td></tr>
-<tr><td>precalculate_job_size</td><td>srv. def.</td><td>boolean</td><td class="yes">Y</td><td class="yes">Y</td><td class="yes">Y</td><td>--precalculate-job-size</td><td>Specifies whether to precalculate the job size.</td></tr>
-<tr><td>delete_source</td><td></td><td>boolean</td><td></td><td class="yes">Y</td><td></td><td>?</td><td>?</td></tr>
-<tr><td>remove_after_transfer</td><td></td><td>boolean</td><td></td><td class="yes">Y</td><td></td><td>?</td><td>Specifies whether to remove file after transfer.</td></tr>
-<tr><td>remove_empty_directories</td><td></td><td>boolean</td><td></td><td class="yes">Y</td><td></td><td>?</td><td>Specifies whether to remove empty directories.</td></tr>
-<tr><td>multi_session</td><td>1</td><td>integer</td><td class="no">N</td><td class="yes">Y</td><td class="no">N</td><td>-C</td><td>Specifies how many parts the transfer is in.</td></tr>
-<tr><td>multi_session_threshold</td><td>null</td><td>integer</td><td class="no">N</td><td class="yes">Y</td><td class="no">N</td><td>-</td><td>in bytes</td></tr>
-<tr><td>exclude_newer_than</td><td></td><td>integer</td><td class="yes">Y</td><td></td><td></td><td>--exclude-newer-than</td><td>-</td></tr>
-<tr><td>exclude_older_than</td><td></td><td>integer</td><td class="yes">Y</td><td></td><td></td><td>--exclude-older-than</td><td>-</td></tr>
-<tr><td>preserve_acls</td><td></td><td>string</td><td class="yes">Y</td><td></td><td></td><td>--preserve-acls</td><td>-</td></tr>
-<tr><td>dgram_size</td><td></td><td>integer</td><td class="yes">Y</td><td></td><td></td><td>-Z</td><td>in bytes</td></tr>
-<tr><td>compression</td><td></td><td>integer</td><td></td><td></td><td></td><td></td><td>ascp4 only, 0 / 1?</td></tr>
-<tr><td>read_threads</td><td></td><td>integer</td><td></td><td></td><td></td><td>-</td><td>ascp4 only</td></tr>
-<tr><td>write_threads</td><td></td><td>integer</td><td></td><td></td><td></td><td>-</td><td>ascp4 only</td></tr>
-<tr><td>use_ascp4</td><td>false</td><td>boolean</td><td></td><td class="yes">Y</td><td></td><td>-</td><td>specify version of protocol</td></tr>
-<tr><td>paths</td><td>source files (dest)</td><td>array</td><td></td><td></td><td></td><td>positional<br/>--file-list<br/>--file-pair-list</td><td>Contains a path to the source (required) and a path to the destination.</td></tr>
-<tr><td>http_fallback_port</td><td></td><td>integer</td><td class="yes">Y</td><td></td><td></td><td>-t</td><td>Specifies http port.</td></tr>
-<tr><td>https_fallback_port</td><td></td><td>integer</td><td></td><td></td><td></td><td>todo</td><td>Specifies https port.</td></tr>
-<tr><td>cipher_allowed</td><td></td><td>string</td><td></td><td></td><td></td><td>-</td><td>returned by node API. Valid literals include "aes-128" and "none".</td></tr>
-<tr><td>target_rate_cap_kbps</td><td></td><td></td><td class="no">N</td><td class="no">?</td><td class="yes">?</td><td>-</td><td>Returned by upload/download_setup node api.</td></tr>
-<tr><td>rate_policy_allowed</td><td></td><td></td><td></td><td></td><td></td><td>-</td><td>returned by node API. Specifies most aggressive rate policy that is allowed. Valid literals include "low", "fair","high" and "fixed".</td></tr>
-<tr><td>ssh_private_key</td><td>-</td><td>string</td><td class="yes">Y</td><td class="no">N</td><td class="no">N</td><td>KEY</td><td>Private key used for SSH authentication, Shall look like: `-----BEGIN RSA PRIVATE KEY-----\nMII`<br/>Note the JSON encoding `\` + `n` for newlines.</td></tr>
-<tr><td>remote_password</td><td>-</td><td>string</td><td class="yes">Y</td><td class="yes">Y</td><td class="yes">Y</td><td>PASS</td><td>SSH session password</td></tr>
-<tr><td>resume_policy</td><td>faspmgr:<br/>none<br/>other:<br/>sparse_csum</td><td>string</td><td class="yes">Y</td><td class="yes">Y</td><td class="yes">Y</td><td>-k</td><td>none,attrs,sparse_csum,full_csum</td></tr>
-<tr><td>authentication</td><td>-</td><td class="no">N</td><td class="no">N</td><td class="yes">Y</td><td>-</td><td>token: Aspera web keys are provided to allow transparent web based session initiation. on connect: password is not asked. Else, password is asked, and keys are not provided.</td></tr>
-<tr><td>EX_ssh_key_paths</td><td>-</td><td>array</td><td class="yes">Y</td><td class="no">N</td><td class="no">N</td><td>-i</td><td>Use public key authentication and specify the private key file</td></tr>
-<tr><td>EX_at_rest_password</td><td>-</td><td>string</td><td class="yes">Y</td><td class="no">N</td><td class="no">N</td><td>FILEPASS</td><td>Passphrase used for at rest encryption or decryption</td></tr>
-<tr><td>EX_proxy_password</td><td>-</td><td>string</td><td class="yes">Y</td><td class="no">N</td><td class="no">N</td><td>PROXY_PASS</td><td>TODO</td></tr>
-<tr><td>EX_fasp_proxy_url</td><td>-</td><td>string</td><td class="yes">Y</td><td class="no">N</td><td class="no">N</td><td>--proxy</td><td>Specify the address of the Aspera high-speed proxy server</td></tr>
-<tr><td>EX_http_proxy_url</td><td>-</td><td>string</td><td class="yes">Y</td><td class="no">N</td><td class="no">N</td><td>-x</td><td>Specify the proxy server address used by HTTP Fallback</td></tr>
-<tr><td>EX_ascp_args</td><td>-</td><td>array</td><td class="yes">Y</td><td class="no">N</td><td class="no">N</td><td>same</td><td>Add command line arguments to ascp</td></tr>
-<tr><td>EX_http_transfer_jpeg</td><td>0</td><td>integer</td><td class="yes">Y</td><td class="no">N</td><td class="no">N</td><td>-j</td><td>HTTP transfers as JPEG file</td></tr>
-<tr><td>EX_license_text</td><td>-</td><td>string</td><td class="yes">Y</td><td class="no">N</td><td class="no">N</td><td>LICENSE</td><td>license file text</td></tr>
-<tr><td>EX_file_list</td><td>-</td><td>string</td><td class="yes">Y</td><td class="no">N</td><td class="no">N</td><td>--file-list</td><td>source file list</td></tr>
-<tr><td>EX_file_pair_list</td><td>-</td><td>string</td><td class="yes">Y</td><td class="no">N</td><td class="no">N</td><td>--file-pair-list</td><td>source file pair list</td></tr>
-<tr><td>EX_multi_session_part</td><td>-</td><td>string</td><td class="yes">Y</td><td class="no">N</td><td class="no">N</td><td>-C</td><td>part for multisession</td></tr>
-<tr><td>EX_no_read</td><td>-</td><td>-</td><td class="yes">Y</td><td class="no">N</td><td class="no">N</td><td>--no-read</td><td>no read source</td></tr>
-<tr><td>EX_no_write</td><td>-</td><td>-</td><td class="yes">Y</td><td class="no">N</td><td class="no">N</td><td>--no-write</td><td>no write estination</td></tr>
-</table>
-</body>
-</html>

data/lib/aspera/api_detector.rb DELETED Viewed

@@ -1,60 +0,0 @@
-require 'aspera/log'
-require 'aspera/rest'
-module Aspera
-  # detect Aspera product by calling API
-  class ApiDetector
-    # @return a hash: {:product=>:node,:version=>'unknown'}
-    # if not found: {:product=>:unknown,:version=>'unknown'}
-    def self.discover_product(url)
-      #uri=URI.parse(url)
-      api=Rest.new({:base_url=>url})
-      # Node
-      begin
-        result=api.call({:operation=>'GET',:subpath=>'ping'})
-        if result[:http].body.eql?('')
-          return {:product=>:node,:version=>'unknown'}
-        end
-      rescue SocketError => e
-        raise e
-      rescue => e
-        Log.log.debug("not node (#{e.class}: #{e})")
-      end
-      # AoC
-      begin
-        result=api.call({:operation=>'GET',:subpath=>'',:headers=>{'Accept'=>'text/html'}})
-        if result[:http].body.include?('content="AoC"')
-          return {:product=>:aoc,:version=>'unknown'}
-        end
-      rescue SocketError => e
-        raise e
-      rescue => e
-        Log.log.debug("not aoc (#{e.class}: #{e})")
-      end
-      # Faspex
-      begin
-        result=api.call({:operation=>'POST',:subpath=>'aspera/faspex',:headers=>{'Accept'=>'application/xrds+xml'},:text_body_params=>''})
-        if result[:http].body.start_with?('<?xml')
-          res_s=XmlSimple.xml_in(result[:http].body, {"ForceArray" => false})
-          version=res_s['XRD']['application']['version']
-          #return JSON.pretty_generate(res_s)
-        end
-        return {:product=>:faspex,:version=>version}
-      rescue
-        Log.log.debug("not faspex")
-      end
-      # Shares
-      begin
-        result=api.read('node_api/app')
-        Log.log.warn("not supposed to work")
-      rescue RestCallError => e
-        if e.response.code.to_s.eql?('401') and e.response.body.eql?('{"error":{"user_message":"API user authentication failed"}}')
-          return {:product=>:shares,:version=>'unknown'}
-        end
-        Log.log.warn("not shares: #{e.response.code} #{e.response.body}")
-      rescue
-      end
-      return {:product=>:unknown,:version=>'unknown'}
-    end
-  end
-end

data/lib/aspera/fasp/aoc.rb DELETED Viewed

@@ -1,24 +0,0 @@
-require 'aspera/fasp/node'
-require 'aspera/log'
-require 'aspera/aoc.rb'
-module Aspera
-  module Fasp
-    class Aoc < Node
-      def initialize(aoc_options)
-        @app=aoc_options[:app] || AoC::FILES_APP
-        @api_aoc=AoC.new(aoc_options)
-        Log.log.warn("Under Development")
-        server_node_file = @api_aoc.resolve_node_file(server_home_node_file,server_folder)
-        # force node as transfer agent
-        node_api=Fasp::Node.new(@api_aoc.get_node_api(client_node_file[:node_info],AoC::SCOPE_NODE_USER))
-        super(node_api)
-        # additional node to node TS info
-        @add_ts={
-          'remote_access_key'   => server_node_file[:node_info]['access_key'],
-          'destination_root_id' => server_node_file[:file_id]
-        }
-      end
-    end
-  end
-end

data/lib/aspera/secrets.rb DELETED Viewed

@@ -1,20 +0,0 @@
-module Aspera
-  # Manage secrets in CLI using secure way (encryption, wallet, etc...)
-  class Secrets
-    attr_accessor :default_secret,:all_secrets
-    def initialize()
-      @default_secret=nil
-      @all_secrets={}
-    end
-    def get_secret(id=nil,mandatory=true)
-      secret=@default_secret || @all_secrets[id]
-      raise "please provide secret for #{id}" if secret.nil? and mandatory
-      return secret
-    end
-    def get_secrets
-      return @all_secrets
-    end
-  end
-end