aspera-cli 4.2.1 → 4.5.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (59) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +1580 -946
  3. data/bin/ascli +1 -1
  4. data/bin/asession +3 -5
  5. data/docs/Makefile +8 -11
  6. data/docs/README.erb.md +1521 -829
  7. data/docs/doc_tools.rb +58 -0
  8. data/docs/test_env.conf +3 -1
  9. data/examples/faspex4.rb +28 -19
  10. data/examples/transfer.rb +2 -2
  11. data/lib/aspera/aoc.rb +157 -134
  12. data/lib/aspera/cli/listener/progress_multi.rb +5 -5
  13. data/lib/aspera/cli/main.rb +106 -48
  14. data/lib/aspera/cli/manager.rb +19 -20
  15. data/lib/aspera/cli/plugin.rb +22 -7
  16. data/lib/aspera/cli/plugins/aoc.rb +260 -208
  17. data/lib/aspera/cli/plugins/ats.rb +11 -10
  18. data/lib/aspera/cli/plugins/bss.rb +2 -2
  19. data/lib/aspera/cli/plugins/config.rb +360 -189
  20. data/lib/aspera/cli/plugins/faspex.rb +119 -56
  21. data/lib/aspera/cli/plugins/faspex5.rb +32 -17
  22. data/lib/aspera/cli/plugins/node.rb +72 -31
  23. data/lib/aspera/cli/plugins/orchestrator.rb +5 -3
  24. data/lib/aspera/cli/plugins/preview.rb +94 -68
  25. data/lib/aspera/cli/plugins/server.rb +16 -5
  26. data/lib/aspera/cli/plugins/shares.rb +17 -0
  27. data/lib/aspera/cli/transfer_agent.rb +64 -82
  28. data/lib/aspera/cli/version.rb +1 -1
  29. data/lib/aspera/command_line_builder.rb +48 -31
  30. data/lib/aspera/cos_node.rb +4 -3
  31. data/lib/aspera/environment.rb +4 -4
  32. data/lib/aspera/fasp/{manager.rb → agent_base.rb} +7 -6
  33. data/lib/aspera/fasp/{connect.rb → agent_connect.rb} +46 -39
  34. data/lib/aspera/fasp/{local.rb → agent_direct.rb} +42 -38
  35. data/lib/aspera/fasp/{http_gw.rb → agent_httpgw.rb} +50 -29
  36. data/lib/aspera/fasp/{node.rb → agent_node.rb} +43 -4
  37. data/lib/aspera/fasp/agent_trsdk.rb +106 -0
  38. data/lib/aspera/fasp/default.rb +17 -0
  39. data/lib/aspera/fasp/installation.rb +64 -48
  40. data/lib/aspera/fasp/parameters.rb +78 -91
  41. data/lib/aspera/fasp/parameters.yaml +531 -0
  42. data/lib/aspera/fasp/uri.rb +1 -1
  43. data/lib/aspera/faspex_gw.rb +12 -11
  44. data/lib/aspera/id_generator.rb +22 -0
  45. data/lib/aspera/keychain/encrypted_hash.rb +120 -0
  46. data/lib/aspera/keychain/macos_security.rb +94 -0
  47. data/lib/aspera/log.rb +45 -32
  48. data/lib/aspera/node.rb +9 -4
  49. data/lib/aspera/oauth.rb +116 -100
  50. data/lib/aspera/persistency_action_once.rb +11 -7
  51. data/lib/aspera/persistency_folder.rb +6 -26
  52. data/lib/aspera/rest.rb +66 -50
  53. data/lib/aspera/sync.rb +40 -35
  54. data/lib/aspera/timer_limiter.rb +22 -0
  55. metadata +86 -29
  56. data/docs/transfer_spec.html +0 -99
  57. data/lib/aspera/api_detector.rb +0 -60
  58. data/lib/aspera/fasp/aoc.rb +0 -24
  59. data/lib/aspera/secrets.rb +0 -20
data/docs/transfer_spec.html DELETED
@@ -1,99 +0,0 @@
1
- <html>
2
- <head>
3
- <title>Transfer Specification</title>
4
- <style type="text/css">
5
- table {border-collapse: collapse;}
6
- table, th, td {border: 1px solid black;}
7
- .yes {color:white;background-color:green;font-weight:bold;}
8
- .no {color:white;background-color:red;font-weight:bold;}
9
- </style>
10
- </head>
11
- <body>
12
- <p>
13
- Columns:
14
- <ul>
15
- <li>F=Fasp Manager(local FASP execution)</li>
16
- <li>N=remote node(node API)</li>
17
- <li>C=Connect Client(web plugin)</li>
18
- </ul>
19
- </p>
20
- <p>
21
- Req/Def : Required or default value (- means emty)
22
- </p>
23
- <p>
24
- Fields with EX_ prefix are specific extensions to local mode.
25
- </p>
26
- <p>
27
- arg: related ascp argument or env var suffix (PASS for ASPERA_SCP_PASS)
28
- </p>
29
- <p>
30
- UNDER CONSTRUCTION<br/>
31
- <a href="https://developer.ibm.com/apis/catalog/?search=aspera">Aspera API Documentation</a>&rarr;Node API&rarr;/opt/transfers<br/>
32
- </p>
33
-
34
- <table>
35
- <tr><th>Field</th><th>Req/Def</th><th>Type</th><th>F</th><th>N</th><th>C</th><th>arg</th><th>Description</th></tr>
36
- <tr><td>direction</td><td>Required</td><td>string</td><td class="yes">Y</td><td class="yes">Y</td><td class="yes">Y</td><td>--mode</td><td>Direction: "send" or "receive"</td></tr>
37
- <tr><td>remote_host</td><td>Required</td><td>string</td><td class="yes">Y</td><td class="yes">Y</td><td class="yes">Y</td><td>--host</td><td>IP or fully qualified domain name of the remote server</td></tr>
38
- <tr><td>remote_user</td><td>Required</td><td>string</td></td><td class="yes">Y</td><td class="yes">Y</td><td class="yes">Y</td><td>--user</td><td>Remote user. Default value is "xfer" on node or connect.</td></tr>
39
- <tr><td>destination_root</td><td>Required</td><td>string</td><td class="yes">Y</td><td class="yes">Y</td><td class="yes">Y</td><td>last arg</td><td>Destination root directory.</td></tr>
40
- <tr><td>title</td><td>-</td><td>string</td><td class="no">N</td><td class="yes">Y</td><td class="yes">Y</td><td>-</td><td>Title of the transfer</td></tr>
41
- <tr><td>tags</td><td>-</td><td>hash</td><td class="yes">Y</td><td class="yes">Y</td><td class="yes">Y</td><td>--tags<br>--tags64</td><td>Metadata for transfer</td></tr>
42
- <tr><td>token</td><td>-</td><td>string</td><td class="yes">Y</td><td class="yes">Y</td><td class="yes">Y</td><td>TOKEN<br/>-W</td><td>Authorization token: Bearer, Basic or ATM</td></tr>
43
- <tr><td>cookie</td><td>-</td><td>string</td><td class="yes">Y</td><td class="yes">Y</td><td class="yes">Y</td><td>COOKIE</td><td>Metadata for transfer (older,string)</td></tr>
44
- <tr><td>remote_access_key</td><td>TODO</td><td>string</td><td></td><td></td><td></td><td>?</td><td>Node only?</td></tr>
45
- <tr><td>source_root</td><td>-</td><td>string</td><td></td><td></td><td></td><td>--source-prefix<br/>--source-prefix64</td><td>Source root directory.(TODO: verify option)</td></tr>
46
- <tr><td>fasp_port</td><td>33001</td><td>integer</td></td><td class="yes">Y</td><td class="yes">Y</td><td class="yes">Y</td><td>-O</td><td>Specifies fasp (UDP) port.</td></tr>
47
- <tr><td>ssh_port</td><td>22 or 33001</td><td>integer</td></td><td class="yes">Y</td><td class="yes">Y</td><td class="yes">Y</td><td>-P</td><td>Specifies ssh (TCP) port.</td></tr>
48
- <tr><td>rate_policy</td><td>server config</td><td>string</td><td class="yes">Y</td><td class="yes">Y</td><td class="yes">Y</td><td>--policy</td><td>Valid literals include "low","fair","high" and "fixed".</td></tr>
49
- <tr><td>symlink_policy</td><td>follow</td><td>string</td><td class="yes">Y</td><td class="yes">Y</td><td class="yes">Y</td><td>--symbolic-links</td><td>copy, follow, copy+force, skip. Default is follow. Handle source side symbolic links by following the link (follow), copying the link itself (copy), skipping (skip), or forcibly copying the link itself (copy+force).</td></tr>
50
- <tr><td>target_rate_kbps</td><td>-</td><td>integer</td><td class="yes">Y</td><td class="yes">Y</td><td class="yes">Y</td><td>-l</td><td>Specifies desired speed for the transfer.</td></tr>
51
- <tr><td>min_rate_kbps</td><td>0</td><td>integer</td><td class="yes">Y</td><td class="yes">Y</td><td class="yes">Y</td><td>-m</td><td>Set the minimum transfer rate in kilobits per second.</td></tr>
52
- <tr><td>cipher</td><td>none</td><td>string</td><td class="yes">Y</td><td class="yes">Y</td><td class="yes">Y</td><td>-c</td><td>in transit encryption type.<br/>none, aes-128, aes-256</td></tr>
53
- <tr><td>content_protection</td><td>encrypt<br/>decrypt</td><td>string</td><td></td><td></td><td></td><td>--file-crypt=</td><td>encryption at rest</td></tr>
54
- <tr><td>content_protection_password</td><td>-</td><td>string</td><td></td><td></td><td></td><td>PASS</td><td>Specifies a string password.</td></tr>
55
- <tr><td>overwrite</td><td>diff</td><td>string</td><td class="yes">Y</td><td class="yes">Y</td><td class="yes">Y</td><td>--overwrite</td><td>Overwrite destination files with the source files of the same name.<br/>never, always, diff, older, or diff+older</td></tr>
56
- <tr><td>retry_duration</td><td></td><td>string</td><td></td><td></td><td></td><td>TODO</td><td>Specifies how long to wait before retrying transfer. (e.g. "5min")</td></tr>
57
- <tr><td>http_fallback</td><td></td><td>bool (node), integer</td><td></td><td></td><td></td><td>-y<br/>TODO</td><td>When true(1), attempts to perform an HTTP transfer if a fasp transfer cannot be performed.</td></tr>
58
- <tr><td>create_dir</td><td></td><td>boolean</td><td class="yes">Y</td><td class="yes">Y</td><td class="yes">Y</td><td>-d</td><td>Specifies whether to create new directories.</td></tr>
59
- <tr><td>precalculate_job_size</td><td>srv. def.</td><td>boolean</td><td class="yes">Y</td><td class="yes">Y</td><td class="yes">Y</td><td>--precalculate-job-size</td><td>Specifies whether to precalculate the job size.</td></tr>
60
- <tr><td>delete_source</td><td></td><td>boolean</td><td></td><td class="yes">Y</td><td></td><td>?</td><td>?</td></tr>
61
- <tr><td>remove_after_transfer</td><td></td><td>boolean</td><td></td><td class="yes">Y</td><td></td><td>?</td><td>Specifies whether to remove file after transfer.</td></tr>
62
- <tr><td>remove_empty_directories</td><td></td><td>boolean</td><td></td><td class="yes">Y</td><td></td><td>?</td><td>Specifies whether to remove empty directories.</td></tr>
63
- <tr><td>multi_session</td><td>1</td><td>integer</td><td class="no">N</td><td class="yes">Y</td><td class="no">N</td><td>-C</td><td>Specifies how many parts the transfer is in.</td></tr>
64
- <tr><td>multi_session_threshold</td><td>null</td><td>integer</td><td class="no">N</td><td class="yes">Y</td><td class="no">N</td><td>-</td><td>in bytes</td></tr>
65
- <tr><td>exclude_newer_than</td><td></td><td>integer</td><td class="yes">Y</td><td></td><td></td><td>--exclude-newer-than</td><td>-</td></tr>
66
- <tr><td>exclude_older_than</td><td></td><td>integer</td><td class="yes">Y</td><td></td><td></td><td>--exclude-older-than</td><td>-</td></tr>
67
- <tr><td>preserve_acls</td><td></td><td>string</td><td class="yes">Y</td><td></td><td></td><td>--preserve-acls</td><td>-</td></tr>
68
- <tr><td>dgram_size</td><td></td><td>integer</td><td class="yes">Y</td><td></td><td></td><td>-Z</td><td>in bytes</td></tr>
69
- <tr><td>compression</td><td></td><td>integer</td><td></td><td></td><td></td><td></td><td>ascp4 only, 0 / 1?</td></tr>
70
- <tr><td>read_threads</td><td></td><td>integer</td><td></td><td></td><td></td><td>-</td><td>ascp4 only</td></tr>
71
- <tr><td>write_threads</td><td></td><td>integer</td><td></td><td></td><td></td><td>-</td><td>ascp4 only</td></tr>
72
- <tr><td>use_ascp4</td><td>false</td><td>boolean</td><td></td><td class="yes">Y</td><td></td><td>-</td><td>specify version of protocol</td></tr>
73
- <tr><td>paths</td><td>source files (dest)</td><td>array</td><td></td><td></td><td></td><td>positional<br/>--file-list<br/>--file-pair-list</td><td>Contains a path to the source (required) and a path to the destination.</td></tr>
74
- <tr><td>http_fallback_port</td><td></td><td>integer</td><td class="yes">Y</td><td></td><td></td><td>-t</td><td>Specifies http port.</td></tr>
75
- <tr><td>https_fallback_port</td><td></td><td>integer</td><td></td><td></td><td></td><td>todo</td><td>Specifies https port.</td></tr>
76
- <tr><td>cipher_allowed</td><td></td><td>string</td><td></td><td></td><td></td><td>-</td><td>returned by node API. Valid literals include "aes-128" and "none".</td></tr>
77
- <tr><td>target_rate_cap_kbps</td><td></td><td></td><td class="no">N</td><td class="no">?</td><td class="yes">?</td><td>-</td><td>Returned by upload/download_setup node api.</td></tr>
78
- <tr><td>rate_policy_allowed</td><td></td><td></td><td></td><td></td><td></td><td>-</td><td>returned by node API. Specifies most aggressive rate policy that is allowed. Valid literals include "low", "fair","high" and "fixed".</td></tr>
79
- <tr><td>ssh_private_key</td><td>-</td><td>string</td><td class="yes">Y</td><td class="no">N</td><td class="no">N</td><td>KEY</td><td>Private key used for SSH authentication, Shall look like: `-----BEGIN RSA PRIVATE KEY-----\nMII`<br/>Note the JSON encoding `\` + `n` for newlines.</td></tr>
80
- <tr><td>remote_password</td><td>-</td><td>string</td><td class="yes">Y</td><td class="yes">Y</td><td class="yes">Y</td><td>PASS</td><td>SSH session password</td></tr>
81
- <tr><td>resume_policy</td><td>faspmgr:<br/>none<br/>other:<br/>sparse_csum</td><td>string</td><td class="yes">Y</td><td class="yes">Y</td><td class="yes">Y</td><td>-k</td><td>none,attrs,sparse_csum,full_csum</td></tr>
82
- <tr><td>authentication</td><td>-</td><td class="no">N</td><td class="no">N</td><td class="yes">Y</td><td>-</td><td>token: Aspera web keys are provided to allow transparent web based session initiation. on connect: password is not asked. Else, password is asked, and keys are not provided.</td></tr>
83
- <tr><td>EX_ssh_key_paths</td><td>-</td><td>array</td><td class="yes">Y</td><td class="no">N</td><td class="no">N</td><td>-i</td><td>Use public key authentication and specify the private key file</td></tr>
84
- <tr><td>EX_at_rest_password</td><td>-</td><td>string</td><td class="yes">Y</td><td class="no">N</td><td class="no">N</td><td>FILEPASS</td><td>Passphrase used for at rest encryption or decryption</td></tr>
85
- <tr><td>EX_proxy_password</td><td>-</td><td>string</td><td class="yes">Y</td><td class="no">N</td><td class="no">N</td><td>PROXY_PASS</td><td>TODO</td></tr>
86
- <tr><td>EX_fasp_proxy_url</td><td>-</td><td>string</td><td class="yes">Y</td><td class="no">N</td><td class="no">N</td><td>--proxy</td><td>Specify the address of the Aspera high-speed proxy server</td></tr>
87
- <tr><td>EX_http_proxy_url</td><td>-</td><td>string</td><td class="yes">Y</td><td class="no">N</td><td class="no">N</td><td>-x</td><td>Specify the proxy server address used by HTTP Fallback</td></tr>
88
- <tr><td>EX_ascp_args</td><td>-</td><td>array</td><td class="yes">Y</td><td class="no">N</td><td class="no">N</td><td>same</td><td>Add command line arguments to ascp</td></tr>
89
- <tr><td>EX_http_transfer_jpeg</td><td>0</td><td>integer</td><td class="yes">Y</td><td class="no">N</td><td class="no">N</td><td>-j</td><td>HTTP transfers as JPEG file</td></tr>
90
- <tr><td>EX_license_text</td><td>-</td><td>string</td><td class="yes">Y</td><td class="no">N</td><td class="no">N</td><td>LICENSE</td><td>license file text</td></tr>
91
- <tr><td>EX_file_list</td><td>-</td><td>string</td><td class="yes">Y</td><td class="no">N</td><td class="no">N</td><td>--file-list</td><td>source file list</td></tr>
92
- <tr><td>EX_file_pair_list</td><td>-</td><td>string</td><td class="yes">Y</td><td class="no">N</td><td class="no">N</td><td>--file-pair-list</td><td>source file pair list</td></tr>
93
- <tr><td>EX_multi_session_part</td><td>-</td><td>string</td><td class="yes">Y</td><td class="no">N</td><td class="no">N</td><td>-C</td><td>part for multisession</td></tr>
94
- <tr><td>EX_no_read</td><td>-</td><td>-</td><td class="yes">Y</td><td class="no">N</td><td class="no">N</td><td>--no-read</td><td>no read source</td></tr>
95
- <tr><td>EX_no_write</td><td>-</td><td>-</td><td class="yes">Y</td><td class="no">N</td><td class="no">N</td><td>--no-write</td><td>no write estination</td></tr>
96
- </table>
97
- </body>
98
- </html>
99
-
data/lib/aspera/api_detector.rb DELETED
@@ -1,60 +0,0 @@
1
- require 'aspera/log'
2
- require 'aspera/rest'
3
-
4
- module Aspera
5
- # detect Aspera product by calling API
6
- class ApiDetector
7
- # @return a hash: {:product=>:node,:version=>'unknown'}
8
- # if not found: {:product=>:unknown,:version=>'unknown'}
9
- def self.discover_product(url)
10
- #uri=URI.parse(url)
11
- api=Rest.new({:base_url=>url})
12
- # Node
13
- begin
14
- result=api.call({:operation=>'GET',:subpath=>'ping'})
15
- if result[:http].body.eql?('')
16
- return {:product=>:node,:version=>'unknown'}
17
- end
18
- rescue SocketError => e
19
- raise e
20
- rescue => e
21
- Log.log.debug("not node (#{e.class}: #{e})")
22
- end
23
- # AoC
24
- begin
25
- result=api.call({:operation=>'GET',:subpath=>'',:headers=>{'Accept'=>'text/html'}})
26
- if result[:http].body.include?('content="AoC"')
27
- return {:product=>:aoc,:version=>'unknown'}
28
- end
29
- rescue SocketError => e
30
- raise e
31
- rescue => e
32
- Log.log.debug("not aoc (#{e.class}: #{e})")
33
- end
34
- # Faspex
35
- begin
36
- result=api.call({:operation=>'POST',:subpath=>'aspera/faspex',:headers=>{'Accept'=>'application/xrds+xml'},:text_body_params=>''})
37
- if result[:http].body.start_with?('<?xml')
38
- res_s=XmlSimple.xml_in(result[:http].body, {"ForceArray" => false})
39
- version=res_s['XRD']['application']['version']
40
- #return JSON.pretty_generate(res_s)
41
- end
42
- return {:product=>:faspex,:version=>version}
43
- rescue
44
- Log.log.debug("not faspex")
45
- end
46
- # Shares
47
- begin
48
- result=api.read('node_api/app')
49
- Log.log.warn("not supposed to work")
50
- rescue RestCallError => e
51
- if e.response.code.to_s.eql?('401') and e.response.body.eql?('{"error":{"user_message":"API user authentication failed"}}')
52
- return {:product=>:shares,:version=>'unknown'}
53
- end
54
- Log.log.warn("not shares: #{e.response.code} #{e.response.body}")
55
- rescue
56
- end
57
- return {:product=>:unknown,:version=>'unknown'}
58
- end
59
- end
60
- end
data/lib/aspera/fasp/aoc.rb DELETED
@@ -1,24 +0,0 @@
1
- require 'aspera/fasp/node'
2
- require 'aspera/log'
3
- require 'aspera/aoc.rb'
4
-
5
- module Aspera
6
- module Fasp
7
- class Aoc < Node
8
- def initialize(aoc_options)
9
- @app=aoc_options[:app] || AoC::FILES_APP
10
- @api_aoc=AoC.new(aoc_options)
11
- Log.log.warn("Under Development")
12
- server_node_file = @api_aoc.resolve_node_file(server_home_node_file,server_folder)
13
- # force node as transfer agent
14
- node_api=Fasp::Node.new(@api_aoc.get_node_api(client_node_file[:node_info],AoC::SCOPE_NODE_USER))
15
- super(node_api)
16
- # additional node to node TS info
17
- @add_ts={
18
- 'remote_access_key' => server_node_file[:node_info]['access_key'],
19
- 'destination_root_id' => server_node_file[:file_id]
20
- }
21
- end
22
- end
23
- end
24
- end
data/lib/aspera/secrets.rb DELETED
@@ -1,20 +0,0 @@
1
- module Aspera
2
- # Manage secrets in CLI using secure way (encryption, wallet, etc...)
3
- class Secrets
4
- attr_accessor :default_secret,:all_secrets
5
- def initialize()
6
- @default_secret=nil
7
- @all_secrets={}
8
- end
9
-
10
- def get_secret(id=nil,mandatory=true)
11
- secret=@default_secret || @all_secrets[id]
12
- raise "please provide secret for #{id}" if secret.nil? and mandatory
13
- return secret
14
- end
15
-
16
- def get_secrets
17
- return @all_secrets
18
- end
19
- end
20
- end