aspera-cli 4.4.0 → 4.7.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +2095 -1503
- data/bin/ascli +2 -1
- data/bin/asession +4 -5
- data/docs/test_env.conf +3 -0
- data/examples/aoc.rb +4 -3
- data/examples/faspex4.rb +25 -25
- data/examples/proxy.pac +1 -1
- data/examples/transfer.rb +17 -17
- data/lib/aspera/aoc.rb +238 -185
- data/lib/aspera/ascmd.rb +93 -83
- data/lib/aspera/ats_api.rb +11 -10
- data/lib/aspera/cli/basic_auth_plugin.rb +13 -14
- data/lib/aspera/cli/extended_value.rb +42 -33
- data/lib/aspera/cli/formater.rb +142 -108
- data/lib/aspera/cli/info.rb +17 -0
- data/lib/aspera/cli/listener/line_dump.rb +3 -2
- data/lib/aspera/cli/listener/logger.rb +2 -1
- data/lib/aspera/cli/listener/progress.rb +16 -18
- data/lib/aspera/cli/listener/progress_multi.rb +18 -21
- data/lib/aspera/cli/main.rb +173 -149
- data/lib/aspera/cli/manager.rb +163 -168
- data/lib/aspera/cli/plugin.rb +43 -31
- data/lib/aspera/cli/plugins/alee.rb +6 -6
- data/lib/aspera/cli/plugins/aoc.rb +405 -370
- data/lib/aspera/cli/plugins/ats.rb +86 -79
- data/lib/aspera/cli/plugins/bss.rb +14 -16
- data/lib/aspera/cli/plugins/config.rb +580 -362
- data/lib/aspera/cli/plugins/console.rb +23 -19
- data/lib/aspera/cli/plugins/cos.rb +18 -18
- data/lib/aspera/cli/plugins/faspex.rb +201 -158
- data/lib/aspera/cli/plugins/faspex5.rb +80 -57
- data/lib/aspera/cli/plugins/node.rb +183 -166
- data/lib/aspera/cli/plugins/orchestrator.rb +71 -67
- data/lib/aspera/cli/plugins/preview.rb +92 -96
- data/lib/aspera/cli/plugins/server.rb +79 -75
- data/lib/aspera/cli/plugins/shares.rb +35 -19
- data/lib/aspera/cli/plugins/sync.rb +20 -22
- data/lib/aspera/cli/transfer_agent.rb +76 -113
- data/lib/aspera/cli/version.rb +2 -1
- data/lib/aspera/colors.rb +35 -27
- data/lib/aspera/command_line_builder.rb +48 -34
- data/lib/aspera/cos_node.rb +29 -21
- data/lib/aspera/data_repository.rb +3 -2
- data/lib/aspera/environment.rb +50 -45
- data/lib/aspera/fasp/{manager.rb → agent_base.rb} +28 -25
- data/lib/aspera/fasp/{connect.rb → agent_connect.rb} +52 -43
- data/lib/aspera/fasp/{local.rb → agent_direct.rb} +58 -72
- data/lib/aspera/fasp/{http_gw.rb → agent_httpgw.rb} +37 -43
- data/lib/aspera/fasp/{node.rb → agent_node.rb} +35 -16
- data/lib/aspera/fasp/agent_trsdk.rb +104 -0
- data/lib/aspera/fasp/error.rb +2 -1
- data/lib/aspera/fasp/error_info.rb +68 -52
- data/lib/aspera/fasp/installation.rb +152 -124
- data/lib/aspera/fasp/listener.rb +1 -0
- data/lib/aspera/fasp/parameters.rb +87 -92
- data/lib/aspera/fasp/parameters.yaml +305 -249
- data/lib/aspera/fasp/resume_policy.rb +11 -14
- data/lib/aspera/fasp/transfer_spec.rb +26 -0
- data/lib/aspera/fasp/uri.rb +22 -21
- data/lib/aspera/faspex_gw.rb +55 -89
- data/lib/aspera/hash_ext.rb +4 -3
- data/lib/aspera/id_generator.rb +8 -7
- data/lib/aspera/keychain/encrypted_hash.rb +121 -0
- data/lib/aspera/keychain/macos_security.rb +90 -0
- data/lib/aspera/log.rb +55 -37
- data/lib/aspera/nagios.rb +13 -12
- data/lib/aspera/node.rb +30 -25
- data/lib/aspera/oauth.rb +175 -226
- data/lib/aspera/open_application.rb +4 -3
- data/lib/aspera/persistency_action_once.rb +6 -6
- data/lib/aspera/persistency_folder.rb +5 -9
- data/lib/aspera/preview/file_types.rb +6 -5
- data/lib/aspera/preview/generator.rb +25 -24
- data/lib/aspera/preview/options.rb +16 -14
- data/lib/aspera/preview/utils.rb +98 -98
- data/lib/aspera/{proxy_auto_config.erb.js → proxy_auto_config.js} +23 -31
- data/lib/aspera/proxy_auto_config.rb +111 -20
- data/lib/aspera/rest.rb +154 -135
- data/lib/aspera/rest_call_error.rb +2 -2
- data/lib/aspera/rest_error_analyzer.rb +23 -25
- data/lib/aspera/rest_errors_aspera.rb +15 -14
- data/lib/aspera/ssh.rb +12 -10
- data/lib/aspera/sync.rb +42 -41
- data/lib/aspera/temp_file_manager.rb +18 -14
- data/lib/aspera/timer_limiter.rb +2 -1
- data/lib/aspera/uri_reader.rb +7 -5
- data/lib/aspera/web_auth.rb +79 -76
- metadata +116 -29
- data/docs/Makefile +0 -66
- data/docs/README.erb.md +0 -3973
- data/docs/README.md +0 -13
- data/docs/diagrams.txt +0 -49
- data/docs/doc_tools.rb +0 -58
- data/lib/aspera/api_detector.rb +0 -60
- data/lib/aspera/cli/plugins/shares2.rb +0 -114
- data/lib/aspera/secrets.rb +0 -20
data/docs/README.md
DELETED
@@ -1,13 +0,0 @@
|
|
1
|
-
docs/README.erb.md uses markdown format with embeded ruby macros (erb)
|
2
|
-
|
3
|
-
/README.md is generated after compilation of README.erb.md
|
4
|
-
|
5
|
-
docs/README.erb.md contains the following macros:
|
6
|
-
|
7
|
-
* `<%=cmd%>` just the command line tool name
|
8
|
-
* `<%=tool%>` the tool name in courrier to be included in text paragraphs
|
9
|
-
* `<%=evp%>` env var prefix
|
10
|
-
* `<%=opprst%>` option preset
|
11
|
-
* `<%=prst%>` link to preset section, name and link to preset
|
12
|
-
* `<%=prstt%>` preset in title
|
13
|
-
|
data/docs/diagrams.txt
DELETED
@@ -1,49 +0,0 @@
|
|
1
|
-
|
2
|
-
The following section contains the diagrams posted on support KB.
|
3
|
-
mde with: https://www.websequencediagrams.com/
|
4
|
-
|
5
|
-
|
6
|
-
-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
|
7
|
-
title general Aspera Files APIs use
|
8
|
-
Client->+Files API: GET authorization(parameters)
|
9
|
-
Files API-->-Client: Result=Bearer token
|
10
|
-
Client-->Files API: use APIs with Authorization=Bearer
|
11
|
-
|
12
|
-
|
13
|
-
-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
|
14
|
-
title basic OAuth with grant_type=password (username/password, only local users, not google, SAML)
|
15
|
-
Client->+Files API: GET authorization(username/password)
|
16
|
-
Files API-->-Client: Result=Bearer token
|
17
|
-
Client-->Files API: use APIs with Authorization=Bearer
|
18
|
-
|
19
|
-
|
20
|
-
-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
|
21
|
-
title OAuth with grant_type=code (any user: local users, google, SAML)
|
22
|
-
note right of Client
|
23
|
-
* Client is registered in "Files", with redirect_uri
|
24
|
-
* Files generates client_id and secret
|
25
|
-
* Listen on "redirect_uri" end point (TCP port)
|
26
|
-
* build "login address" for user:
|
27
|
-
api.asperafiles.com/oauth2/<org>/authorize
|
28
|
-
* open a browser for user with this URL
|
29
|
-
end note
|
30
|
-
Client->*Local Port: listen
|
31
|
-
Client->Browser: open login url in browser (give redirect_uri and scope)
|
32
|
-
Browser->Files: load login page
|
33
|
-
note right of Browser: user clicks on "google auth"
|
34
|
-
Browser->+Google: display google login page
|
35
|
-
note right of Browser: user authenticates
|
36
|
-
Google-->-Browser: browser is redirected to the "redirect URI" providing a "code"
|
37
|
-
Browser->Local Port: browser provides code to client
|
38
|
-
Local Port-->Client: read code
|
39
|
-
Client->Local Port: close
|
40
|
-
destroy Local Port
|
41
|
-
note right of Client: close listening port after use
|
42
|
-
Client->+Files: GET oauth/token(grant_type=code)
|
43
|
-
Files-->-Client: Bearer token
|
44
|
-
Client-->Files API: use APIs with Authorization=Bearer
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
data/docs/doc_tools.rb
DELETED
@@ -1,58 +0,0 @@
|
|
1
|
-
# get transfer spec parameter description
|
2
|
-
$LOAD_PATH.unshift(ENV["INCL_DIR_GEM"])
|
3
|
-
require 'aspera/fasp/parameters'
|
4
|
-
|
5
|
-
# check that required env vars exist, and files
|
6
|
-
%w{EXENAME GEMSPEC INCL_USAGE INCL_COMMANDS INCL_ASESSION INCL_DIR_GEM}.each do |e|
|
7
|
-
raise "missing env var #{e}" unless ENV.has_key?(e)
|
8
|
-
raise "missing file #{ENV[e]}" unless File.exist?(ENV[e]) or !e.start_with?('INCL_')
|
9
|
-
end
|
10
|
-
|
11
|
-
# set values used in ERB
|
12
|
-
# just command name
|
13
|
-
def cmd;ENV['EXENAME'];end
|
14
|
-
|
15
|
-
# used in text with formatting of command
|
16
|
-
def tool;'`'+cmd+'`';end
|
17
|
-
|
18
|
-
# prefix for env vars
|
19
|
-
def evp;cmd.upcase+'_';end
|
20
|
-
|
21
|
-
# just the name for "option preset"
|
22
|
-
def opprst;'option preset';end
|
23
|
-
|
24
|
-
# name with link
|
25
|
-
def prst;'['+opprst+'](#lprt)';end
|
26
|
-
|
27
|
-
# name with link (plural)
|
28
|
-
def prsts;'['+opprst+'s](#lprt)';end
|
29
|
-
|
30
|
-
# in title
|
31
|
-
def prstt;opprst.capitalize;end
|
32
|
-
|
33
|
-
def gemspec;Gem::Specification::load(ENV['GEMSPEC']) or raise "error loading #{ENV["GEMSPEC"]}";end
|
34
|
-
|
35
|
-
def geminstadd;gemspec.version.to_s.match(/\.[^0-9]/)?' --pre':'';end
|
36
|
-
|
37
|
-
# transfer spec description generation
|
38
|
-
def spec_table
|
39
|
-
r='<table><tr><th>Field</th><th>Type</th>'
|
40
|
-
Aspera::Fasp::Parameters::SUPPORTED_AGENTS_SHORT.each do |c|
|
41
|
-
r << '<th>'<<c.to_s.upcase<<'</th>'
|
42
|
-
end
|
43
|
-
r << '<th>Description</th></tr>'
|
44
|
-
Aspera::Fasp::Parameters.man_table.each do |p|
|
45
|
-
p[:description] << (p[:description].empty? ? '' : "\n") << "(" << p[:cli] << ")" unless p[:cli].to_s.empty?
|
46
|
-
p.delete(:cli)
|
47
|
-
p.keys.each{|c|p[c]=' ' if p[c].to_s.empty?}
|
48
|
-
p[:description].gsub!("\n",'<br/>')
|
49
|
-
p[:type].gsub!(',','<br/>')
|
50
|
-
r << '<tr><td>'<<p[:name]<<'</td><td>'<<p[:type]<<'</td>'
|
51
|
-
Aspera::Fasp::Parameters::SUPPORTED_AGENTS_SHORT.each do |c|
|
52
|
-
r << '<td>'<<p[c]<<'</td>'
|
53
|
-
end
|
54
|
-
r << '<td>'<<p[:description]<<'</td></tr>'
|
55
|
-
end
|
56
|
-
r << '</table>'
|
57
|
-
return r
|
58
|
-
end
|
data/lib/aspera/api_detector.rb
DELETED
@@ -1,60 +0,0 @@
|
|
1
|
-
require 'aspera/log'
|
2
|
-
require 'aspera/rest'
|
3
|
-
|
4
|
-
module Aspera
|
5
|
-
# detect Aspera product by calling API
|
6
|
-
class ApiDetector
|
7
|
-
# @return a hash: {:product=>:node,:version=>'unknown'}
|
8
|
-
# if not found: {:product=>:unknown,:version=>'unknown'}
|
9
|
-
def self.discover_product(url)
|
10
|
-
#uri=URI.parse(url)
|
11
|
-
api=Rest.new({:base_url=>url})
|
12
|
-
# Node
|
13
|
-
begin
|
14
|
-
result=api.call({:operation=>'GET',:subpath=>'ping'})
|
15
|
-
if result[:http].body.eql?('')
|
16
|
-
return {:product=>:node,:version=>'unknown'}
|
17
|
-
end
|
18
|
-
rescue SocketError => e
|
19
|
-
raise e
|
20
|
-
rescue => e
|
21
|
-
Log.log.debug("not node (#{e.class}: #{e})")
|
22
|
-
end
|
23
|
-
# AoC
|
24
|
-
begin
|
25
|
-
result=api.call({:operation=>'GET',:subpath=>'',:headers=>{'Accept'=>'text/html'}})
|
26
|
-
if result[:http].body.include?('content="AoC"')
|
27
|
-
return {:product=>:aoc,:version=>'unknown'}
|
28
|
-
end
|
29
|
-
rescue SocketError => e
|
30
|
-
raise e
|
31
|
-
rescue => e
|
32
|
-
Log.log.debug("not aoc (#{e.class}: #{e})")
|
33
|
-
end
|
34
|
-
# Faspex
|
35
|
-
begin
|
36
|
-
result=api.call({:operation=>'POST',:subpath=>'aspera/faspex',:headers=>{'Accept'=>'application/xrds+xml'},:text_body_params=>''})
|
37
|
-
if result[:http].body.start_with?('<?xml')
|
38
|
-
res_s=XmlSimple.xml_in(result[:http].body, {"ForceArray" => false})
|
39
|
-
version=res_s['XRD']['application']['version']
|
40
|
-
#return JSON.pretty_generate(res_s)
|
41
|
-
end
|
42
|
-
return {:product=>:faspex,:version=>version}
|
43
|
-
rescue
|
44
|
-
Log.log.debug("not faspex")
|
45
|
-
end
|
46
|
-
# Shares
|
47
|
-
begin
|
48
|
-
result=api.read('node_api/app')
|
49
|
-
Log.log.warn("not supposed to work")
|
50
|
-
rescue RestCallError => e
|
51
|
-
if e.response.code.to_s.eql?('401') and e.response.body.eql?('{"error":{"user_message":"API user authentication failed"}}')
|
52
|
-
return {:product=>:shares,:version=>'unknown'}
|
53
|
-
end
|
54
|
-
Log.log.warn("not shares: #{e.response.code} #{e.response.body}")
|
55
|
-
rescue
|
56
|
-
end
|
57
|
-
return {:product=>:unknown,:version=>'unknown'}
|
58
|
-
end
|
59
|
-
end
|
60
|
-
end
|
@@ -1,114 +0,0 @@
|
|
1
|
-
require 'aspera/cli/plugins/node'
|
2
|
-
require 'uri'
|
3
|
-
|
4
|
-
module Aspera
|
5
|
-
module Cli
|
6
|
-
module Plugins
|
7
|
-
class Shares2 < BasicAuthPlugin
|
8
|
-
def initialize(env)
|
9
|
-
super(env)
|
10
|
-
self.options.add_opt_simple(:organization,"organization")
|
11
|
-
self.options.add_opt_simple(:project,"project")
|
12
|
-
self.options.add_opt_simple(:share,"share")
|
13
|
-
self.options.parse_options!
|
14
|
-
return if env[:man_only]
|
15
|
-
unless env[:man_only]
|
16
|
-
# get parameters
|
17
|
-
shares2_api_base_url=self.options.get_option(:url,:mandatory)
|
18
|
-
shares2_username=self.options.get_option(:username,:mandatory)
|
19
|
-
shares2_password=self.options.get_option(:password,:mandatory)
|
20
|
-
|
21
|
-
# create object for REST calls to Shares2
|
22
|
-
@api_shares2_oauth=Rest.new({
|
23
|
-
:base_url => shares2_api_base_url,
|
24
|
-
:auth => {
|
25
|
-
:type => :oauth2,
|
26
|
-
:base_url => shares2_api_base_url+'/oauth2',
|
27
|
-
:grant => :header_userpass,
|
28
|
-
:user_name => shares2_username,
|
29
|
-
:user_pass => shares2_password
|
30
|
-
}})
|
31
|
-
|
32
|
-
@api_node=Rest.new({
|
33
|
-
:base_url => shares2_api_base_url+'/node_api',
|
34
|
-
:auth => {
|
35
|
-
:type => :basic,
|
36
|
-
:username => shares2_username,
|
37
|
-
:password => shares2_password}})
|
38
|
-
end
|
39
|
-
end
|
40
|
-
|
41
|
-
# path_prefix is either "" or "res/id/"
|
42
|
-
# adds : prefix+"res/id/"
|
43
|
-
# modify parameter string
|
44
|
-
def set_resource_path_by_id_or_name(resource_path,resource_sym)
|
45
|
-
res_id=self.options.get_option(resource_sym,:mandatory)
|
46
|
-
# lets get the class path
|
47
|
-
resource_path<<resource_sym.to_s+'s'
|
48
|
-
# is this an integer ? or a name
|
49
|
-
if res_id.to_i.to_s != res_id
|
50
|
-
all=@api_shares2_oauth.read(resource_path)[:data]
|
51
|
-
one=all.select{|i|i['name'].start_with?(res_id)}
|
52
|
-
Log.log.debug(one)
|
53
|
-
raise CliBadArgument,"No matching name for #{res_id} in #{all}" if one.empty?
|
54
|
-
raise CliBadArgument,"More than one match: #{one}" if one.length > 1
|
55
|
-
res_id=one.first['id'].to_s
|
56
|
-
end
|
57
|
-
Log.log.debug("res_id=#{res_id}")
|
58
|
-
resource_path<<'/'+res_id+'/'
|
59
|
-
return resource_path
|
60
|
-
end
|
61
|
-
|
62
|
-
# path_prefix is empty or ends with slash
|
63
|
-
def process_entity_action(resource_sym,path_prefix)
|
64
|
-
resource_path=path_prefix+resource_sym.to_s+'s'
|
65
|
-
operations=[:list,:create,:delete]
|
66
|
-
command=self.options.get_next_command(operations)
|
67
|
-
case command
|
68
|
-
when :create
|
69
|
-
params=self.options.get_next_argument("creation data (json structure)")
|
70
|
-
resp=@api_shares2_oauth.create(resource_path,params)
|
71
|
-
return {:data=>resp[:data],:type => :other_struct}
|
72
|
-
when :list
|
73
|
-
default_fields=['id','name']
|
74
|
-
query=self.options.get_option(:query,:optional)
|
75
|
-
args=query.nil? ? nil : {'json_query'=>query}
|
76
|
-
Log.log.debug("#{args}".bg_red)
|
77
|
-
return {:data=>@api_shares2_oauth.read(resource_path,args)[:data],:fields=>default_fields,:type=>:object_list}
|
78
|
-
when :delete
|
79
|
-
@api_shares2_oauth.delete(set_resource_path_by_id_or_name(path_prefix,resource_sym))
|
80
|
-
return Main.result_status('deleted')
|
81
|
-
when :info
|
82
|
-
return {:type=>:other_struct,:data=>@api_shares2_oauth.read(set_resource_path_by_id_or_name(path_prefix,resource_sym),args)[:data]}
|
83
|
-
else raise :ERROR
|
84
|
-
end
|
85
|
-
end
|
86
|
-
|
87
|
-
ACTIONS=[ :repository,:organization,:project,:team,:share,:appinfo,:userinfo,:admin]
|
88
|
-
|
89
|
-
def execute_action
|
90
|
-
command=self.options.get_next_command(ACTIONS)
|
91
|
-
case command
|
92
|
-
when :repository
|
93
|
-
command=self.options.get_next_command(Node::COMMON_ACTIONS)
|
94
|
-
return Node.new(@agents.merge(skip_basic_auth_options: true, node_api: @api_node)).execute_action(command)
|
95
|
-
when :appinfo
|
96
|
-
node_info=@api_node.call({:operation=>'GET',:subpath=>'app',:headers=>{'Accept'=>'application/json','Content-Type'=>'application/json'}})[:data]
|
97
|
-
return { :type=>:single_object ,:data => node_info }
|
98
|
-
when :userinfo
|
99
|
-
node_info=@api_node.call({:operation=>'GET',:subpath=>'current_user',:headers=>{'Accept'=>'application/json','Content-Type'=>'application/json'}})[:data]
|
100
|
-
return { :type=>:single_object ,:data => node_info }
|
101
|
-
when :organization,:project,:share,:team
|
102
|
-
prefix=''
|
103
|
-
set_resource_path_by_id_or_name(prefix,:organization) if [:project,:team,:share].include?(command)
|
104
|
-
set_resource_path_by_id_or_name(prefix,:project) if [:share].include?(command)
|
105
|
-
process_entity_action(command,prefix)
|
106
|
-
when :admin
|
107
|
-
command=self.options.get_next_command([:users,:groups,:nodes])
|
108
|
-
return self.entity_action(@api_shares2_oauth,"system/#{command}",nil,:id)
|
109
|
-
end # command
|
110
|
-
end # execute_action
|
111
|
-
end # Files
|
112
|
-
end # Plugins
|
113
|
-
end # Cli
|
114
|
-
end # Aspera
|
data/lib/aspera/secrets.rb
DELETED
@@ -1,20 +0,0 @@
|
|
1
|
-
module Aspera
|
2
|
-
# Manage secrets in CLI using secure way (encryption, wallet, etc...)
|
3
|
-
class Secrets
|
4
|
-
attr_accessor :default_secret,:all_secrets
|
5
|
-
def initialize()
|
6
|
-
@default_secret=nil
|
7
|
-
@all_secrets={}
|
8
|
-
end
|
9
|
-
|
10
|
-
def get_secret(id=nil,mandatory=true)
|
11
|
-
secret=@default_secret || @all_secrets[id]
|
12
|
-
raise "please provide secret for #{id}" if secret.nil? and mandatory
|
13
|
-
return secret
|
14
|
-
end
|
15
|
-
|
16
|
-
def get_secrets
|
17
|
-
return @all_secrets
|
18
|
-
end
|
19
|
-
end
|
20
|
-
end
|