aspera-cli 4.2.1 → 4.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +1580 -946
- data/bin/ascli +1 -1
- data/bin/asession +3 -5
- data/docs/Makefile +8 -11
- data/docs/README.erb.md +1521 -829
- data/docs/doc_tools.rb +58 -0
- data/docs/test_env.conf +3 -1
- data/examples/faspex4.rb +28 -19
- data/examples/transfer.rb +2 -2
- data/lib/aspera/aoc.rb +157 -134
- data/lib/aspera/cli/listener/progress_multi.rb +5 -5
- data/lib/aspera/cli/main.rb +106 -48
- data/lib/aspera/cli/manager.rb +19 -20
- data/lib/aspera/cli/plugin.rb +22 -7
- data/lib/aspera/cli/plugins/aoc.rb +260 -208
- data/lib/aspera/cli/plugins/ats.rb +11 -10
- data/lib/aspera/cli/plugins/bss.rb +2 -2
- data/lib/aspera/cli/plugins/config.rb +360 -189
- data/lib/aspera/cli/plugins/faspex.rb +119 -56
- data/lib/aspera/cli/plugins/faspex5.rb +32 -17
- data/lib/aspera/cli/plugins/node.rb +72 -31
- data/lib/aspera/cli/plugins/orchestrator.rb +5 -3
- data/lib/aspera/cli/plugins/preview.rb +94 -68
- data/lib/aspera/cli/plugins/server.rb +16 -5
- data/lib/aspera/cli/plugins/shares.rb +17 -0
- data/lib/aspera/cli/transfer_agent.rb +64 -82
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/command_line_builder.rb +48 -31
- data/lib/aspera/cos_node.rb +4 -3
- data/lib/aspera/environment.rb +4 -4
- data/lib/aspera/fasp/{manager.rb → agent_base.rb} +7 -6
- data/lib/aspera/fasp/{connect.rb → agent_connect.rb} +46 -39
- data/lib/aspera/fasp/{local.rb → agent_direct.rb} +42 -38
- data/lib/aspera/fasp/{http_gw.rb → agent_httpgw.rb} +50 -29
- data/lib/aspera/fasp/{node.rb → agent_node.rb} +43 -4
- data/lib/aspera/fasp/agent_trsdk.rb +106 -0
- data/lib/aspera/fasp/default.rb +17 -0
- data/lib/aspera/fasp/installation.rb +64 -48
- data/lib/aspera/fasp/parameters.rb +78 -91
- data/lib/aspera/fasp/parameters.yaml +531 -0
- data/lib/aspera/fasp/uri.rb +1 -1
- data/lib/aspera/faspex_gw.rb +12 -11
- data/lib/aspera/id_generator.rb +22 -0
- data/lib/aspera/keychain/encrypted_hash.rb +120 -0
- data/lib/aspera/keychain/macos_security.rb +94 -0
- data/lib/aspera/log.rb +45 -32
- data/lib/aspera/node.rb +9 -4
- data/lib/aspera/oauth.rb +116 -100
- data/lib/aspera/persistency_action_once.rb +11 -7
- data/lib/aspera/persistency_folder.rb +6 -26
- data/lib/aspera/rest.rb +66 -50
- data/lib/aspera/sync.rb +40 -35
- data/lib/aspera/timer_limiter.rb +22 -0
- metadata +86 -29
- data/docs/transfer_spec.html +0 -99
- data/lib/aspera/api_detector.rb +0 -60
- data/lib/aspera/fasp/aoc.rb +0 -24
- data/lib/aspera/secrets.rb +0 -20
data/docs/doc_tools.rb
ADDED
@@ -0,0 +1,58 @@
|
|
1
|
+
# get transfer spec parameter description
|
2
|
+
$LOAD_PATH.unshift(ENV["INCL_DIR_GEM"])
|
3
|
+
require 'aspera/fasp/parameters'
|
4
|
+
|
5
|
+
# check that required env vars exist, and files
|
6
|
+
%w{EXENAME GEMSPEC INCL_USAGE INCL_COMMANDS INCL_ASESSION INCL_DIR_GEM}.each do |e|
|
7
|
+
raise "missing env var #{e}" unless ENV.has_key?(e)
|
8
|
+
raise "missing file #{ENV[e]}" unless File.exist?(ENV[e]) or !e.start_with?('INCL_')
|
9
|
+
end
|
10
|
+
|
11
|
+
# set values used in ERB
|
12
|
+
# just command name
|
13
|
+
def cmd;ENV['EXENAME'];end
|
14
|
+
|
15
|
+
# used in text with formatting of command
|
16
|
+
def tool;'`'+cmd+'`';end
|
17
|
+
|
18
|
+
# prefix for env vars
|
19
|
+
def evp;cmd.upcase+'_';end
|
20
|
+
|
21
|
+
# just the name for "option preset"
|
22
|
+
def opprst;'option preset';end
|
23
|
+
|
24
|
+
# name with link
|
25
|
+
def prst;'['+opprst+'](#lprt)';end
|
26
|
+
|
27
|
+
# name with link (plural)
|
28
|
+
def prsts;'['+opprst+'s](#lprt)';end
|
29
|
+
|
30
|
+
# in title
|
31
|
+
def prstt;opprst.capitalize;end
|
32
|
+
|
33
|
+
def gemspec;Gem::Specification::load(ENV['GEMSPEC']) or raise "error loading #{ENV["GEMSPEC"]}";end
|
34
|
+
|
35
|
+
def geminstadd;gemspec.version.to_s.match(/\.[^0-9]/)?' --pre':'';end
|
36
|
+
|
37
|
+
# transfer spec description generation
|
38
|
+
def spec_table
|
39
|
+
r='<table><tr><th>Field</th><th>Type</th>'
|
40
|
+
Aspera::Fasp::Parameters::SUPPORTED_AGENTS_SHORT.each do |c|
|
41
|
+
r << '<th>'<<c.to_s.upcase<<'</th>'
|
42
|
+
end
|
43
|
+
r << '<th>Description</th></tr>'
|
44
|
+
Aspera::Fasp::Parameters.man_table.each do |p|
|
45
|
+
p[:description] << (p[:description].empty? ? '' : "\n") << "(" << p[:cli] << ")" unless p[:cli].to_s.empty?
|
46
|
+
p.delete(:cli)
|
47
|
+
p.keys.each{|c|p[c]=' ' if p[c].to_s.empty?}
|
48
|
+
p[:description].gsub!("\n",'<br/>')
|
49
|
+
p[:type].gsub!(',','<br/>')
|
50
|
+
r << '<tr><td>'<<p[:name]<<'</td><td>'<<p[:type]<<'</td>'
|
51
|
+
Aspera::Fasp::Parameters::SUPPORTED_AGENTS_SHORT.each do |c|
|
52
|
+
r << '<td>'<<p[c]<<'</td>'
|
53
|
+
end
|
54
|
+
r << '<td>'<<p[:description]<<'</td></tr>'
|
55
|
+
end
|
56
|
+
r << '</table>'
|
57
|
+
return r
|
58
|
+
end
|
data/docs/test_env.conf
CHANGED
@@ -5,7 +5,7 @@ default:
|
|
5
5
|
config: cli_default
|
6
6
|
aoc: tst_aoc1
|
7
7
|
faspex: tst_faspex
|
8
|
-
faspex5:
|
8
|
+
faspex5: tst_faspex5
|
9
9
|
shares: tst_shares_1
|
10
10
|
shares2: tst_shares2
|
11
11
|
node: tst_node
|
@@ -65,6 +65,7 @@ tst_faspex5:
|
|
65
65
|
client_secret: your value here
|
66
66
|
private_key: your value here
|
67
67
|
username: your value here
|
68
|
+
password: your value here
|
68
69
|
tst_shares:
|
69
70
|
url: your value here
|
70
71
|
username: your value here
|
@@ -155,4 +156,5 @@ misc:
|
|
155
156
|
email_external: your value here
|
156
157
|
aoc_org: your value here
|
157
158
|
aoc_user_email: your value here
|
159
|
+
aoc_workspace2: your value here
|
158
160
|
http_gw_fqdn_port: your value here
|
data/examples/faspex4.rb
CHANGED
@@ -1,29 +1,37 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
|
+
# find Faspex API here: https://developer.ibm.com/apis/catalog/?search=faspex
|
3
|
+
# this example makes use of class Aspera::Rest for REST calls, alternatively class RestClient of gem rest-client could be used
|
4
|
+
# this example makes use of class Aspera::Fasp::AgentDirect for transfers, alternatively the official "Transfer SDK" could be used
|
5
|
+
# Aspera SDK can be downloaded with: `ascli conf ascp install` , it installs in $HOME/.aspera/ascli/sdk
|
2
6
|
require 'aspera/rest'
|
3
7
|
require 'aspera/log'
|
4
|
-
require 'aspera/fasp/
|
5
|
-
require 'aspera/cli/listener/line_dump'
|
6
|
-
require 'aspera/cli/extended_value'
|
8
|
+
require 'aspera/fasp/agent_direct'
|
7
9
|
|
8
|
-
|
10
|
+
tmpdir=ENV['tmp']||Dir.tmpdir || '.'
|
11
|
+
|
12
|
+
# Set high log level for the example, decrease to :warn usually
|
9
13
|
Aspera::Log.instance.level=:debug
|
10
14
|
|
11
|
-
#
|
15
|
+
# Set folder where SDK is installed (mandatory)
|
12
16
|
# (if ascp is not there, the lib will try to find in usual locations)
|
13
|
-
|
17
|
+
# (if data files are not there, they will be created)
|
18
|
+
Aspera::Fasp::Installation.instance.folder = tmpdir
|
14
19
|
|
15
20
|
if ! ARGV.length.eql?(3)
|
16
|
-
Aspera::Log.log.error("
|
21
|
+
Aspera::Log.log.error("Wrong number of args: #{ARGV.length}")
|
17
22
|
Aspera::Log.log.error("Usage: #{$0} <faspex URL> <faspex username> <faspex password>")
|
18
23
|
Aspera::Log.log.error("Example: #{$0} https://faspex.com/aspera/faspex john p@sSw0rd")
|
19
24
|
Process.exit(1)
|
20
25
|
end
|
21
26
|
|
22
|
-
faspex_url=ARGV[0]
|
27
|
+
faspex_url=ARGV[0] # typically: https://faspex.example.com/aspera/faspex
|
23
28
|
faspex_user=ARGV[1]
|
24
29
|
faspex_pass=ARGV[2]
|
25
30
|
|
26
|
-
#
|
31
|
+
# comment out this if certificate is valid, keep line to ignore certificate
|
32
|
+
Aspera::Rest.insecure=true
|
33
|
+
|
34
|
+
# 1: Faspex 4 API v3
|
27
35
|
#---------------
|
28
36
|
|
29
37
|
# create REST API object
|
@@ -35,34 +43,34 @@ api_v3=Aspera::Rest.new({
|
|
35
43
|
:password => faspex_pass
|
36
44
|
}})
|
37
45
|
|
46
|
+
# very simple api call
|
38
47
|
api_v3.read('me')
|
39
48
|
|
40
49
|
# 2: send a package
|
41
50
|
#---------------
|
42
51
|
|
43
52
|
# create a sample file to send
|
44
|
-
file_to_send='
|
53
|
+
file_to_send=File.join(tmpdir,'myfile.bin')
|
45
54
|
File.open(file_to_send, "w") {|f| f.write("sample data") }
|
46
55
|
# package creation parameters
|
47
56
|
package_create_params={'delivery'=>{'title'=>'test package','recipients'=>['aspera.user1@gmail.com'],'sources'=>[{'paths'=>[file_to_send]}]}}
|
48
57
|
pkg_created=api_v3.create('send',package_create_params)[:data]
|
49
|
-
# get transfer specification
|
58
|
+
# get transfer specification (normally: only one)
|
50
59
|
transfer_spec=pkg_created['xfer_sessions'].first
|
51
60
|
# set paths of files to send
|
52
|
-
transfer_spec['paths']=['source'=>file_to_send]
|
53
|
-
# get
|
54
|
-
|
55
|
-
# disable ascp output on stdout to not mix with JSON events
|
56
|
-
client.quiet=true
|
61
|
+
transfer_spec['paths']=[{'source'=>file_to_send}]
|
62
|
+
# get local agent (ascp), disable ascp output on stdout to not mix with JSON events
|
63
|
+
transfer_client=Aspera::Fasp::AgentDirect.new({quiet: true})
|
57
64
|
# start transfer (asynchronous)
|
58
|
-
job_id=
|
59
|
-
|
65
|
+
job_id=transfer_client.start_transfer(transfer_spec)
|
66
|
+
# wait for all transfer completion (for the example)
|
67
|
+
result=transfer_client.wait_for_transfers_completion
|
60
68
|
# notify of any transfer error
|
61
69
|
result.select{|i|!i.eql?(:success)}.each do |e|
|
62
70
|
Aspera::Log.log.error("A transfer error occured: #{e.message}")
|
63
71
|
end
|
64
72
|
|
65
|
-
#
|
73
|
+
# 3: Faspex 4 API v4
|
66
74
|
#---------------
|
67
75
|
api_v4=Aspera::Rest.new({
|
68
76
|
:base_url => faspex_url+'/api',
|
@@ -75,4 +83,5 @@ api_v4=Aspera::Rest.new({
|
|
75
83
|
:scope => 'admin'
|
76
84
|
}})
|
77
85
|
|
86
|
+
# Use it. Note that Faspex 4 API v4 is totally different from Faspex 4 v3 APIs, see ref on line 2
|
78
87
|
Aspera::Log.dump('users',api_v4.read('users')[:data])
|
data/examples/transfer.rb
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
# Example: transfer a file using one of the provided transfer agents
|
3
3
|
# location of ascp can be specified with env var "ascp"
|
4
4
|
# temp folder can be specified with env var "tmp"
|
5
|
-
require 'aspera/fasp/
|
5
|
+
require 'aspera/fasp/agent_direct'
|
6
6
|
require 'aspera/fasp/listener'
|
7
7
|
require 'aspera/fasp/installation'
|
8
8
|
require 'aspera/log'
|
@@ -39,7 +39,7 @@ Aspera::Fasp::Installation.instance.ascp_path=ENV['ascp'] if ENV.has_key?('ascp'
|
|
39
39
|
#
|
40
40
|
|
41
41
|
# get FASP Manager singleton based on above ascp location
|
42
|
-
fasp_manager=Aspera::Fasp::
|
42
|
+
fasp_manager=Aspera::Fasp::AgentDirect.new
|
43
43
|
|
44
44
|
# Note that it would also be possible to start transfers using other agents
|
45
45
|
#require 'aspera/fasp/connect'
|
data/lib/aspera/aoc.rb
CHANGED
@@ -2,6 +2,7 @@ require 'aspera/log'
|
|
2
2
|
require 'aspera/rest'
|
3
3
|
require 'aspera/hash_ext'
|
4
4
|
require 'aspera/data_repository'
|
5
|
+
require 'aspera/fasp/default'
|
5
6
|
require 'base64'
|
6
7
|
|
7
8
|
module Aspera
|
@@ -17,19 +18,20 @@ module Aspera
|
|
17
18
|
# to avoid infinite loop in pub link redirection
|
18
19
|
MAX_REDIRECT=10
|
19
20
|
CLIENT_APPS=['aspera.global-cli-client','aspera.drive']
|
21
|
+
# index offset in data repository of client app
|
20
22
|
DATA_REPO_INDEX_START = 4
|
23
|
+
# cookie prefix so that console can decode identity
|
24
|
+
COOKIE_PREFIX='aspera.aoc'
|
21
25
|
|
22
26
|
# path in URL of public links
|
23
|
-
|
27
|
+
PUBLIC_LINK_PATHS=['/packages/public/receive','/packages/public/send','/files/public']
|
24
28
|
JWT_AUDIENCE='https://api.asperafiles.com/api/v1/oauth2/token'
|
25
29
|
OAUTH_API_SUBPATH='api/v1/oauth2'
|
26
|
-
|
27
|
-
|
28
|
-
'ssh_port' => 33001,
|
29
|
-
'fasp_port' => 33001
|
30
|
-
}
|
30
|
+
# minimum fields for user info if retrieval fails
|
31
|
+
USER_INFO_FIELDS_MIN=['name','email','id','default_workspace_id','organization_id']
|
31
32
|
|
32
|
-
private_constant :PRODUCT_NAME,:PROD_DOMAIN,:MAX_REDIRECT,:CLIENT_APPS,:
|
33
|
+
private_constant :PRODUCT_NAME,:PROD_DOMAIN,:MAX_REDIRECT,:CLIENT_APPS,:PUBLIC_LINK_PATHS,:JWT_AUDIENCE,
|
34
|
+
:OAUTH_API_SUBPATH,:COOKIE_PREFIX,:USER_INFO_FIELDS_MIN
|
33
35
|
|
34
36
|
public
|
35
37
|
# various API scopes supported
|
@@ -44,84 +46,119 @@ module Aspera
|
|
44
46
|
FILES_APP='files'
|
45
47
|
PACKAGES_APP='packages'
|
46
48
|
|
47
|
-
|
48
|
-
|
49
|
-
raise "no such pre-defined client: #{client_name}" if client_index.nil?
|
49
|
+
# class static methods
|
50
|
+
class << self
|
50
51
|
# strings /Applications/Aspera\ Drive.app/Contents/MacOS/AsperaDrive|grep -E '.{100}==$'|base64 --decode
|
51
|
-
|
52
|
-
|
52
|
+
def get_client_info(client_name=CLIENT_APPS.first)
|
53
|
+
client_index=CLIENT_APPS.index(client_name)
|
54
|
+
raise "no such pre-defined client: #{client_name}" if client_index.nil?
|
55
|
+
return client_name,Base64.urlsafe_encode64(DataRepository.instance.get_bin(DATA_REPO_INDEX_START+client_index))
|
56
|
+
end
|
53
57
|
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
58
|
+
# @param url of AoC instance
|
59
|
+
# @return organization id in url and AoC domain: ibmaspera.com, asperafiles.com or qa.asperafiles.com, etc...
|
60
|
+
def parse_url(aoc_org_url)
|
61
|
+
uri=URI.parse(aoc_org_url.gsub(/\/+$/,''))
|
62
|
+
instance_fqdn=uri.host
|
63
|
+
Log.log.debug("instance_fqdn=#{instance_fqdn}")
|
64
|
+
raise "No host found in URL.Please check URL format: https://myorg.#{PROD_DOMAIN}" if instance_fqdn.nil?
|
65
|
+
organization,instance_domain=instance_fqdn.split('.',2)
|
66
|
+
Log.log.debug("instance_domain=#{instance_domain}")
|
67
|
+
Log.log.debug("organization=#{organization}")
|
68
|
+
raise "expecting a public FQDN for #{PRODUCT_NAME}" if instance_domain.nil?
|
69
|
+
return organization,instance_domain
|
70
|
+
end
|
67
71
|
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
+
# base API url depends on domain, which could be "qa.xxx"
|
73
|
+
def api_base_url(api_domain=PROD_DOMAIN)
|
74
|
+
return "https://api.#{api_domain}"
|
75
|
+
end
|
72
76
|
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
77
|
+
def metering_api(entitlement_id,customer_id,api_domain=PROD_DOMAIN)
|
78
|
+
return Rest.new({
|
79
|
+
:base_url => "#{api_base_url(api_domain)}/metering/v1",
|
80
|
+
:headers => {'X-Aspera-Entitlement-Authorization' => Rest.basic_creds(entitlement_id,customer_id)}
|
81
|
+
})
|
82
|
+
end
|
79
83
|
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
+
# node API scopes
|
85
|
+
def node_scope(access_key,scope)
|
86
|
+
return 'node.'+access_key+':'+scope
|
87
|
+
end
|
84
88
|
|
85
|
-
|
86
|
-
|
87
|
-
|
89
|
+
def set_use_default_ports(val)
|
90
|
+
@@use_standard_ports=val
|
91
|
+
end
|
88
92
|
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
93
|
+
# check option "link"
|
94
|
+
# if present try to get token value (resolve redirection if short links used)
|
95
|
+
# then set options url/token/auth
|
96
|
+
def resolve_pub_link(rest_opts,public_link_url)
|
97
|
+
return if public_link_url.nil?
|
98
|
+
# set to token if available after redirection
|
99
|
+
url_param_token_pair=nil
|
100
|
+
redirect_count=0
|
101
|
+
loop do
|
102
|
+
uri=URI.parse(public_link_url)
|
103
|
+
if PUBLIC_LINK_PATHS.include?(uri.path)
|
104
|
+
url_param_token_pair=URI::decode_www_form(uri.query).select{|e|e.first.eql?('token')}.first
|
105
|
+
if url_param_token_pair.nil?
|
106
|
+
raise ArgumentError,"link option must be URL with 'token' parameter"
|
107
|
+
end
|
108
|
+
# ok we get it !
|
109
|
+
rest_opts[:org_url]='https://'+uri.host
|
110
|
+
rest_opts[:auth][:grant]=:url_token
|
111
|
+
rest_opts[:auth][:url_token]=url_param_token_pair.last
|
112
|
+
return
|
103
113
|
end
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
114
|
+
Log.log.debug("no expected format: #{public_link_url}")
|
115
|
+
raise "exceeded max redirection: #{MAX_REDIRECT}" if redirect_count > MAX_REDIRECT
|
116
|
+
r = Net::HTTP.get_response(uri)
|
117
|
+
if r.code.start_with?("3")
|
118
|
+
public_link_url = r['location']
|
119
|
+
raise "no location in redirection" if public_link_url.nil?
|
120
|
+
Log.log.debug("redirect to: #{public_link_url}")
|
121
|
+
else
|
122
|
+
# not a redirection
|
123
|
+
raise ArgumentError,'link option must be redirect or have token parameter'
|
124
|
+
end
|
125
|
+
end # loop
|
126
|
+
|
127
|
+
raise RuntimeError,'too many redirections'
|
128
|
+
end
|
129
|
+
|
130
|
+
# additional transfer spec (tags) for package information
|
131
|
+
def package_tags(package_info,operation)
|
132
|
+
return {'tags'=>{'aspera'=>{'files'=>{
|
133
|
+
'package_id' => package_info['id'],
|
134
|
+
'package_name' => package_info['name'],
|
135
|
+
'package_operation' => operation
|
136
|
+
}}}}
|
137
|
+
end
|
138
|
+
|
139
|
+
# add details to show in analytics
|
140
|
+
def analytics_ts(app,direction,ws_id,ws_name)
|
141
|
+
# translate transfer to operation
|
142
|
+
operation=case direction
|
143
|
+
when 'send'; 'upload'
|
144
|
+
when 'receive'; 'download'
|
145
|
+
else raise "ERROR: unexpected value: #{direction}"
|
120
146
|
end
|
121
|
-
end # loop
|
122
147
|
|
123
|
-
|
124
|
-
|
148
|
+
return {
|
149
|
+
'tags' => {
|
150
|
+
'aspera' => {
|
151
|
+
'usage_id' => "aspera.files.workspace.#{ws_id}", # activity tracking
|
152
|
+
'files' => {
|
153
|
+
'files_transfer_action' => "#{operation}_#{app.gsub(/s$/,'')}",
|
154
|
+
'workspace_name' => ws_name, # activity tracking
|
155
|
+
'workspace_id' => ws_id,
|
156
|
+
}
|
157
|
+
}
|
158
|
+
}
|
159
|
+
}
|
160
|
+
end
|
161
|
+
end # static methods
|
125
162
|
|
126
163
|
# @param :link,:url,:auth,:client_id,:client_secret,:scope,:redirect_uri,:private_key,:username,:subpath,:password (for pub link)
|
127
164
|
def initialize(opt)
|
@@ -129,6 +166,7 @@ module Aspera
|
|
129
166
|
# key: access key
|
130
167
|
# value: associated secret
|
131
168
|
@key_chain=nil
|
169
|
+
@user_info=nil
|
132
170
|
|
133
171
|
# init rest params
|
134
172
|
aoc_rest_p={:auth=>{:type =>:oauth2}}
|
@@ -196,50 +234,28 @@ module Aspera
|
|
196
234
|
|
197
235
|
def key_chain=(keychain)
|
198
236
|
raise "keychain already set" unless @key_chain.nil?
|
237
|
+
raise "keychain must have get_secret" unless keychain.respond_to?(:get_secret)
|
199
238
|
@key_chain=keychain
|
200
239
|
nil
|
201
240
|
end
|
202
241
|
|
203
|
-
#
|
204
|
-
def
|
205
|
-
|
206
|
-
'
|
207
|
-
'
|
208
|
-
|
209
|
-
}
|
210
|
-
end
|
211
|
-
|
212
|
-
# add details to show in analytics
|
213
|
-
def self.analytics_ts(app,direction,ws_id,ws_name)
|
214
|
-
# translate transfer to operation
|
215
|
-
operation=case direction
|
216
|
-
when 'send'; 'upload'
|
217
|
-
when 'receive'; 'download'
|
218
|
-
else raise "ERROR: unexpected value: #{direction}"
|
242
|
+
# cached user information
|
243
|
+
def user_info
|
244
|
+
if @user_info.nil?
|
245
|
+
# get our user's default information
|
246
|
+
@user_info=self.read('self')[:data] rescue nil
|
247
|
+
@user_info=USER_INFO_FIELDS_MIN.inject({}){|m,f|m[f]=nil;m} if @user_info.nil?
|
248
|
+
USER_INFO_FIELDS_MIN.each{|f|@user_info[f]='unknown' if @user_info[f].nil?}
|
219
249
|
end
|
220
|
-
|
221
|
-
return {
|
222
|
-
'tags' => {
|
223
|
-
'aspera' => {
|
224
|
-
'usage_id' => "aspera.files.workspace.#{ws_id}", # activity tracking
|
225
|
-
'files' => {
|
226
|
-
'files_transfer_action' => "#{operation}_#{app.gsub(/s$/,'')}",
|
227
|
-
'workspace_name' => ws_name, # activity tracking
|
228
|
-
'workspace_id' => ws_id,
|
229
|
-
}
|
230
|
-
}
|
231
|
-
}
|
232
|
-
}
|
250
|
+
return @user_info
|
233
251
|
end
|
234
252
|
|
235
253
|
# build ts addon for IBM Aspera Console (cookie)
|
236
|
-
def
|
237
|
-
|
238
|
-
elements.
|
239
|
-
|
240
|
-
return {
|
241
|
-
'cookie'=>elements.join(':')
|
242
|
-
}
|
254
|
+
def console_ts(app)
|
255
|
+
# we are sure that fields are not nil
|
256
|
+
elements=[app,user_info['name'],user_info['email']].map{|e|Base64.strict_encode64(e)}
|
257
|
+
elements.unshift(COOKIE_PREFIX)
|
258
|
+
return {'cookie'=>elements.join(':')}
|
243
259
|
end
|
244
260
|
|
245
261
|
# build "transfer info", 2 elements array with:
|
@@ -247,12 +263,12 @@ module Aspera
|
|
247
263
|
# - source and token regeneration method
|
248
264
|
def tr_spec(app,direction,node_file,ts_add)
|
249
265
|
# prepare the rest end point is used to generate the bearer token
|
250
|
-
|
266
|
+
token_generation_lambda=lambda {|do_refresh|self.oauth_token(scope: self.class.node_scope(node_file[:node_info]['access_key'],SCOPE_NODE_USER), refresh: do_refresh)}
|
251
267
|
# prepare transfer specification
|
252
268
|
# note xfer_id and xfer_retry are set by the transfer agent itself
|
253
269
|
transfer_spec={
|
254
270
|
'direction' => direction,
|
255
|
-
'token' =>
|
271
|
+
'token' => token_generation_lambda.call(false), # first time, use cache
|
256
272
|
'tags' => {
|
257
273
|
'aspera' => {
|
258
274
|
'app' => app,
|
@@ -269,11 +285,17 @@ module Aspera
|
|
269
285
|
}
|
270
286
|
# add remote host info
|
271
287
|
if @@use_standard_ports
|
272
|
-
|
288
|
+
# get default TCP/UDP ports and transfer user
|
289
|
+
transfer_spec.merge!(Fasp::Default::AK_TSPEC_BASE)
|
290
|
+
# by default: same address as node API
|
273
291
|
transfer_spec['remote_host']=node_file[:node_info]['host']
|
292
|
+
# 30 it's necessarily https scheme: webui does not allow anything else
|
293
|
+
if node_file[:node_info]['transfer_url'].is_a?(String) and !node_file[:node_info]['transfer_url'].empty?
|
294
|
+
transfer_spec['remote_host']=URI.parse(node_file[:node_info]['transfer_url']).host
|
295
|
+
end
|
274
296
|
else
|
275
297
|
# retrieve values from API
|
276
|
-
std_t_spec=get_node_api(node_file[:node_info],SCOPE_NODE_USER).create('files/download_setup',{:transfer_requests => [ { :transfer_request => {:paths => [ {"source"=>'/'} ] } } ] } )[:data]['transfer_specs'].first['transfer_spec']
|
298
|
+
std_t_spec=get_node_api(node_file[:node_info],scope: SCOPE_NODE_USER).create('files/download_setup',{:transfer_requests => [ { :transfer_request => {:paths => [ {"source"=>'/'} ] } } ] } )[:data]['transfer_specs'].first['transfer_spec']
|
277
299
|
['remote_host','remote_user','ssh_port','fasp_port'].each {|i| transfer_spec[i]=std_t_spec[i]}
|
278
300
|
end
|
279
301
|
# add caller provided transfer spec
|
@@ -281,7 +303,7 @@ module Aspera
|
|
281
303
|
# additional information for transfer agent
|
282
304
|
source_and_token_generator={
|
283
305
|
:src => :node_gen4,
|
284
|
-
:regenerate_token =>
|
306
|
+
:regenerate_token => token_generation_lambda
|
285
307
|
}
|
286
308
|
return transfer_spec,source_and_token_generator
|
287
309
|
end
|
@@ -290,26 +312,27 @@ module Aspera
|
|
290
312
|
# @param scope e.g. SCOPE_NODE_USER
|
291
313
|
# no scope: requires secret
|
292
314
|
# if secret provided beforehand: use it
|
293
|
-
def get_node_api(node_info,
|
294
|
-
|
295
|
-
|
296
|
-
|
297
|
-
|
298
|
-
|
299
|
-
ak_secret=@key_chain.get_secret(node_info['access_key'],false)
|
300
|
-
if ak_secret.nil? and node_scope.nil?
|
315
|
+
def get_node_api(node_info,options={})
|
316
|
+
raise "INTERNAL ERROR: method parameters: options must be Hash" unless options.is_a?(Hash)
|
317
|
+
options.keys.each {|k| raise "INTERNAL ERROR: not valid option: #{k}" unless [:scope,:use_secret].include?(k)}
|
318
|
+
# get optional secret unless :use_secret is false (default is true)
|
319
|
+
ak_secret=@key_chain.get_secret(url: node_info['url'], username: node_info['access_key'], mandatory: false) if !@key_chain.nil? and ( !options.has_key?(:use_secret) or options[:use_secret] )
|
320
|
+
if ak_secret.nil? and !options.has_key?(:scope)
|
301
321
|
raise "There must be at least one of: 'secret' or 'scope' for access key #{node_info['access_key']}"
|
302
322
|
end
|
303
|
-
|
304
|
-
if
|
323
|
+
node_rest_params={base_url: node_info['url']}
|
324
|
+
# if secret is available
|
325
|
+
if !ak_secret.nil?
|
305
326
|
node_rest_params[:auth]={
|
306
|
-
:
|
307
|
-
:
|
308
|
-
:
|
327
|
+
type: :basic,
|
328
|
+
username: node_info['access_key'],
|
329
|
+
password: ak_secret
|
309
330
|
}
|
310
331
|
else
|
332
|
+
# X-Aspera-AccessKey required for bearer token only
|
333
|
+
node_rest_params[:headers]= {'X-Aspera-AccessKey'=>node_info['access_key']}
|
311
334
|
node_rest_params[:auth]=self.params[:auth].clone
|
312
|
-
node_rest_params[:auth][:scope]=self.class.node_scope(node_info['access_key'],
|
335
|
+
node_rest_params[:auth][:scope]=self.class.node_scope(node_info['access_key'],options[:scope])
|
313
336
|
end
|
314
337
|
return Node.new(node_rest_params)
|
315
338
|
end
|
@@ -336,7 +359,7 @@ module Aspera
|
|
336
359
|
if entry[:type].eql?('link')
|
337
360
|
sub_node_info=self.read("nodes/#{entry['target_node_id']}")[:data]
|
338
361
|
sub_opt={method: process_find_files, top_file_id: entry['target_id'], top_file_path: path}
|
339
|
-
get_node_api(sub_node_info,SCOPE_NODE_USER).crawl(self,sub_opt)
|
362
|
+
get_node_api(sub_node_info,scope: SCOPE_NODE_USER).crawl(self,sub_opt)
|
340
363
|
end
|
341
364
|
rescue => e
|
342
365
|
Log.log.error("#{path}: #{e.message}")
|
@@ -349,7 +372,7 @@ module Aspera
|
|
349
372
|
top_node_info,top_file_id=check_get_node_file(top_node_file)
|
350
373
|
Log.log.debug("find_files: node_info=#{top_node_info}, fileid=#{top_file_id}")
|
351
374
|
@find_state={found: [], test_block: test_block}
|
352
|
-
get_node_api(top_node_info,SCOPE_NODE_USER).crawl(self,{method: :process_find_files, top_file_id: top_file_id})
|
375
|
+
get_node_api(top_node_info,scope: SCOPE_NODE_USER).crawl(self,{method: :process_find_files, top_file_id: top_file_id})
|
353
376
|
result=@find_state[:found]
|
354
377
|
@find_state=nil
|
355
378
|
return result
|
@@ -370,7 +393,7 @@ module Aspera
|
|
370
393
|
if @resolve_state[:path].empty?
|
371
394
|
@resolve_state[:result][:file_id]=entry['target_id']
|
372
395
|
else
|
373
|
-
get_node_api(@resolve_state[:result][:node_info],SCOPE_NODE_USER).crawl(self,{method: :process_resolve_node_file, top_file_id: entry['target_id']})
|
396
|
+
get_node_api(@resolve_state[:result][:node_info],scope: SCOPE_NODE_USER).crawl(self,{method: :process_resolve_node_file, top_file_id: entry['target_id']})
|
374
397
|
end
|
375
398
|
when 'folder'
|
376
399
|
if @resolve_state[:path].empty?
|
@@ -397,7 +420,7 @@ module Aspera
|
|
397
420
|
result[:file_id]=top_file_id
|
398
421
|
else
|
399
422
|
@resolve_state={path: path_elements, result: result}
|
400
|
-
get_node_api(top_node_info,SCOPE_NODE_USER).crawl(self,{method: :process_resolve_node_file, top_file_id: top_file_id})
|
423
|
+
get_node_api(top_node_info,scope: SCOPE_NODE_USER).crawl(self,{method: :process_resolve_node_file, top_file_id: top_file_id})
|
401
424
|
not_found=@resolve_state[:path]
|
402
425
|
@resolve_state=nil
|
403
426
|
raise "entry not found: #{not_found}" if result[:file_id].nil?
|
@@ -1,5 +1,5 @@
|
|
1
1
|
require 'aspera/fasp/listener'
|
2
|
-
require 'aspera/fasp/
|
2
|
+
require 'aspera/fasp/agent_base'
|
3
3
|
require 'ruby-progressbar'
|
4
4
|
|
5
5
|
module Aspera
|
@@ -43,13 +43,13 @@ module Aspera
|
|
43
43
|
:title => '',
|
44
44
|
:total => nil)
|
45
45
|
end
|
46
|
-
if !data.has_key?(Fasp::
|
47
|
-
Log.log.error("Internal error: no #{Fasp::
|
46
|
+
if !data.has_key?(Fasp::AgentBase::LISTENER_SESSION_ID_S)
|
47
|
+
Log.log.error("Internal error: no #{Fasp::AgentBase::LISTENER_SESSION_ID_S} in event: #{data}")
|
48
48
|
return
|
49
49
|
end
|
50
50
|
newtitle=@sessions.length < 2 ? '' : "multi=#{@sessions.length}"
|
51
51
|
@progress_bar.title=newtitle unless @progress_bar.title.eql?(newtitle)
|
52
|
-
session=@sessions[data[Fasp::
|
52
|
+
session=@sessions[data[Fasp::AgentBase::LISTENER_SESSION_ID_S]]||={
|
53
53
|
cumulative: 0,
|
54
54
|
job_size: 0,
|
55
55
|
current: 0
|
@@ -78,7 +78,7 @@ module Aspera
|
|
78
78
|
# stop event when one file is completed
|
79
79
|
session[:cumulative]=session[:cumulative]+data['size'].to_i
|
80
80
|
when 'DONE' # end of session
|
81
|
-
@sessions.delete(data[Fasp::
|
81
|
+
@sessions.delete(data[Fasp::AgentBase::LISTENER_SESSION_ID_S])
|
82
82
|
update_progress
|
83
83
|
update_total
|
84
84
|
else
|