aspera-cli 4.2.1 → 4.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +1580 -946
  3. data/bin/ascli +1 -1
  4. data/bin/asession +3 -5
  5. data/docs/Makefile +8 -11
  6. data/docs/README.erb.md +1521 -829
  7. data/docs/doc_tools.rb +58 -0
  8. data/docs/test_env.conf +3 -1
  9. data/examples/faspex4.rb +28 -19
  10. data/examples/transfer.rb +2 -2
  11. data/lib/aspera/aoc.rb +157 -134
  12. data/lib/aspera/cli/listener/progress_multi.rb +5 -5
  13. data/lib/aspera/cli/main.rb +106 -48
  14. data/lib/aspera/cli/manager.rb +19 -20
  15. data/lib/aspera/cli/plugin.rb +22 -7
  16. data/lib/aspera/cli/plugins/aoc.rb +260 -208
  17. data/lib/aspera/cli/plugins/ats.rb +11 -10
  18. data/lib/aspera/cli/plugins/bss.rb +2 -2
  19. data/lib/aspera/cli/plugins/config.rb +360 -189
  20. data/lib/aspera/cli/plugins/faspex.rb +119 -56
  21. data/lib/aspera/cli/plugins/faspex5.rb +32 -17
  22. data/lib/aspera/cli/plugins/node.rb +72 -31
  23. data/lib/aspera/cli/plugins/orchestrator.rb +5 -3
  24. data/lib/aspera/cli/plugins/preview.rb +94 -68
  25. data/lib/aspera/cli/plugins/server.rb +16 -5
  26. data/lib/aspera/cli/plugins/shares.rb +17 -0
  27. data/lib/aspera/cli/transfer_agent.rb +64 -82
  28. data/lib/aspera/cli/version.rb +1 -1
  29. data/lib/aspera/command_line_builder.rb +48 -31
  30. data/lib/aspera/cos_node.rb +4 -3
  31. data/lib/aspera/environment.rb +4 -4
  32. data/lib/aspera/fasp/{manager.rb → agent_base.rb} +7 -6
  33. data/lib/aspera/fasp/{connect.rb → agent_connect.rb} +46 -39
  34. data/lib/aspera/fasp/{local.rb → agent_direct.rb} +42 -38
  35. data/lib/aspera/fasp/{http_gw.rb → agent_httpgw.rb} +50 -29
  36. data/lib/aspera/fasp/{node.rb → agent_node.rb} +43 -4
  37. data/lib/aspera/fasp/agent_trsdk.rb +106 -0
  38. data/lib/aspera/fasp/default.rb +17 -0
  39. data/lib/aspera/fasp/installation.rb +64 -48
  40. data/lib/aspera/fasp/parameters.rb +78 -91
  41. data/lib/aspera/fasp/parameters.yaml +531 -0
  42. data/lib/aspera/fasp/uri.rb +1 -1
  43. data/lib/aspera/faspex_gw.rb +12 -11
  44. data/lib/aspera/id_generator.rb +22 -0
  45. data/lib/aspera/keychain/encrypted_hash.rb +120 -0
  46. data/lib/aspera/keychain/macos_security.rb +94 -0
  47. data/lib/aspera/log.rb +45 -32
  48. data/lib/aspera/node.rb +9 -4
  49. data/lib/aspera/oauth.rb +116 -100
  50. data/lib/aspera/persistency_action_once.rb +11 -7
  51. data/lib/aspera/persistency_folder.rb +6 -26
  52. data/lib/aspera/rest.rb +66 -50
  53. data/lib/aspera/sync.rb +40 -35
  54. data/lib/aspera/timer_limiter.rb +22 -0
  55. metadata +86 -29
  56. data/docs/transfer_spec.html +0 -99
  57. data/lib/aspera/api_detector.rb +0 -60
  58. data/lib/aspera/fasp/aoc.rb +0 -24
  59. data/lib/aspera/secrets.rb +0 -20
data/docs/doc_tools.rb ADDED
@@ -0,0 +1,58 @@
1
+ # get transfer spec parameter description
2
+ $LOAD_PATH.unshift(ENV["INCL_DIR_GEM"])
3
+ require 'aspera/fasp/parameters'
4
+
5
+ # check that required env vars exist, and files
6
+ %w{EXENAME GEMSPEC INCL_USAGE INCL_COMMANDS INCL_ASESSION INCL_DIR_GEM}.each do |e|
7
+ raise "missing env var #{e}" unless ENV.has_key?(e)
8
+ raise "missing file #{ENV[e]}" unless File.exist?(ENV[e]) or !e.start_with?('INCL_')
9
+ end
10
+
11
+ # set values used in ERB
12
+ # just command name
13
+ def cmd;ENV['EXENAME'];end
14
+
15
+ # used in text with formatting of command
16
+ def tool;'`'+cmd+'`';end
17
+
18
+ # prefix for env vars
19
+ def evp;cmd.upcase+'_';end
20
+
21
+ # just the name for "option preset"
22
+ def opprst;'option preset';end
23
+
24
+ # name with link
25
+ def prst;'['+opprst+'](#lprt)';end
26
+
27
+ # name with link (plural)
28
+ def prsts;'['+opprst+'s](#lprt)';end
29
+
30
+ # in title
31
+ def prstt;opprst.capitalize;end
32
+
33
+ def gemspec;Gem::Specification::load(ENV['GEMSPEC']) or raise "error loading #{ENV["GEMSPEC"]}";end
34
+
35
+ def geminstadd;gemspec.version.to_s.match(/\.[^0-9]/)?' --pre':'';end
36
+
37
+ # transfer spec description generation
38
+ def spec_table
39
+ r='<table><tr><th>Field</th><th>Type</th>'
40
+ Aspera::Fasp::Parameters::SUPPORTED_AGENTS_SHORT.each do |c|
41
+ r << '<th>'<<c.to_s.upcase<<'</th>'
42
+ end
43
+ r << '<th>Description</th></tr>'
44
+ Aspera::Fasp::Parameters.man_table.each do |p|
45
+ p[:description] << (p[:description].empty? ? '' : "\n") << "(" << p[:cli] << ")" unless p[:cli].to_s.empty?
46
+ p.delete(:cli)
47
+ p.keys.each{|c|p[c]='&nbsp;' if p[c].to_s.empty?}
48
+ p[:description].gsub!("\n",'<br/>')
49
+ p[:type].gsub!(',','<br/>')
50
+ r << '<tr><td>'<<p[:name]<<'</td><td>'<<p[:type]<<'</td>'
51
+ Aspera::Fasp::Parameters::SUPPORTED_AGENTS_SHORT.each do |c|
52
+ r << '<td>'<<p[c]<<'</td>'
53
+ end
54
+ r << '<td>'<<p[:description]<<'</td></tr>'
55
+ end
56
+ r << '</table>'
57
+ return r
58
+ end
data/docs/test_env.conf CHANGED
@@ -5,7 +5,7 @@ default:
5
5
  config: cli_default
6
6
  aoc: tst_aoc1
7
7
  faspex: tst_faspex
8
- faspex5: tst_faspex5_web
8
+ faspex5: tst_faspex5
9
9
  shares: tst_shares_1
10
10
  shares2: tst_shares2
11
11
  node: tst_node
@@ -65,6 +65,7 @@ tst_faspex5:
65
65
  client_secret: your value here
66
66
  private_key: your value here
67
67
  username: your value here
68
+ password: your value here
68
69
  tst_shares:
69
70
  url: your value here
70
71
  username: your value here
@@ -155,4 +156,5 @@ misc:
155
156
  email_external: your value here
156
157
  aoc_org: your value here
157
158
  aoc_user_email: your value here
159
+ aoc_workspace2: your value here
158
160
  http_gw_fqdn_port: your value here
data/examples/faspex4.rb CHANGED
@@ -1,29 +1,37 @@
1
1
  #!/usr/bin/env ruby
2
+ # find Faspex API here: https://developer.ibm.com/apis/catalog/?search=faspex
3
+ # this example makes use of class Aspera::Rest for REST calls, alternatively class RestClient of gem rest-client could be used
4
+ # this example makes use of class Aspera::Fasp::AgentDirect for transfers, alternatively the official "Transfer SDK" could be used
5
+ # Aspera SDK can be downloaded with: `ascli conf ascp install` , it installs in $HOME/.aspera/ascli/sdk
2
6
  require 'aspera/rest'
3
7
  require 'aspera/log'
4
- require 'aspera/fasp/local'
5
- require 'aspera/cli/listener/line_dump'
6
- require 'aspera/cli/extended_value'
8
+ require 'aspera/fasp/agent_direct'
7
9
 
8
- # set high log level for the example
10
+ tmpdir=ENV['tmp']||Dir.tmpdir || '.'
11
+
12
+ # Set high log level for the example, decrease to :warn usually
9
13
  Aspera::Log.instance.level=:debug
10
14
 
11
- # set folder where SDK is installed
15
+ # Set folder where SDK is installed (mandatory)
12
16
  # (if ascp is not there, the lib will try to find in usual locations)
13
- Aspera::Fasp::Installation.instance.folder='.'
17
+ # (if data files are not there, they will be created)
18
+ Aspera::Fasp::Installation.instance.folder = tmpdir
14
19
 
15
20
  if ! ARGV.length.eql?(3)
16
- Aspera::Log.log.error("wrong number of args: #{ARGV.length}")
21
+ Aspera::Log.log.error("Wrong number of args: #{ARGV.length}")
17
22
  Aspera::Log.log.error("Usage: #{$0} <faspex URL> <faspex username> <faspex password>")
18
23
  Aspera::Log.log.error("Example: #{$0} https://faspex.com/aspera/faspex john p@sSw0rd")
19
24
  Process.exit(1)
20
25
  end
21
26
 
22
- faspex_url=ARGV[0]
27
+ faspex_url=ARGV[0] # typically: https://faspex.example.com/aspera/faspex
23
28
  faspex_user=ARGV[1]
24
29
  faspex_pass=ARGV[2]
25
30
 
26
- # 1: demo API v3
31
+ # comment out this if certificate is valid, keep line to ignore certificate
32
+ Aspera::Rest.insecure=true
33
+
34
+ # 1: Faspex 4 API v3
27
35
  #---------------
28
36
 
29
37
  # create REST API object
@@ -35,34 +43,34 @@ api_v3=Aspera::Rest.new({
35
43
  :password => faspex_pass
36
44
  }})
37
45
 
46
+ # very simple api call
38
47
  api_v3.read('me')
39
48
 
40
49
  # 2: send a package
41
50
  #---------------
42
51
 
43
52
  # create a sample file to send
44
- file_to_send='./myfile.bin'
53
+ file_to_send=File.join(tmpdir,'myfile.bin')
45
54
  File.open(file_to_send, "w") {|f| f.write("sample data") }
46
55
  # package creation parameters
47
56
  package_create_params={'delivery'=>{'title'=>'test package','recipients'=>['aspera.user1@gmail.com'],'sources'=>[{'paths'=>[file_to_send]}]}}
48
57
  pkg_created=api_v3.create('send',package_create_params)[:data]
49
- # get transfer specification
58
+ # get transfer specification (normally: only one)
50
59
  transfer_spec=pkg_created['xfer_sessions'].first
51
60
  # set paths of files to send
52
- transfer_spec['paths']=['source'=>file_to_send]
53
- # get the local agent (i.e. ascp)
54
- client=Aspera::Fasp::Local.new
55
- # disable ascp output on stdout to not mix with JSON events
56
- client.quiet=true
61
+ transfer_spec['paths']=[{'source'=>file_to_send}]
62
+ # get local agent (ascp), disable ascp output on stdout to not mix with JSON events
63
+ transfer_client=Aspera::Fasp::AgentDirect.new({quiet: true})
57
64
  # start transfer (asynchronous)
58
- job_id=client.start_transfer(transfer_spec)
59
- result=client.wait_for_transfers_completion
65
+ job_id=transfer_client.start_transfer(transfer_spec)
66
+ # wait for all transfer completion (for the example)
67
+ result=transfer_client.wait_for_transfers_completion
60
68
  # notify of any transfer error
61
69
  result.select{|i|!i.eql?(:success)}.each do |e|
62
70
  Aspera::Log.log.error("A transfer error occured: #{e.message}")
63
71
  end
64
72
 
65
- # 1: demo API v4
73
+ # 3: Faspex 4 API v4
66
74
  #---------------
67
75
  api_v4=Aspera::Rest.new({
68
76
  :base_url => faspex_url+'/api',
@@ -75,4 +83,5 @@ api_v4=Aspera::Rest.new({
75
83
  :scope => 'admin'
76
84
  }})
77
85
 
86
+ # Use it. Note that Faspex 4 API v4 is totally different from Faspex 4 v3 APIs, see ref on line 2
78
87
  Aspera::Log.dump('users',api_v4.read('users')[:data])
data/examples/transfer.rb CHANGED
@@ -2,7 +2,7 @@
2
2
  # Example: transfer a file using one of the provided transfer agents
3
3
  # location of ascp can be specified with env var "ascp"
4
4
  # temp folder can be specified with env var "tmp"
5
- require 'aspera/fasp/local'
5
+ require 'aspera/fasp/agent_direct'
6
6
  require 'aspera/fasp/listener'
7
7
  require 'aspera/fasp/installation'
8
8
  require 'aspera/log'
@@ -39,7 +39,7 @@ Aspera::Fasp::Installation.instance.ascp_path=ENV['ascp'] if ENV.has_key?('ascp'
39
39
  #
40
40
 
41
41
  # get FASP Manager singleton based on above ascp location
42
- fasp_manager=Aspera::Fasp::Local.new
42
+ fasp_manager=Aspera::Fasp::AgentDirect.new
43
43
 
44
44
  # Note that it would also be possible to start transfers using other agents
45
45
  #require 'aspera/fasp/connect'
data/lib/aspera/aoc.rb CHANGED
@@ -2,6 +2,7 @@ require 'aspera/log'
2
2
  require 'aspera/rest'
3
3
  require 'aspera/hash_ext'
4
4
  require 'aspera/data_repository'
5
+ require 'aspera/fasp/default'
5
6
  require 'base64'
6
7
 
7
8
  module Aspera
@@ -17,19 +18,20 @@ module Aspera
17
18
  # to avoid infinite loop in pub link redirection
18
19
  MAX_REDIRECT=10
19
20
  CLIENT_APPS=['aspera.global-cli-client','aspera.drive']
21
+ # index offset in data repository of client app
20
22
  DATA_REPO_INDEX_START = 4
23
+ # cookie prefix so that console can decode identity
24
+ COOKIE_PREFIX='aspera.aoc'
21
25
 
22
26
  # path in URL of public links
23
- PATHS_PUBLIC_LINK=['/packages/public/receive','/packages/public/send','/files/public']
27
+ PUBLIC_LINK_PATHS=['/packages/public/receive','/packages/public/send','/files/public']
24
28
  JWT_AUDIENCE='https://api.asperafiles.com/api/v1/oauth2/token'
25
29
  OAUTH_API_SUBPATH='api/v1/oauth2'
26
- DEFAULT_TSPEC_INFO={
27
- 'remote_user' => 'xfer',
28
- 'ssh_port' => 33001,
29
- 'fasp_port' => 33001
30
- }
30
+ # minimum fields for user info if retrieval fails
31
+ USER_INFO_FIELDS_MIN=['name','email','id','default_workspace_id','organization_id']
31
32
 
32
- private_constant :PRODUCT_NAME,:PROD_DOMAIN,:MAX_REDIRECT,:CLIENT_APPS,:PATHS_PUBLIC_LINK,:JWT_AUDIENCE,:OAUTH_API_SUBPATH,:DEFAULT_TSPEC_INFO
33
+ private_constant :PRODUCT_NAME,:PROD_DOMAIN,:MAX_REDIRECT,:CLIENT_APPS,:PUBLIC_LINK_PATHS,:JWT_AUDIENCE,
34
+ :OAUTH_API_SUBPATH,:COOKIE_PREFIX,:USER_INFO_FIELDS_MIN
33
35
 
34
36
  public
35
37
  # various API scopes supported
@@ -44,84 +46,119 @@ module Aspera
44
46
  FILES_APP='files'
45
47
  PACKAGES_APP='packages'
46
48
 
47
- def self.get_client_info(client_name=CLIENT_APPS.first)
48
- client_index=CLIENT_APPS.index(client_name)
49
- raise "no such pre-defined client: #{client_name}" if client_index.nil?
49
+ # class static methods
50
+ class << self
50
51
  # strings /Applications/Aspera\ Drive.app/Contents/MacOS/AsperaDrive|grep -E '.{100}==$'|base64 --decode
51
- return client_name,Base64.urlsafe_encode64(DataRepository.instance.get_bin(DATA_REPO_INDEX_START+client_index))
52
- end
52
+ def get_client_info(client_name=CLIENT_APPS.first)
53
+ client_index=CLIENT_APPS.index(client_name)
54
+ raise "no such pre-defined client: #{client_name}" if client_index.nil?
55
+ return client_name,Base64.urlsafe_encode64(DataRepository.instance.get_bin(DATA_REPO_INDEX_START+client_index))
56
+ end
53
57
 
54
- # @param url of AoC instance
55
- # @return organization id in url and AoC domain: ibmaspera.com, asperafiles.com or qa.asperafiles.com, etc...
56
- def self.parse_url(aoc_org_url)
57
- uri=URI.parse(aoc_org_url.gsub(/\/+$/,''))
58
- instance_fqdn=uri.host
59
- Log.log.debug("instance_fqdn=#{instance_fqdn}")
60
- raise "No host found in URL.Please check URL format: https://myorg.#{PROD_DOMAIN}" if instance_fqdn.nil?
61
- organization,instance_domain=instance_fqdn.split('.',2)
62
- Log.log.debug("instance_domain=#{instance_domain}")
63
- Log.log.debug("organization=#{organization}")
64
- raise "expecting a public FQDN for #{PRODUCT_NAME}" if instance_domain.nil?
65
- return organization,instance_domain
66
- end
58
+ # @param url of AoC instance
59
+ # @return organization id in url and AoC domain: ibmaspera.com, asperafiles.com or qa.asperafiles.com, etc...
60
+ def parse_url(aoc_org_url)
61
+ uri=URI.parse(aoc_org_url.gsub(/\/+$/,''))
62
+ instance_fqdn=uri.host
63
+ Log.log.debug("instance_fqdn=#{instance_fqdn}")
64
+ raise "No host found in URL.Please check URL format: https://myorg.#{PROD_DOMAIN}" if instance_fqdn.nil?
65
+ organization,instance_domain=instance_fqdn.split('.',2)
66
+ Log.log.debug("instance_domain=#{instance_domain}")
67
+ Log.log.debug("organization=#{organization}")
68
+ raise "expecting a public FQDN for #{PRODUCT_NAME}" if instance_domain.nil?
69
+ return organization,instance_domain
70
+ end
67
71
 
68
- # base API url depends on domain, which could be "qa.xxx"
69
- def self.api_base_url(api_domain=PROD_DOMAIN)
70
- return "https://api.#{api_domain}"
71
- end
72
+ # base API url depends on domain, which could be "qa.xxx"
73
+ def api_base_url(api_domain=PROD_DOMAIN)
74
+ return "https://api.#{api_domain}"
75
+ end
72
76
 
73
- def self.metering_api(entitlement_id,customer_id,api_domain=PROD_DOMAIN)
74
- return Rest.new({
75
- :base_url => "#{api_base_url(api_domain)}/metering/v1",
76
- :headers => {'X-Aspera-Entitlement-Authorization' => Rest.basic_creds(entitlement_id,customer_id)}
77
- })
78
- end
77
+ def metering_api(entitlement_id,customer_id,api_domain=PROD_DOMAIN)
78
+ return Rest.new({
79
+ :base_url => "#{api_base_url(api_domain)}/metering/v1",
80
+ :headers => {'X-Aspera-Entitlement-Authorization' => Rest.basic_creds(entitlement_id,customer_id)}
81
+ })
82
+ end
79
83
 
80
- # node API scopes
81
- def self.node_scope(access_key,scope)
82
- return 'node.'+access_key+':'+scope
83
- end
84
+ # node API scopes
85
+ def node_scope(access_key,scope)
86
+ return 'node.'+access_key+':'+scope
87
+ end
84
88
 
85
- def self.set_use_default_ports(val)
86
- @@use_standard_ports=val
87
- end
89
+ def set_use_default_ports(val)
90
+ @@use_standard_ports=val
91
+ end
88
92
 
89
- # check option "link"
90
- # if present try to get token value (resolve redirection if short links used)
91
- # then set options url/token/auth
92
- def self.resolve_pub_link(rest_opts,public_link_url)
93
- return if public_link_url.nil?
94
- # set to token if available after redirection
95
- url_param_token_pair=nil
96
- redirect_count=0
97
- loop do
98
- uri=URI.parse(public_link_url)
99
- if PATHS_PUBLIC_LINK.include?(uri.path)
100
- url_param_token_pair=URI::decode_www_form(uri.query).select{|e|e.first.eql?('token')}.first
101
- if url_param_token_pair.nil?
102
- raise ArgumentError,"link option must be URL with 'token' parameter"
93
+ # check option "link"
94
+ # if present try to get token value (resolve redirection if short links used)
95
+ # then set options url/token/auth
96
+ def resolve_pub_link(rest_opts,public_link_url)
97
+ return if public_link_url.nil?
98
+ # set to token if available after redirection
99
+ url_param_token_pair=nil
100
+ redirect_count=0
101
+ loop do
102
+ uri=URI.parse(public_link_url)
103
+ if PUBLIC_LINK_PATHS.include?(uri.path)
104
+ url_param_token_pair=URI::decode_www_form(uri.query).select{|e|e.first.eql?('token')}.first
105
+ if url_param_token_pair.nil?
106
+ raise ArgumentError,"link option must be URL with 'token' parameter"
107
+ end
108
+ # ok we get it !
109
+ rest_opts[:org_url]='https://'+uri.host
110
+ rest_opts[:auth][:grant]=:url_token
111
+ rest_opts[:auth][:url_token]=url_param_token_pair.last
112
+ return
103
113
  end
104
- # ok we get it !
105
- rest_opts[:org_url]='https://'+uri.host
106
- rest_opts[:auth][:grant]=:url_token
107
- rest_opts[:auth][:url_token]=url_param_token_pair.last
108
- return
109
- end
110
- Log.log.debug("no expected format: #{public_link_url}")
111
- raise "exceeded max redirection: #{MAX_REDIRECT}" if redirect_count > MAX_REDIRECT
112
- r = Net::HTTP.get_response(uri)
113
- if r.code.start_with?("3")
114
- public_link_url = r['location']
115
- raise "no location in redirection" if public_link_url.nil?
116
- Log.log.debug("redirect to: #{public_link_url}")
117
- else
118
- # not a redirection
119
- raise ArgumentError,'link option must be redirect or have token parameter'
114
+ Log.log.debug("no expected format: #{public_link_url}")
115
+ raise "exceeded max redirection: #{MAX_REDIRECT}" if redirect_count > MAX_REDIRECT
116
+ r = Net::HTTP.get_response(uri)
117
+ if r.code.start_with?("3")
118
+ public_link_url = r['location']
119
+ raise "no location in redirection" if public_link_url.nil?
120
+ Log.log.debug("redirect to: #{public_link_url}")
121
+ else
122
+ # not a redirection
123
+ raise ArgumentError,'link option must be redirect or have token parameter'
124
+ end
125
+ end # loop
126
+
127
+ raise RuntimeError,'too many redirections'
128
+ end
129
+
130
+ # additional transfer spec (tags) for package information
131
+ def package_tags(package_info,operation)
132
+ return {'tags'=>{'aspera'=>{'files'=>{
133
+ 'package_id' => package_info['id'],
134
+ 'package_name' => package_info['name'],
135
+ 'package_operation' => operation
136
+ }}}}
137
+ end
138
+
139
+ # add details to show in analytics
140
+ def analytics_ts(app,direction,ws_id,ws_name)
141
+ # translate transfer to operation
142
+ operation=case direction
143
+ when 'send'; 'upload'
144
+ when 'receive'; 'download'
145
+ else raise "ERROR: unexpected value: #{direction}"
120
146
  end
121
- end # loop
122
147
 
123
- raise RuntimeError,'too many redirections'
124
- end
148
+ return {
149
+ 'tags' => {
150
+ 'aspera' => {
151
+ 'usage_id' => "aspera.files.workspace.#{ws_id}", # activity tracking
152
+ 'files' => {
153
+ 'files_transfer_action' => "#{operation}_#{app.gsub(/s$/,'')}",
154
+ 'workspace_name' => ws_name, # activity tracking
155
+ 'workspace_id' => ws_id,
156
+ }
157
+ }
158
+ }
159
+ }
160
+ end
161
+ end # static methods
125
162
 
126
163
  # @param :link,:url,:auth,:client_id,:client_secret,:scope,:redirect_uri,:private_key,:username,:subpath,:password (for pub link)
127
164
  def initialize(opt)
@@ -129,6 +166,7 @@ module Aspera
129
166
  # key: access key
130
167
  # value: associated secret
131
168
  @key_chain=nil
169
+ @user_info=nil
132
170
 
133
171
  # init rest params
134
172
  aoc_rest_p={:auth=>{:type =>:oauth2}}
@@ -196,50 +234,28 @@ module Aspera
196
234
 
197
235
  def key_chain=(keychain)
198
236
  raise "keychain already set" unless @key_chain.nil?
237
+ raise "keychain must have get_secret" unless keychain.respond_to?(:get_secret)
199
238
  @key_chain=keychain
200
239
  nil
201
240
  end
202
241
 
203
- # additional transfer spec (tags) for package information
204
- def self.package_tags(package_info,operation)
205
- return {'tags'=>{'aspera'=>{'files'=>{
206
- 'package_id' => package_info['id'],
207
- 'package_name' => package_info['name'],
208
- 'package_operation' => operation
209
- }}}}
210
- end
211
-
212
- # add details to show in analytics
213
- def self.analytics_ts(app,direction,ws_id,ws_name)
214
- # translate transfer to operation
215
- operation=case direction
216
- when 'send'; 'upload'
217
- when 'receive'; 'download'
218
- else raise "ERROR: unexpected value: #{direction}"
242
+ # cached user information
243
+ def user_info
244
+ if @user_info.nil?
245
+ # get our user's default information
246
+ @user_info=self.read('self')[:data] rescue nil
247
+ @user_info=USER_INFO_FIELDS_MIN.inject({}){|m,f|m[f]=nil;m} if @user_info.nil?
248
+ USER_INFO_FIELDS_MIN.each{|f|@user_info[f]='unknown' if @user_info[f].nil?}
219
249
  end
220
-
221
- return {
222
- 'tags' => {
223
- 'aspera' => {
224
- 'usage_id' => "aspera.files.workspace.#{ws_id}", # activity tracking
225
- 'files' => {
226
- 'files_transfer_action' => "#{operation}_#{app.gsub(/s$/,'')}",
227
- 'workspace_name' => ws_name, # activity tracking
228
- 'workspace_id' => ws_id,
229
- }
230
- }
231
- }
232
- }
250
+ return @user_info
233
251
  end
234
252
 
235
253
  # build ts addon for IBM Aspera Console (cookie)
236
- def self.console_ts(app,user_name,user_email)
237
- elements=[app,user_name,user_email].map{|e|Base64.strict_encode64(e)}
238
- elements.unshift('aspera.aoc')
239
- #Log.dump('elem1'.bg_red,elements[1])
240
- return {
241
- 'cookie'=>elements.join(':')
242
- }
254
+ def console_ts(app)
255
+ # we are sure that fields are not nil
256
+ elements=[app,user_info['name'],user_info['email']].map{|e|Base64.strict_encode64(e)}
257
+ elements.unshift(COOKIE_PREFIX)
258
+ return {'cookie'=>elements.join(':')}
243
259
  end
244
260
 
245
261
  # build "transfer info", 2 elements array with:
@@ -247,12 +263,12 @@ module Aspera
247
263
  # - source and token regeneration method
248
264
  def tr_spec(app,direction,node_file,ts_add)
249
265
  # prepare the rest end point is used to generate the bearer token
250
- token_generation_method=lambda {|do_refresh|self.oauth_token(scope: self.class.node_scope(node_file[:node_info]['access_key'],SCOPE_NODE_USER), refresh: do_refresh)}
266
+ token_generation_lambda=lambda {|do_refresh|self.oauth_token(scope: self.class.node_scope(node_file[:node_info]['access_key'],SCOPE_NODE_USER), refresh: do_refresh)}
251
267
  # prepare transfer specification
252
268
  # note xfer_id and xfer_retry are set by the transfer agent itself
253
269
  transfer_spec={
254
270
  'direction' => direction,
255
- 'token' => token_generation_method.call(false), # first time, use cache
271
+ 'token' => token_generation_lambda.call(false), # first time, use cache
256
272
  'tags' => {
257
273
  'aspera' => {
258
274
  'app' => app,
@@ -269,11 +285,17 @@ module Aspera
269
285
  }
270
286
  # add remote host info
271
287
  if @@use_standard_ports
272
- transfer_spec.merge!(DEFAULT_TSPEC_INFO)
288
+ # get default TCP/UDP ports and transfer user
289
+ transfer_spec.merge!(Fasp::Default::AK_TSPEC_BASE)
290
+ # by default: same address as node API
273
291
  transfer_spec['remote_host']=node_file[:node_info]['host']
292
+ # 30 it's necessarily https scheme: webui does not allow anything else
293
+ if node_file[:node_info]['transfer_url'].is_a?(String) and !node_file[:node_info]['transfer_url'].empty?
294
+ transfer_spec['remote_host']=URI.parse(node_file[:node_info]['transfer_url']).host
295
+ end
274
296
  else
275
297
  # retrieve values from API
276
- std_t_spec=get_node_api(node_file[:node_info],SCOPE_NODE_USER).create('files/download_setup',{:transfer_requests => [ { :transfer_request => {:paths => [ {"source"=>'/'} ] } } ] } )[:data]['transfer_specs'].first['transfer_spec']
298
+ std_t_spec=get_node_api(node_file[:node_info],scope: SCOPE_NODE_USER).create('files/download_setup',{:transfer_requests => [ { :transfer_request => {:paths => [ {"source"=>'/'} ] } } ] } )[:data]['transfer_specs'].first['transfer_spec']
277
299
  ['remote_host','remote_user','ssh_port','fasp_port'].each {|i| transfer_spec[i]=std_t_spec[i]}
278
300
  end
279
301
  # add caller provided transfer spec
@@ -281,7 +303,7 @@ module Aspera
281
303
  # additional information for transfer agent
282
304
  source_and_token_generator={
283
305
  :src => :node_gen4,
284
- :regenerate_token => token_generation_method
306
+ :regenerate_token => token_generation_lambda
285
307
  }
286
308
  return transfer_spec,source_and_token_generator
287
309
  end
@@ -290,26 +312,27 @@ module Aspera
290
312
  # @param scope e.g. SCOPE_NODE_USER
291
313
  # no scope: requires secret
292
314
  # if secret provided beforehand: use it
293
- def get_node_api(node_info,node_scope=nil)
294
- # X-Aspera-AccessKey required for bearer token only
295
- node_rest_params={
296
- :base_url => node_info['url'],
297
- :headers => {'X-Aspera-AccessKey'=>node_info['access_key']},
298
- }
299
- ak_secret=@key_chain.get_secret(node_info['access_key'],false)
300
- if ak_secret.nil? and node_scope.nil?
315
+ def get_node_api(node_info,options={})
316
+ raise "INTERNAL ERROR: method parameters: options must be Hash" unless options.is_a?(Hash)
317
+ options.keys.each {|k| raise "INTERNAL ERROR: not valid option: #{k}" unless [:scope,:use_secret].include?(k)}
318
+ # get optional secret unless :use_secret is false (default is true)
319
+ ak_secret=@key_chain.get_secret(url: node_info['url'], username: node_info['access_key'], mandatory: false) if !@key_chain.nil? and ( !options.has_key?(:use_secret) or options[:use_secret] )
320
+ if ak_secret.nil? and !options.has_key?(:scope)
301
321
  raise "There must be at least one of: 'secret' or 'scope' for access key #{node_info['access_key']}"
302
322
  end
303
- # if secret provided on command line or if there is no scope
304
- if !ak_secret.nil? or node_scope.nil?
323
+ node_rest_params={base_url: node_info['url']}
324
+ # if secret is available
325
+ if !ak_secret.nil?
305
326
  node_rest_params[:auth]={
306
- :type => :basic,
307
- :username => node_info['access_key'],
308
- :password => ak_secret
327
+ type: :basic,
328
+ username: node_info['access_key'],
329
+ password: ak_secret
309
330
  }
310
331
  else
332
+ # X-Aspera-AccessKey required for bearer token only
333
+ node_rest_params[:headers]= {'X-Aspera-AccessKey'=>node_info['access_key']}
311
334
  node_rest_params[:auth]=self.params[:auth].clone
312
- node_rest_params[:auth][:scope]=self.class.node_scope(node_info['access_key'],node_scope)
335
+ node_rest_params[:auth][:scope]=self.class.node_scope(node_info['access_key'],options[:scope])
313
336
  end
314
337
  return Node.new(node_rest_params)
315
338
  end
@@ -336,7 +359,7 @@ module Aspera
336
359
  if entry[:type].eql?('link')
337
360
  sub_node_info=self.read("nodes/#{entry['target_node_id']}")[:data]
338
361
  sub_opt={method: process_find_files, top_file_id: entry['target_id'], top_file_path: path}
339
- get_node_api(sub_node_info,SCOPE_NODE_USER).crawl(self,sub_opt)
362
+ get_node_api(sub_node_info,scope: SCOPE_NODE_USER).crawl(self,sub_opt)
340
363
  end
341
364
  rescue => e
342
365
  Log.log.error("#{path}: #{e.message}")
@@ -349,7 +372,7 @@ module Aspera
349
372
  top_node_info,top_file_id=check_get_node_file(top_node_file)
350
373
  Log.log.debug("find_files: node_info=#{top_node_info}, fileid=#{top_file_id}")
351
374
  @find_state={found: [], test_block: test_block}
352
- get_node_api(top_node_info,SCOPE_NODE_USER).crawl(self,{method: :process_find_files, top_file_id: top_file_id})
375
+ get_node_api(top_node_info,scope: SCOPE_NODE_USER).crawl(self,{method: :process_find_files, top_file_id: top_file_id})
353
376
  result=@find_state[:found]
354
377
  @find_state=nil
355
378
  return result
@@ -370,7 +393,7 @@ module Aspera
370
393
  if @resolve_state[:path].empty?
371
394
  @resolve_state[:result][:file_id]=entry['target_id']
372
395
  else
373
- get_node_api(@resolve_state[:result][:node_info],SCOPE_NODE_USER).crawl(self,{method: :process_resolve_node_file, top_file_id: entry['target_id']})
396
+ get_node_api(@resolve_state[:result][:node_info],scope: SCOPE_NODE_USER).crawl(self,{method: :process_resolve_node_file, top_file_id: entry['target_id']})
374
397
  end
375
398
  when 'folder'
376
399
  if @resolve_state[:path].empty?
@@ -397,7 +420,7 @@ module Aspera
397
420
  result[:file_id]=top_file_id
398
421
  else
399
422
  @resolve_state={path: path_elements, result: result}
400
- get_node_api(top_node_info,SCOPE_NODE_USER).crawl(self,{method: :process_resolve_node_file, top_file_id: top_file_id})
423
+ get_node_api(top_node_info,scope: SCOPE_NODE_USER).crawl(self,{method: :process_resolve_node_file, top_file_id: top_file_id})
401
424
  not_found=@resolve_state[:path]
402
425
  @resolve_state=nil
403
426
  raise "entry not found: #{not_found}" if result[:file_id].nil?
@@ -1,5 +1,5 @@
1
1
  require 'aspera/fasp/listener'
2
- require 'aspera/fasp/manager'
2
+ require 'aspera/fasp/agent_base'
3
3
  require 'ruby-progressbar'
4
4
 
5
5
  module Aspera
@@ -43,13 +43,13 @@ module Aspera
43
43
  :title => '',
44
44
  :total => nil)
45
45
  end
46
- if !data.has_key?(Fasp::Manager::LISTENER_SESSION_ID_S)
47
- Log.log.error("Internal error: no #{Fasp::Manager::LISTENER_SESSION_ID_S} in event: #{data}")
46
+ if !data.has_key?(Fasp::AgentBase::LISTENER_SESSION_ID_S)
47
+ Log.log.error("Internal error: no #{Fasp::AgentBase::LISTENER_SESSION_ID_S} in event: #{data}")
48
48
  return
49
49
  end
50
50
  newtitle=@sessions.length < 2 ? '' : "multi=#{@sessions.length}"
51
51
  @progress_bar.title=newtitle unless @progress_bar.title.eql?(newtitle)
52
- session=@sessions[data[Fasp::Manager::LISTENER_SESSION_ID_S]]||={
52
+ session=@sessions[data[Fasp::AgentBase::LISTENER_SESSION_ID_S]]||={
53
53
  cumulative: 0,
54
54
  job_size: 0,
55
55
  current: 0
@@ -78,7 +78,7 @@ module Aspera
78
78
  # stop event when one file is completed
79
79
  session[:cumulative]=session[:cumulative]+data['size'].to_i
80
80
  when 'DONE' # end of session
81
- @sessions.delete(data[Fasp::Manager::LISTENER_SESSION_ID_S])
81
+ @sessions.delete(data[Fasp::AgentBase::LISTENER_SESSION_ID_S])
82
82
  update_progress
83
83
  update_total
84
84
  else