asperalm 0.10.11 → 0.10.12

Sign up to get free protection for your applications and to get access to all the features.
@@ -2,25 +2,20 @@ require 'asperalm/fasp/manager'
2
2
  require 'asperalm/rest'
3
3
  require 'asperalm/open_application'
4
4
  require 'securerandom'
5
- require 'singleton'
6
5
  require 'tty-spinner'
7
6
 
8
7
  module Asperalm
9
8
  module Fasp
10
9
  class Connect < Manager
11
- include Singleton
12
- private
10
+ MAX_CONNECT_START_RETRY=3
11
+ SLEEP_SEC_BETWEEN_RETRY=2
12
+ private_constant :MAX_CONNECT_START_RETRY,:SLEEP_SEC_BETWEEN_RETRY
13
13
  # mode=node : activate, set to the REST api object for the node API
14
14
  def initialize
15
15
  super
16
16
  @connect_app_id=SecureRandom.uuid
17
17
  # TODO: start here and create monitor
18
18
  end
19
- MAX_CONNECT_START_RETRY=3
20
- SLEEP_SEC_BETWEEN_RETRY=2
21
- private_constant :MAX_CONNECT_START_RETRY,:SLEEP_SEC_BETWEEN_RETRY
22
-
23
- public
24
19
 
25
20
  def start_transfer(transfer_spec,options=nil)
26
21
  raise "Using connect requires a graphical environment" if !OpenApplication.default_gui_mode.eql?(:graphical)
@@ -1,21 +1,12 @@
1
1
  #!/bin/echo this is a ruby class:
2
2
  require 'asperalm/fasp/manager'
3
- require 'asperalm/fasp/resume_policy'
4
3
  require 'asperalm/log'
5
4
  require 'asperalm/rest'
6
- require 'socket'
7
- require 'timeout'
8
- require 'singleton'
9
- require 'securerandom'
10
5
 
11
6
  module Asperalm
12
7
  module Fasp
13
8
  # executes a local "ascp", connects mgt port, equivalent of "Fasp Manager"
14
9
  class HttpGW < Manager
15
- include Singleton
16
- # set to false to keep ascp progress bar display (basically: removes ascp's option -q)
17
- attr_accessor :gw_api
18
- attr_accessor :resume_policy_parameters
19
10
  # start FASP transfer based on transfer spec (hash table)
20
11
  # note that it is asynchronous
21
12
  # HTTP download only supports file list
@@ -52,16 +43,18 @@ module Asperalm
52
43
  end
53
44
 
54
45
  def url=(api_url)
55
- @gw_api=Rest.new({:base_url => api_url})
56
- api_info = @gw_api.read('info')[:data]
57
- Log.log.error("#{api_info}")
58
46
  end
59
47
 
60
48
  private
61
49
 
62
- def initialize
50
+ def initialize(params)
51
+ raise "params must be Hash" unless params.is_a?(Hash)
52
+ params=params.symbolize_keys
53
+ raise "must have only one param: url" unless params.keys.eql?([:url])
63
54
  super
64
- @gw_url=nil
55
+ @gw_api=Rest.new({:base_url => params[:url]})
56
+ api_info = @gw_api.read('info')[:data]
57
+ Log.log.info("#{api_info}")
65
58
  end
66
59
 
67
60
  end # LocalHttp
@@ -13,7 +13,6 @@ require 'asperalm/fasp/resume_policy'
13
13
  require 'asperalm/log'
14
14
  require 'socket'
15
15
  require 'timeout'
16
- require 'singleton'
17
16
  require 'securerandom'
18
17
 
19
18
  module Asperalm
@@ -22,10 +21,8 @@ module Asperalm
22
21
  ACCESS_KEY_TRANSFER_USER='xfer'
23
22
  # executes a local "ascp", connects mgt port, equivalent of "Fasp Manager"
24
23
  class Local < Manager
25
- include Singleton
26
24
  # set to false to keep ascp progress bar display (basically: removes ascp's option -q)
27
25
  attr_accessor :quiet
28
- attr_accessor :resume_policy_parameters
29
26
  # start FASP transfer based on transfer spec (hash table)
30
27
  # note that it is asynchronous
31
28
  def start_transfer(transfer_spec,options={})
@@ -72,7 +69,7 @@ module Asperalm
72
69
  session={
73
70
  :state => :initial, # :initial, :started, :success, :failed
74
71
  :env_args => env_args,
75
- :resumer => options['resume_policy'] || ResumePolicy.new(@resume_policy_parameters),
72
+ :resumer => options['resume_policy'] || @resume_policy,
76
73
  :options => options
77
74
  }
78
75
 
@@ -309,8 +306,8 @@ module Asperalm
309
306
 
310
307
  private
311
308
 
312
- def initialize
313
- super
309
+ def initialize(resume_policy_parameters=nil)
310
+ super()
314
311
  # by default no interactive progress bar
315
312
  @quiet=true
316
313
  # shared data between transfer threads and others: protected by mutex, CV on change
@@ -322,7 +319,7 @@ module Asperalm
322
319
  # must be set before starting monitor, set to false to stop thread. also shared and protected by mutex
323
320
  @monitor_stop=false
324
321
  @monitor_thread=Thread.new{monitor_thread_entry}
325
- @resume_policy_parameters=ResumePolicy::DEFAULTS
322
+ @resume_policy=ResumePolicy.new(resume_policy_parameters)
326
323
  end
327
324
 
328
325
  # transfer thread entry
@@ -1,6 +1,5 @@
1
1
  require 'asperalm/fasp/manager'
2
2
  require 'asperalm/log'
3
- require 'singleton'
4
3
  require 'tty-spinner'
5
4
 
6
5
  module Asperalm
@@ -8,11 +7,9 @@ module Asperalm
8
7
  # this singleton class is used by the CLI to provide a common interface to start a transfer
9
8
  # before using it, the use must set the `node_api` member.
10
9
  class Node < Manager
11
- include Singleton
12
- private
13
- def initialize
14
- super
15
- @node_api=nil
10
+ def initialize(node_api)
11
+ super()
12
+ @node_api=node_api
16
13
  # TODO: currently only supports one transfer. This is bad shortcut. but ok for CLI.
17
14
  @transfer_id=nil
18
15
  end
@@ -22,8 +19,6 @@ module Asperalm
22
19
  raise StandardError,'Before using this object, set the node_api attribute to a Asperalm::Rest object' if @node_api.nil?
23
20
  return @node_api
24
21
  end
25
-
26
- public
27
22
  # use this to read the node_api end point.
28
23
  attr_reader :node_api
29
24
 
@@ -57,12 +52,12 @@ module Asperalm
57
52
  # lets emulate management events to display progress bar
58
53
  loop do
59
54
  # status is empty sometimes with status 200...
60
- trdata=node_api_.read("ops/transfers/#{@transfer_id}")[:data] || {"status"=>"waiting"} rescue {"status"=>"waiting"}
55
+ trdata=node_api_.read("ops/transfers/#{@transfer_id}")[:data] || {"status"=>"unknown"} rescue {"status"=>"waiting(read error)"}
61
56
  case trdata['status']
62
57
  when 'completed'
63
58
  notify_listeners('emulated',{'Type'=>'DONE'})
64
59
  break
65
- when 'waiting','partially_completed'
60
+ when 'waiting','partially_completed','unknown','waiting(read error)'
66
61
  if spinner.nil?
67
62
  spinner = TTY::Spinner.new("[:spinner] :title", format: :classic)
68
63
  spinner.start
@@ -16,17 +16,16 @@ module Asperalm
16
16
 
17
17
  def initialize(params=nil)
18
18
  @parameters=DEFAULTS.clone
19
+ return if params.nil?
20
+ raise "expecting Hash (or nil), but have #{params.class}" unless params.is_a?(Hash)
19
21
  params.each do |k,v|
20
22
  if DEFAULTS.has_key?(k)
21
23
  @parameters[k]=v
22
24
  else
23
- raise "unknown parameter: #{k}"
25
+ raise "unknown resume parameter: #{k}, expect one of #{DEFAULTS.keys.map{|i|i.to_s}.join(",")}"
24
26
  end
25
- end unless params.nil?
27
+ end
26
28
  end
27
-
28
- # use this to modify the values of the resumer to change behaviour
29
- attr_reader :parameters
30
29
 
31
30
  # calls block a number of times (resumes) until success or limit reached
32
31
  # this is re-entrant, one resumer can handle multiple transfers in //
@@ -18,3 +18,11 @@ class ::Hash
18
18
  self.merge!(second){|key,v1,v2|Hash===v1&&Hash===v2 ? v1.deep_merge!(v2) : v2}
19
19
  end
20
20
  end
21
+
22
+ unless Hash.method_defined?(:symbolize_keys)
23
+ class Hash
24
+ def symbolize_keys
25
+ return self.inject({}){|memo,(k,v)| memo[k.to_sym] = v; memo}
26
+ end
27
+ end
28
+ end
@@ -5,6 +5,22 @@ require 'base64'
5
5
 
6
6
  module Asperalm
7
7
  class OnCloud < Rest
8
+ private
9
+ PRODUCT_NAME='Aspera on Cloud'
10
+ # Production domain of AoC
11
+ PROD_DOMAIN='ibmaspera.com'
12
+ # to avoid infinite loop in pub link redirection
13
+ MAX_REDIRECT=10
14
+ DEFAULT_CLIENT='aspera.global-cli-client'
15
+ # Random generator seed
16
+ # strings /Applications/Aspera\ Drive.app/Contents/MacOS/AsperaDrive|grep -E '.{100}==$'|base64 --decode
17
+ CLIENT_RANDOM={
18
+ 'aspera.drive' => '1FwelGbL9xsv3M8H-VXPs5k69OdbaMgfB66qfBqlELFPk6r9ANztmGMOSLqaXEWXEPwk-6JnMZ7-RaXAYLd5thLbcL3QzgeU',
19
+ 'aspera.global-cli-client' => 'bK_qpqFpP-OPEuRJ9mnmdw_ebLtpSqCnqhuAKfKdoLXC6OF2yLMgsfAMBmXg7XI_zplV4gBqNOvlJdgCxlP0Zjm4GsRsmprf'
20
+ }
21
+ # path in URL of public links
22
+ PATHS_PUBLIC_LINK=['/packages/public/receive','/packages/public/send','/files/public']
23
+ private_constant :PRODUCT_NAME,:PROD_DOMAIN,:MAX_REDIRECT,:DEFAULT_CLIENT,:CLIENT_RANDOM,:PATHS_PUBLIC_LINK
8
24
 
9
25
  public
10
26
  # various API scopes supported
@@ -15,39 +31,22 @@ module Asperalm
15
31
  SCOPE_FILES_ADMIN_USER_USER=SCOPE_FILES_ADMIN_USER+'+'+SCOPE_FILES_USER
16
32
  SCOPE_NODE_USER='user:all'
17
33
  SCOPE_NODE_ADMIN='admin:all'
18
- # path in URL of public package links
19
- PATHS_PUBLIC_LINK=['/packages/public/receive','/packages/public/send','/files/public']
20
34
  PATH_SEPARATOR='/'
21
-
22
- FILES='files'
23
- PACKAGES='packages'
24
-
25
- PRODUCT_NAME='Aspera on Cloud'
26
- PRODUCT_DOMAIN='ibmaspera.com'
27
- private_constant :PRODUCT_NAME,:PRODUCT_DOMAIN
28
-
29
- # strings /Applications/Aspera\ Drive.app/Contents/MacOS/AsperaDrive|grep -E '.{100}==$'|rev
30
- RANDOM_DRIVE='==QMGdXZsdkYMlDezZ3MNhDStYFWQNXNrZTOPRmYh10ZmJkN2EnZCFHbFxkRQtmNylTQOpHdtdUTPNFTxFGWFdFWFB1dr1iNK5WTadTLSFGWBlFTkVDdoxkYjx0MRp3ZlVlOlZXayRmLhJXZwNXY'
31
- # found in aspera CLI
32
- RANDOM_CLIENT='==gYL9VcwFnRwBVLPBVR1JlS50mbtR2dfVmYMRHcTF3QuFHa1F0SmtEZvxEWDZzTGJTeM10ZzZWQNJUbYd2NYl0X6BHbWRzZCFnTPZHbKR2ZDhHbQBjWq1GNHNnUz1GcyZmO05WZpx2YtkGbj1CbhJ2bsdmLhJXZwNXY'
33
- private_constant :RANDOM_DRIVE,:RANDOM_CLIENT
34
-
35
- def self.random_drive
36
- Base64.strict_decode64(RANDOM_DRIVE.reverse).split(':')
37
- end
38
-
39
- def self.random_cli
40
- Base64.strict_decode64(RANDOM_CLIENT.reverse).split(':')
35
+ FILES_APP='files'
36
+ PACKAGES_APP='packages'
37
+
38
+ def self.get_client_ids(id,secret,client_name=DEFAULT_CLIENT)
39
+ return id,secret unless id.nil? and secret.nil?
40
+ return client_name,CLIENT_RANDOM[client_name].reverse
41
41
  end
42
42
 
43
43
  # @param url of AoC instance
44
- # @return organization id and AoC domain
45
- # AoC domain is: ibmaspera.com, asperafiles.com or qa.asperafiles.com, etc...
44
+ # @return organization id in url and AoC domain: ibmaspera.com, asperafiles.com or qa.asperafiles.com, etc...
46
45
  def self.parse_url(aoc_org_url)
47
46
  uri=URI.parse(aoc_org_url.gsub(/\/+$/,''))
48
47
  instance_fqdn=uri.host
49
48
  Log.log.debug("instance_fqdn=#{instance_fqdn}")
50
- raise "No host found in URL.Please check URL format: https://myorg.#{PRODUCT_DOMAIN}" if instance_fqdn.nil?
49
+ raise "No host found in URL.Please check URL format: https://myorg.#{PROD_DOMAIN}" if instance_fqdn.nil?
51
50
  organization,instance_domain=instance_fqdn.split('.',2)
52
51
  Log.log.debug("instance_domain=#{instance_domain}")
53
52
  Log.log.debug("organization=#{organization}")
@@ -55,21 +54,7 @@ module Asperalm
55
54
  return organization,instance_domain
56
55
  end
57
56
 
58
- # @param url of AoC instance
59
- # @return necessary fixed information to create JWT or call API
60
- def self.base_rest_params(aoc_org_url)
61
- organization,instance_domain=parse_url(aoc_org_url)
62
- base_url='https://api.'+instance_domain+'/api/v1'
63
- return {
64
- :base_url => base_url,
65
- :auth => {
66
- :type => :oauth2,
67
- :base_url => "#{base_url}/oauth2/#{organization}",
68
- :jwt_audience => 'https://api.asperafiles.com/api/v1/oauth2/token'
69
- }}
70
- end
71
-
72
- def self.metering_api(entitlement_id,customer_id,api_domain=PRODUCT_DOMAIN)
57
+ def self.metering_api(entitlement_id,customer_id,api_domain=PROD_DOMAIN)
73
58
  return Rest.new({
74
59
  :base_url => "https://api.#{api_domain}/metering/v1",
75
60
  :headers => {'X-Aspera-Entitlement-Authorization' => Rest.basic_creds(entitlement_id,customer_id)}
@@ -81,17 +66,100 @@ module Asperalm
81
66
  return 'node.'+access_key+':'+scope
82
67
  end
83
68
 
84
- # @return true if the OAuth client_id is globally defined in AoC
85
- def self.is_global_client_id?(client_id)
86
- client_id.is_a?(String) and client_id.start_with?('aspera.global')
69
+ # check option "link"
70
+ # if present try to get token value (resolve redirection if short links used)
71
+ # then set options url/token/auth
72
+ def self.resolve_pub_link(rest_opts,public_link_url)
73
+ return if public_link_url.nil?
74
+ # set to token if available after redirection
75
+ url_param_token_pair=nil
76
+ redirect_count=0
77
+ loop do
78
+ uri=URI.parse(public_link_url)
79
+ if PATHS_PUBLIC_LINK.include?(uri.path)
80
+ url_param_token_pair=URI::decode_www_form(uri.query).select{|e|e.first.eql?('token')}.first
81
+ if url_param_token_pair.nil?
82
+ raise ArgumentError,"link option must be URL with 'token' parameter"
83
+ end
84
+ # ok we get it !
85
+ rest_opts[:org_url]='https://'+uri.host
86
+ rest_opts[:auth][:grant]=:url_token
87
+ rest_opts[:auth][:url_token]=url_param_token_pair.last
88
+ return
89
+ end
90
+ Log.log.debug("no expected format: #{public_link_url}")
91
+ raise "exceeded max redirection: #{MAX_REDIRECT}" if redirect_count > MAX_REDIRECT
92
+ r = Net::HTTP.get_response(uri)
93
+ if r.code.start_with?("3")
94
+ public_link_url = r['location']
95
+ raise "no location in redirection" if public_link_url.nil?
96
+ Log.log.debug("redirect to: #{public_link_url}")
97
+ else
98
+ # not a redirection
99
+ raise ArgumentError,'link option must be redirect or have token parameter'
100
+ end
101
+ end # loop
102
+
103
+ raise RuntimeError,'too many redirections'
87
104
  end
88
105
 
89
- def initialize(rest_params)
90
- super(rest_params)
106
+ # @param :link,:url,:auth,:client_id,:client_secret,:scope,:redirect_uri,:private_key,:username,:subpath
107
+ def initialize(opt)
91
108
  # access key secrets are provided out of band to get node api access
92
109
  # key: access key
93
110
  # value: associated secret
94
111
  @secrets={}
112
+
113
+ # init rest params
114
+ aoc_rest_p={:auth=>{:type =>:oauth2}}
115
+ # shortcut to auth section
116
+ aoc_auth_p=aoc_rest_p[:auth]
117
+
118
+ # sets [:org_url], [:auth][:grant], [:auth][:url_token]
119
+ self.class.resolve_pub_link(aoc_rest_p,opt[:link])
120
+
121
+ # get org url from pub link or options
122
+ if aoc_rest_p.has_key?(:org_url)
123
+ opt[:url] = aoc_rest_p[:org_url]
124
+ aoc_rest_p.delete(:org_url)
125
+ else
126
+ raise ArgumentError,"Missing mandatory option: url" if opt[:url].nil?
127
+ end
128
+
129
+ # set API and OAuth URLs
130
+ organization,instance_domain=self.class.parse_url(opt[:url])
131
+ aoc_rest_p[:base_url]="https://api.#{instance_domain}/#{opt[:subpath]}"
132
+ aoc_auth_p[:base_url] = "#{aoc_rest_p[:base_url]}/oauth2/#{organization}"
133
+
134
+ if !aoc_auth_p.has_key?(:grant)
135
+ raise ArgumentError,"Missing mandatory option: auth" if opt[:auth].nil?
136
+ aoc_auth_p[:grant] = opt[:auth]
137
+ end
138
+
139
+ aoc_auth_p[:client_id],aoc_auth_p[:client_secret] = self.class.get_client_ids(opt[:client_id],opt[:client_secret])
140
+ aoc_auth_p[:scope] = opt[:scope]
141
+
142
+ # fill other auth parameters based on Oauth method
143
+ case aoc_auth_p[:grant]
144
+ when :web
145
+ raise ArgumentError,"Missing mandatory option: redirect_uri" if opt[:redirect_uri].nil?
146
+ aoc_auth_p[:redirect_uri] = opt[:redirect_uri]
147
+ when :jwt
148
+ # add jwt payload for global ids
149
+ if CLIENT_RANDOM.keys.include?(aoc_auth_p[:client_id])
150
+ aoc_auth_p.merge!({:jwt_add=>{org: organization}})
151
+ end
152
+ raise ArgumentError,"Missing mandatory option: private_key" if opt[:private_key].nil?
153
+ raise ArgumentError,"Missing mandatory option: username" if opt[:username].nil?
154
+ private_key_PEM_string=opt[:private_key]
155
+ aoc_auth_p[:jwt_audience] = 'https://api.asperafiles.com/api/v1/oauth2/token'
156
+ aoc_auth_p[:jwt_subject] = opt[:username]
157
+ aoc_auth_p[:jwt_private_key_obj] = OpenSSL::PKey::RSA.new(private_key_PEM_string)
158
+ when :url_token
159
+ # nothing more
160
+ else raise "ERROR: unsupported auth method: #{aoc_auth_p[:grant]}"
161
+ end
162
+ super(aoc_rest_p)
95
163
  end
96
164
 
97
165
  def add_secrets(secrets)
@@ -115,7 +183,7 @@ module Asperalm
115
183
  end
116
184
 
117
185
  # get transfer connection parameters
118
- def tr_spec_remote_info(node_info)
186
+ def self.tr_spec_remote_info(node_info)
119
187
  #TODO: add option to request those parameters by calling /upload_setup on node api
120
188
  return {
121
189
  'remote_user' => 'xfer',
@@ -148,7 +216,7 @@ module Asperalm
148
216
  }
149
217
  end
150
218
 
151
- # build ts addon for IBM Aspera Console
219
+ # build ts addon for IBM Aspera Console (cookie)
152
220
  def self.console_ts(app,user_name,user_email)
153
221
  elements=[app,user_name,user_email].map{|e|Base64.strict_encode64(e)}
154
222
  elements.unshift('aspera.aoc')
@@ -183,7 +251,7 @@ module Asperalm
183
251
  } # aspera
184
252
  } # tags
185
253
  }
186
- transfer_spec.merge!(tr_spec_remote_info(node_file[:node_info]))
254
+ transfer_spec.merge!(self.class.tr_spec_remote_info(node_file[:node_info]))
187
255
  # add caller provided transfer spec
188
256
  transfer_spec.deep_merge!(ts_add)
189
257
  # additional information for transfer agent
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: asperalm
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.10.11
4
+ version: 0.10.12
5
5
  platform: ruby
6
6
  authors:
7
7
  - Laurent Martin
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-04-29 00:00:00.000000000 Z
11
+ date: 2020-05-15 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: xml-simple