aspera-cli 4.10.0 → 4.11.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (94) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +20 -0
  4. data/CHANGELOG.md +509 -0
  5. data/CONTRIBUTING.md +118 -0
  6. data/README.md +621 -378
  7. data/bin/ascli +4 -4
  8. data/bin/asession +11 -11
  9. data/docs/test_env.conf +28 -19
  10. data/examples/aoc.rb +4 -4
  11. data/examples/dascli +11 -9
  12. data/examples/faspex4.rb +8 -8
  13. data/examples/node.rb +11 -11
  14. data/examples/server.rb +9 -9
  15. data/lib/aspera/aoc.rb +273 -266
  16. data/lib/aspera/ascmd.rb +56 -54
  17. data/lib/aspera/ats_api.rb +4 -4
  18. data/lib/aspera/cli/basic_auth_plugin.rb +15 -12
  19. data/lib/aspera/cli/extended_value.rb +5 -5
  20. data/lib/aspera/cli/formater.rb +64 -64
  21. data/lib/aspera/cli/listener/line_dump.rb +1 -1
  22. data/lib/aspera/cli/listener/logger.rb +1 -1
  23. data/lib/aspera/cli/listener/progress.rb +5 -6
  24. data/lib/aspera/cli/listener/progress_multi.rb +14 -19
  25. data/lib/aspera/cli/main.rb +66 -67
  26. data/lib/aspera/cli/manager.rb +110 -110
  27. data/lib/aspera/cli/plugin.rb +54 -37
  28. data/lib/aspera/cli/plugins/alee.rb +4 -4
  29. data/lib/aspera/cli/plugins/aoc.rb +308 -669
  30. data/lib/aspera/cli/plugins/ats.rb +44 -46
  31. data/lib/aspera/cli/plugins/bss.rb +10 -10
  32. data/lib/aspera/cli/plugins/config.rb +447 -344
  33. data/lib/aspera/cli/plugins/console.rb +12 -12
  34. data/lib/aspera/cli/plugins/cos.rb +18 -20
  35. data/lib/aspera/cli/plugins/faspex.rb +110 -112
  36. data/lib/aspera/cli/plugins/faspex5.rb +67 -46
  37. data/lib/aspera/cli/plugins/node.rb +364 -288
  38. data/lib/aspera/cli/plugins/orchestrator.rb +46 -46
  39. data/lib/aspera/cli/plugins/preview.rb +122 -114
  40. data/lib/aspera/cli/plugins/server.rb +137 -83
  41. data/lib/aspera/cli/plugins/shares.rb +30 -29
  42. data/lib/aspera/cli/plugins/sync.rb +13 -33
  43. data/lib/aspera/cli/transfer_agent.rb +57 -57
  44. data/lib/aspera/cli/version.rb +1 -1
  45. data/lib/aspera/colors.rb +3 -3
  46. data/lib/aspera/command_line_builder.rb +27 -27
  47. data/lib/aspera/cos_node.rb +22 -20
  48. data/lib/aspera/data_repository.rb +1 -1
  49. data/lib/aspera/environment.rb +30 -28
  50. data/lib/aspera/fasp/agent_base.rb +15 -15
  51. data/lib/aspera/fasp/agent_connect.rb +23 -21
  52. data/lib/aspera/fasp/agent_direct.rb +65 -67
  53. data/lib/aspera/fasp/agent_httpgw.rb +72 -68
  54. data/lib/aspera/fasp/agent_node.rb +23 -21
  55. data/lib/aspera/fasp/agent_trsdk.rb +20 -20
  56. data/lib/aspera/fasp/error.rb +3 -2
  57. data/lib/aspera/fasp/error_info.rb +11 -8
  58. data/lib/aspera/fasp/installation.rb +78 -78
  59. data/lib/aspera/fasp/listener.rb +1 -1
  60. data/lib/aspera/fasp/parameters.rb +75 -72
  61. data/lib/aspera/fasp/parameters.yaml +2 -2
  62. data/lib/aspera/fasp/resume_policy.rb +8 -8
  63. data/lib/aspera/fasp/transfer_spec.rb +35 -2
  64. data/lib/aspera/fasp/uri.rb +7 -7
  65. data/lib/aspera/faspex_gw.rb +7 -5
  66. data/lib/aspera/hash_ext.rb +3 -3
  67. data/lib/aspera/id_generator.rb +5 -5
  68. data/lib/aspera/keychain/encrypted_hash.rb +23 -28
  69. data/lib/aspera/keychain/macos_security.rb +21 -20
  70. data/lib/aspera/log.rb +7 -7
  71. data/lib/aspera/nagios.rb +19 -18
  72. data/lib/aspera/node.rb +209 -35
  73. data/lib/aspera/oauth.rb +37 -36
  74. data/lib/aspera/open_application.rb +19 -11
  75. data/lib/aspera/persistency_action_once.rb +4 -4
  76. data/lib/aspera/persistency_folder.rb +13 -13
  77. data/lib/aspera/preview/file_types.rb +8 -8
  78. data/lib/aspera/preview/generator.rb +67 -67
  79. data/lib/aspera/preview/utils.rb +27 -27
  80. data/lib/aspera/proxy_auto_config.js +41 -41
  81. data/lib/aspera/proxy_auto_config.rb +16 -16
  82. data/lib/aspera/rest.rb +56 -60
  83. data/lib/aspera/rest_call_error.rb +2 -1
  84. data/lib/aspera/rest_error_analyzer.rb +18 -17
  85. data/lib/aspera/rest_errors_aspera.rb +16 -16
  86. data/lib/aspera/secret_hider.rb +15 -13
  87. data/lib/aspera/ssh.rb +11 -10
  88. data/lib/aspera/sync.rb +158 -44
  89. data/lib/aspera/temp_file_manager.rb +2 -2
  90. data/lib/aspera/uri_reader.rb +4 -4
  91. data/lib/aspera/web_auth.rb +14 -13
  92. data.tar.gz.sig +0 -0
  93. metadata +8 -5
  94. metadata.gz.sig +0 -0
data/lib/aspera/node.rb CHANGED
@@ -9,22 +9,23 @@ require 'zlib'
9
9
  require 'base64'
10
10
 
11
11
  module Aspera
12
- # Provides additional functions using node API.
13
- class Node < Rest
12
+ # Provides additional functions using node API with gen4 extensions (access keys)
13
+ class Node < Aspera::Rest
14
14
  # permissions
15
15
  ACCESS_LEVELS = %w[delete list mkdir preview read rename write].freeze
16
16
  # prefix for ruby code for filter
17
17
  MATCH_EXEC_PREFIX = 'exec:'
18
+ X_ASPERA_ACCESSKEY = 'X-Aspera-AccessKey'
19
+ PATH_SEPARATOR = '/'
18
20
 
19
21
  # register node special token decoder
20
22
  Oauth.register_decoder(lambda{|token|JSON.parse(Zlib::Inflate.inflate(Base64.decode64(token)).partition('==SIGNATURE==').first)})
21
23
 
24
+ # class instance variable, access with accessors on class
25
+ @use_standard_ports = true
26
+
22
27
  class << self
23
- def set_ak_basic_token(ts,ak,secret)
24
- Log.log.warn("Expected transfer user: #{Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER}, "\
25
- "but have #{ts['remote_user']}") unless ts['remote_user'].eql?(Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER)
26
- ts['token'] = Rest.basic_creds(ak,secret)
27
- end
28
+ attr_accessor :use_standard_ports
28
29
 
29
30
  # for access keys: provide expression to match entry in folder
30
31
  # if no prefix: regex
@@ -39,49 +40,222 @@ module Aspera
39
40
  end
40
41
  end
41
42
 
42
- # def initialize(rest_params)
43
- # super(rest_params)
44
- # end
43
+ REQUIRED_APP_INFO_FIELDS = %i[node_info app api plugin].freeze
44
+ REQUIRED_APP_API_METHODS = %i[node_id_to_api add_ts_tags].freeze
45
+ private_constant :REQUIRED_APP_INFO_FIELDS, :REQUIRED_APP_API_METHODS
46
+
47
+ attr_reader :app_info
48
+
49
+ # @param params [Hash] Rest parameters
50
+ # @param app_info [Hash,NilClass] special processing for AoC
51
+ def initialize(params:, app_info: nil, add_tspec: nil)
52
+ super(params)
53
+ @app_info = app_info
54
+ # this is added to transfer spec, for instance to add tags (COS)
55
+ @add_tspec = add_tspec
56
+ if !@app_info.nil?
57
+ REQUIRED_APP_INFO_FIELDS.each do |field|
58
+ raise "INTERNAL ERROR: app_info lacks field #{field}" unless @app_info.key?(field)
59
+ end
60
+ REQUIRED_APP_API_METHODS.each do |method|
61
+ raise "INTERNAL ERROR: #{@app_info[:api].class} lacks method #{method}" unless @app_info[:api].respond_to?(method)
62
+ end
63
+ end
64
+ end
65
+
66
+ # update transfer spec with special additional tags
67
+ def add_tspec_info(tspec)
68
+ tspec.deep_merge!(@add_tspec) unless @add_tspec.nil?
69
+ return tspec
70
+ end
71
+
72
+ # @returns [Aspera::Node] a Node or nil
73
+ def node_id_to_node(node_id)
74
+ return self if !@app_info.nil? && @app_info[:node_info]['id'].eql?(node_id)
75
+ return @app_info[:api].node_id_to_api(node_id) unless @app_info.nil?
76
+ Log.log.warn{"cannot resolve link with node id #{node_id}"}
77
+ return nil
78
+ end
45
79
 
46
- # recursively crawl in a folder.
80
+ # recursively browse in a folder (with non-recursive method)
47
81
  # subfolders a processed if the processing method returns true
48
- # @param processor must provide a method to process each entry
49
- # @param opt options
50
- # - top_file_id file id to start at (default = access key root file id)
51
- # - top_file_path path of top folder (default = /)
52
- # - method processing method (default= process_entry)
53
- def crawl(processor,opt={})
54
- Log.log.debug("crawl1 #{opt}")
55
- # not possible with bearer token
56
- opt[:top_file_id] ||= read('access_keys/self')[:data]['root_file_id']
57
- opt[:top_file_path] ||= '/'
58
- opt[:method] ||= :process_entry
59
- raise "processor must have #{opt[:method]}" unless processor.respond_to?(opt[:method])
60
- Log.log.debug("crawl #{opt}")
61
- #top_info=read("files/#{opt[:top_file_id]}")[:data]
62
- folders_to_explore = [{id: opt[:top_file_id], relpath: opt[:top_file_path]}]
63
- Log.dump(:folders_to_explore,folders_to_explore)
64
- while !folders_to_explore.empty?
82
+ # @param state [Object] state object sent to processing method
83
+ # @param method [Symbol] processing method name
84
+ # @param top_file_id [String] file id to start at (default = access key root file id)
85
+ # @param top_file_path [String] path of top folder (default = /)
86
+ def process_folder_tree(state:, method:, top_file_id:, top_file_path: '/')
87
+ raise 'INTERNAL ERROR: top_file_path not set' if top_file_path.nil?
88
+ raise "INTERNAL ERROR: Missing method #{method}" unless respond_to?(method)
89
+ folders_to_explore = [{id: top_file_id, relpath: top_file_path}]
90
+ Log.dump(:folders_to_explore, folders_to_explore)
91
+ until folders_to_explore.empty?
65
92
  current_item = folders_to_explore.shift
66
- Log.log.debug("searching #{current_item[:relpath]}".bg_green)
93
+ Log.log.debug{"searching #{current_item[:relpath]}".bg_green}
67
94
  # get folder content
68
95
  folder_contents =
69
96
  begin
70
97
  read("files/#{current_item[:id]}/files")[:data]
71
98
  rescue StandardError => e
72
- Log.log.warn("#{current_item[:relpath]}: #{e.class} #{e.message}")
99
+ Log.log.warn{"#{current_item[:relpath]}: #{e.class} #{e.message}"}
73
100
  []
74
101
  end
75
- Log.dump(:folder_contents,folder_contents)
102
+ Log.dump(:folder_contents, folder_contents)
76
103
  folder_contents.each do |entry|
77
- relative_path = File.join(current_item[:relpath],entry['name'])
78
- Log.log.debug("looking #{relative_path}".bg_green)
104
+ relative_path = File.join(current_item[:relpath], entry['name'])
105
+ Log.log.debug{"looking #{relative_path}".bg_green}
106
+ # continue only if method returns true
107
+ next unless send(method, entry, relative_path, state)
79
108
  # entry type is file, folder or link
80
- if processor.send(opt[:method],entry,relative_path) && entry['type'].eql?('folder')
81
- folders_to_explore.push({id: entry['id'],relpath: relative_path})
109
+ case entry['type']
110
+ when 'folder'
111
+ folders_to_explore.push({id: entry['id'], relpath: relative_path})
112
+ when 'link'
113
+ node_id_to_node(entry['target_node_id'])&.process_folder_tree(
114
+ state: state,
115
+ method: method,
116
+ top_file_id: entry['target_id'],
117
+ top_file_path: relative_path)
82
118
  end
83
119
  end
84
120
  end
121
+ end # process_folder_tree
122
+
123
+ # processing method to resolve a file path to id
124
+ # @returns true if processing need to continue
125
+ def process_resolve_node_path(entry, _path, state)
126
+ # stop digging here if not in right path
127
+ return false unless entry['name'].eql?(state[:path].first)
128
+ # ok it matches, so we remove the match
129
+ state[:path].shift
130
+ case entry['type']
131
+ when 'file'
132
+ # file must be terminal
133
+ raise "#{entry['name']} is a file, expecting folder to find: #{state[:path]}" unless state[:path].empty?
134
+ # it's terminal, we found it
135
+ state[:result] = {api: self, file_id: entry['id']}
136
+ return false
137
+ when 'folder'
138
+ if state[:path].empty?
139
+ # we found it
140
+ state[:result] = {api: self, file_id: entry['id']}
141
+ return false
142
+ end
143
+ when 'link'
144
+ if state[:path].empty?
145
+ # we found it
146
+ other_node = node_id_to_node(entry['target_node_id'])
147
+ raise 'cannot resolve link' if other_node.nil?
148
+ state[:result] = {api: other_node, file_id: entry['target_id']}
149
+ return false
150
+ end
151
+ else
152
+ Log.log.warn{"Unknown element type: #{entry['type']}"}
153
+ end
154
+ # continue to dig folder
155
+ return true
156
+ end
157
+
158
+ # Navigate the path from given file id
159
+ # @param id initial file id
160
+ # @param path file path
161
+ # @return {.api,.file_id}
162
+ def resolve_api_fid(top_file_id, path)
163
+ raise 'file id shall be String' unless top_file_id.is_a?(String)
164
+ path_elements = path.split(PATH_SEPARATOR).reject(&:empty?)
165
+ return {api: self, file_id: top_file_id} if path_elements.empty?
166
+ resolve_state = {path: path_elements, result: nil}
167
+ process_folder_tree(state: resolve_state, method: :process_resolve_node_path, top_file_id: top_file_id)
168
+ raise "entry not found: #{resolve_state[:path]}" if resolve_state[:result].nil?
169
+ return resolve_state[:result]
170
+ end
171
+
172
+ # add entry to list if test block is success
173
+ def process_find_files(entry, path, state)
174
+ begin
175
+ # add to result if match filter
176
+ state[:found].push(entry.merge({'path' => path})) if state[:test_block].call(entry)
177
+ # process link
178
+ if entry[:type].eql?('link')
179
+ other_node = node_id_to_node(entry['target_node_id'])
180
+ other_node.process_folder_tree(state: state, method: process_find_files, top_file_id: entry['target_id'], top_file_path: path)
181
+ end
182
+ rescue StandardError => e
183
+ Log.log.error{"#{path}: #{e.message}"}
184
+ end
185
+ # process all folders
186
+ return true
187
+ end
188
+
189
+ def find_files(top_file_id, test_block)
190
+ Log.log.debug{"find_files: fileid=#{top_file_id}"}
191
+ find_state = {found: [], test_block: test_block}
192
+ process_folder_tree(state: find_state, method: :process_find_files, top_file_id: top_file_id)
193
+ return find_state[:found]
194
+ end
195
+
196
+ # Create transfer spec for gen4
197
+ def transfer_spec_gen4(file_id, direction, ts_merge=nil)
198
+ ak_name = nil
199
+ ak_token = nil
200
+ case params[:auth][:type]
201
+ when :basic
202
+ ak_name = params[:auth][:username]
203
+ when :oauth2
204
+ ak_name = params[:headers][X_ASPERA_ACCESSKEY]
205
+ token_generation_lambda = lambda{|do_refresh|oauth_token(force_refresh: do_refresh)}
206
+ ak_token = token_generation_lambda.call(false) # first time, use cache
207
+ @app_info[:plugin].transfer.token_regenerator = token_generation_lambda unless @app_info.nil?
208
+ else raise "Unsupported auth method for node gen4: #{params[:auth][:type]}"
209
+ end
210
+ transfer_spec = {
211
+ 'direction' => direction,
212
+ 'token' => ak_token,
213
+ 'tags' => {
214
+ 'aspera' => {
215
+ 'node' => {
216
+ 'access_key' => ak_name,
217
+ 'file_id' => file_id
218
+ } # node
219
+ } # aspera
220
+ } # tags
221
+ }
222
+ # add specials tags (cos)
223
+ add_tspec_info(transfer_spec)
224
+ transfer_spec.deep_merge!(ts_merge) unless ts_merge.nil?
225
+ # add application specific tags (AoC)
226
+ the_app = app_info
227
+ the_app[:api].add_ts_tags(transfer_spec: transfer_spec, app_info: the_app) unless the_app.nil?
228
+ # add basic token
229
+ if transfer_spec['token'].nil?
230
+ ts_basic_token(transfer_spec)
231
+ end
232
+ # add remote host info
233
+ if self.class.use_standard_ports
234
+ # get default TCP/UDP ports and transfer user
235
+ transfer_spec.merge!(Fasp::TransferSpec::AK_TSPEC_BASE)
236
+ # by default: same address as node API
237
+ transfer_spec['remote_host'] = URI.parse(params[:base_url]).host
238
+ if !@app_info.nil? && !@app_info[:node_info]['transfer_url'].nil? && !@app_info[:node_info]['transfer_url'].empty?
239
+ transfer_spec['remote_host'] = @app_info[:node_info]['transfer_url']
240
+ end
241
+ else
242
+ # retrieve values from API
243
+ std_t_spec = create(
244
+ 'files/download_setup',
245
+ {transfer_requests: [{ transfer_request: {paths: [{'source' => '/'}] } }] }
246
+ )[:data]['transfer_specs'].first['transfer_spec']
247
+ # copy some parts
248
+ %w[remote_host remote_user ssh_port fasp_port wss_enabled wss_port].each {|i| transfer_spec[i] = std_t_spec[i] if std_t_spec.key?(i)}
249
+ end
250
+ return transfer_spec
251
+ end
252
+
253
+ # set basic token in transfer spec
254
+ def ts_basic_token(ts)
255
+ Log.log.warn{"Expected transfer user: #{Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER}, but have #{ts['remote_user']}"} \
256
+ unless ts['remote_user'].eql?(Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER)
257
+ raise 'ERROR: no secret in node object' unless params[:auth][:password]
258
+ ts['token'] = Rest.basic_creds(params[:auth][:username], params[:auth][:password])
85
259
  end
86
260
  end
87
261
  end
data/lib/aspera/oauth.rb CHANGED
@@ -35,7 +35,7 @@ module Aspera
35
35
  # a prefix for persistency of tokens (simplify garbage collect)
36
36
  PERSIST_CATEGORY_TOKEN = 'token'
37
37
 
38
- private_constant :JWT_NOTBEFORE_OFFSET_SEC,:JWT_EXPIRY_OFFSET_SEC,:TOKEN_CACHE_EXPIRY_SEC,:PERSIST_CATEGORY_TOKEN,:TOKEN_EXPIRATION_GUARD_SEC
38
+ private_constant :JWT_NOTBEFORE_OFFSET_SEC, :JWT_EXPIRY_OFFSET_SEC, :TOKEN_CACHE_EXPIRY_SEC, :PERSIST_CATEGORY_TOKEN, :TOKEN_EXPIRATION_GUARD_SEC
39
39
 
40
40
  # persistency manager
41
41
  @persist = nil
@@ -48,7 +48,7 @@ module Aspera
48
48
  def persist_mgr=(manager)
49
49
  @persist = manager
50
50
  # cleanup expired tokens
51
- @persist.garbage_collect(PERSIST_CATEGORY_TOKEN,TOKEN_CACHE_EXPIRY_SEC)
51
+ @persist.garbage_collect(PERSIST_CATEGORY_TOKEN, TOKEN_CACHE_EXPIRY_SEC)
52
52
  end
53
53
 
54
54
  def persist_mgr
@@ -56,7 +56,7 @@ module Aspera
56
56
  Log.log.debug('Not using persistency') # (use Aspera::Oauth.persist_mgr=Aspera::PersistencyFolder.new)
57
57
  # create NULL persistency class
58
58
  @persist = Class.new do
59
- def get(_x);nil;end;def delete(_x);nil;end;def put(_x,_y);nil;end;def garbage_collect(_x,_y);nil;end # rubocop:disable Layout/EmptyLineBetweenDefs
59
+ def get(_x); nil; end; def delete(_x); nil; end; def put(_x, _y); nil; end; def garbage_collect(_x, _y); nil; end # rubocop:disable Layout/EmptyLineBetweenDefs, Style/Semicolon, Layout/LineLength
60
60
  end.new
61
61
  end
62
62
  return @persist
@@ -64,7 +64,7 @@ module Aspera
64
64
 
65
65
  # delete all existing tokens
66
66
  def flush_tokens
67
- persist_mgr.garbage_collect(PERSIST_CATEGORY_TOKEN,nil)
67
+ persist_mgr.garbage_collect(PERSIST_CATEGORY_TOKEN, nil)
68
68
  end
69
69
 
70
70
  # register a bearer token decoder, mainly to inspect expiry date
@@ -94,24 +94,24 @@ module Aspera
94
94
 
95
95
  # @return one of the registered creators for the given create type
96
96
  def token_creator(id)
97
- raise "token creator type unknown: #{id}/#{id.class}" unless @create_handlers.has_key?(id)
97
+ raise "token creator type unknown: #{id}/#{id.class}" unless @create_handlers.key?(id)
98
98
  @create_handlers[id]
99
99
  end
100
100
 
101
101
  # list of identifiers foundn in creation parameters that can be used to uniquely identify the token
102
102
  def id_creator(id)
103
- raise "id creator type unknown: #{id}/#{id.class}" unless @id_handlers.has_key?(id)
103
+ raise "id creator type unknown: #{id}/#{id.class}" unless @id_handlers.key?(id)
104
104
  @id_handlers[id]
105
105
  end
106
106
  end # self
107
107
 
108
108
  # JSON Web Signature (JWS) compact serialization: https://datatracker.ietf.org/doc/html/rfc7515
109
- register_decoder lambda { |token| parts = token.split('.'); raise 'not aoc token' unless parts.length.eql?(3); JSON.parse(Base64.decode64(parts[1]))}
109
+ register_decoder lambda { |token| parts = token.split('.'); raise 'not aoc token' unless parts.length.eql?(3); JSON.parse(Base64.decode64(parts[1]))} # rubocop:disable Style/Semicolon, Layout/LineLength
110
110
 
111
111
  # generic token creation, parameters are provided in :generic
112
- register_token_creator :generic,lambda { |oauth|
112
+ register_token_creator :generic, lambda { |oauth|
113
113
  return oauth.create_token(oauth.sparams)
114
- },lambda { |oauth|
114
+ }, lambda { |oauth|
115
115
  return [
116
116
  oauth.sparams[:grant_type]&.split(':')&.last,
117
117
  oauth.sparams[:apikey],
@@ -120,12 +120,13 @@ module Aspera
120
120
  }
121
121
 
122
122
  # Authentication using Web browser
123
- register_token_creator :web,lambda { |oauth|
123
+ register_token_creator :web, lambda { |oauth|
124
124
  random_state = SecureRandom.uuid # used to check later
125
- login_page_url = Rest.build_uri("#{oauth.api[:base_url]}/#{oauth.sparams[:path_authorize]}",
125
+ login_page_url = Rest.build_uri(
126
+ "#{oauth.api.params[:base_url]}/#{oauth.sparams[:path_authorize]}",
126
127
  oauth.optional_scope_client_id.merge(response_type: 'code', redirect_uri: oauth.sparams[:redirect_uri], state: random_state))
127
128
  # here, we need a human to authorize on a web page
128
- Log.log.info("login_page_url=#{login_page_url}".bg_red.gray)
129
+ Log.log.info{"login_page_url=#{login_page_url}".bg_red.gray}
129
130
  # start a web server to receive request code
130
131
  webserver = WebAuth.new(oauth.sparams[:redirect_uri])
131
132
  # start browser on login page
@@ -138,17 +139,17 @@ module Aspera
138
139
  grant_type: 'authorization_code',
139
140
  code: received_params['code'],
140
141
  redirect_uri: oauth.sparams[:redirect_uri]))
141
- },lambda { |_oauth|
142
+ }, lambda { |_oauth|
142
143
  return []
143
144
  }
144
145
 
145
146
  # Authentication using private key
146
- register_token_creator :jwt,lambda { |oauth|
147
+ register_token_creator :jwt, lambda { |oauth|
147
148
  # https://tools.ietf.org/html/rfc7523
148
149
  # https://tools.ietf.org/html/rfc7519
149
150
  require 'jwt'
150
151
  seconds_since_epoch = Time.new.to_i
151
- Log.log.info("seconds=#{seconds_since_epoch}")
152
+ Log.log.info{"seconds=#{seconds_since_epoch}"}
152
153
  raise 'missing JWT payload' unless oauth.sparams[:payload].is_a?(Hash)
153
154
  jwt_payload = {
154
155
  exp: seconds_since_epoch + JWT_EXPIRY_OFFSET_SEC, # expiration time
@@ -156,14 +157,14 @@ module Aspera
156
157
  iat: seconds_since_epoch, # issued at
157
158
  jti: SecureRandom.uuid # JWT id
158
159
  }.merge(oauth.sparams[:payload])
159
- Log.log.debug("JWT jwt_payload=[#{jwt_payload}]")
160
+ Log.log.debug{"JWT jwt_payload=[#{jwt_payload}]"}
160
161
  rsa_private = oauth.sparams[:private_key_obj] # type: OpenSSL::PKey::RSA
161
- Log.log.debug("private=[#{rsa_private}]")
162
+ Log.log.debug{"private=[#{rsa_private}]"}
162
163
  assertion = JWT.encode(jwt_payload, rsa_private, 'RS256', oauth.sparams[:headers] || {})
163
- Log.log.debug("assertion=[#{assertion}]")
164
+ Log.log.debug{"assertion=[#{assertion}]"}
164
165
  return oauth.create_token(oauth.optional_scope_client_id.merge(grant_type: 'urn:ietf:params:oauth:grant-type:jwt-bearer', assertion: assertion))
165
- },lambda { |oauth|
166
- return [oauth.sparams.dig(:payload,:sub)]
166
+ }, lambda { |oauth|
167
+ return [oauth.sparams.dig(:payload, :sub)]
167
168
  }
168
169
 
169
170
  attr_reader :gparams, :sparams, :api
@@ -186,14 +187,14 @@ module Aspera
186
187
  # :web:path_authorize [D] for type :web
187
188
  # :generic [M] for type :generic
188
189
  def initialize(a_params)
189
- Log.log.debug("auth=#{a_params}")
190
+ Log.log.debug{"auth=#{a_params}"}
190
191
  # replace default values
191
192
  @gparams = DEFAULT_CREATE_PARAMS.deep_merge(a_params)
192
193
  # check that type is known
193
194
  self.class.token_creator(@gparams[:crtype])
194
195
  # specific parameters for the creation type
195
- @sparams=@gparams[@gparams[:crtype]]
196
- if @gparams[:crtype].eql?(:web) && @sparams.has_key?(:redirect_uri)
196
+ @sparams = @gparams[@gparams[:crtype]]
197
+ if @gparams[:crtype].eql?(:web) && @sparams.key?(:redirect_uri)
197
198
  uri = URI.parse(@sparams[:redirect_uri])
198
199
  raise 'redirect_uri scheme must be http or https' unless %w[http https].include?(uri.scheme)
199
200
  raise 'redirect_uri must have a port' if uri.port.nil?
@@ -203,14 +204,14 @@ module Aspera
203
204
  base_url: @gparams[:base_url],
204
205
  redirect_max: 2
205
206
  }
206
- rest_params[:auth] = a_params[:auth] if a_params.has_key?(:auth)
207
+ rest_params[:auth] = a_params[:auth] if a_params.key?(:auth)
207
208
  @api = Rest.new(rest_params)
208
209
  # if needed use from api
209
210
  @gparams.delete(:base_url)
210
211
  @gparams.delete(:auth)
211
212
  @gparams.delete(@gparams[:crtype])
212
- Log.dump(:gparams,@gparams)
213
- Log.dump(:sparams,@sparams)
213
+ Log.dump(:gparams, @gparams)
214
+ Log.dump(:sparams, @sparams)
214
215
  end
215
216
 
216
217
  public
@@ -226,7 +227,7 @@ module Aspera
226
227
 
227
228
  # @return Hash with optional general parameters
228
229
  def optional_scope_client_id(add_secret: false)
229
- call_params={}
230
+ call_params = {}
230
231
  call_params[:scope] = @gparams[:scope] unless @gparams[:scope].nil?
231
232
  call_params[:client_id] = @gparams[:client_id] unless @gparams[:client_id].nil?
232
233
  call_params[:client_secret] = @gparams[:client_secret] if add_secret && !@gparams[:client_id].nil?
@@ -254,7 +255,7 @@ module Aspera
254
255
  # `direct` agent is equipped with refresh code
255
256
  if !use_refresh_token && !token_data.nil?
256
257
  decoded_token = self.class.decode_token(token_data[@gparams[:token_field]])
257
- Log.dump('decoded_token',decoded_token) unless decoded_token.nil?
258
+ Log.dump('decoded_token', decoded_token) unless decoded_token.nil?
258
259
  if decoded_token.is_a?(Hash)
259
260
  expires_at_sec =
260
261
  if decoded_token['expires_at'].is_a?(String) then DateTime.parse(decoded_token['expires_at']).to_time
@@ -262,13 +263,13 @@ module Aspera
262
263
  end
263
264
  # force refresh if we see a token too close from expiration
264
265
  use_refresh_token = true if expires_at_sec.is_a?(Time) && (expires_at_sec - Time.now) < TOKEN_EXPIRATION_GUARD_SEC
265
- Log.log.debug("Expiration: #{expires_at_sec} / #{use_refresh_token}")
266
+ Log.log.debug{"Expiration: #{expires_at_sec} / #{use_refresh_token}"}
266
267
  end
267
268
  end
268
269
 
269
270
  # an API was already called, but failed, we need to regenerate or refresh
270
271
  if use_refresh_token
271
- if token_data.is_a?(Hash) && token_data.has_key?('refresh_token')
272
+ if token_data.is_a?(Hash) && token_data.key?('refresh_token')
272
273
  # save possible refresh token, before deleting the cache
273
274
  refresh_token = token_data['refresh_token']
274
275
  end
@@ -277,17 +278,17 @@ module Aspera
277
278
  token_data = nil
278
279
  # lets try the existing refresh token
279
280
  if !refresh_token.nil?
280
- Log.log.info("refresh=[#{refresh_token}]".bg_green)
281
+ Log.log.info{"refresh=[#{refresh_token}]".bg_green}
281
282
  # try to refresh
282
283
  # note: AoC admin token has no refresh, and lives by default 1800secs
283
- resp = create_token(optional_scope_client_id.merge(grant_type: 'refresh_token',refresh_token: refresh_token))
284
+ resp = create_token(optional_scope_client_id.merge(grant_type: 'refresh_token', refresh_token: refresh_token))
284
285
  if resp[:http].code.start_with?('2')
285
286
  # save only if success
286
287
  json_data = resp[:http].body
287
288
  token_data = JSON.parse(json_data)
288
- self.class.persist_mgr.put(token_id,json_data)
289
+ self.class.persist_mgr.put(token_id, json_data)
289
290
  else
290
- Log.log.debug("refresh failed: #{resp[:http].body}".bg_red)
291
+ Log.log.debug{"refresh failed: #{resp[:http].body}".bg_red}
291
292
  end
292
293
  end
293
294
  end
@@ -297,9 +298,9 @@ module Aspera
297
298
  resp = self.class.token_creator(@gparams[:crtype]).call(self)
298
299
  json_data = resp[:http].body
299
300
  token_data = JSON.parse(json_data)
300
- self.class.persist_mgr.put(token_id,json_data)
301
+ self.class.persist_mgr.put(token_id, json_data)
301
302
  end # if ! in_cache
302
- raise "API error: No such field in answer: #{@gparams[:token_field]}" unless token_data.has_key?(@gparams[:token_field])
303
+ raise "API error: No such field in answer: #{@gparams[:token_field]}" unless token_data.key?(@gparams[:token_field])
303
304
  # ok we shall have a token here
304
305
  return 'Bearer ' + token_data[@gparams[:token_field]]
305
306
  end
@@ -12,30 +12,38 @@ module Aspera
12
12
  class OpenApplication
13
13
  include Singleton
14
14
  class << self
15
+ USER_INTERFACES = %i[text graphical].freeze
15
16
  # User Interfaces
16
- def user_interfaces; %i[text graphical]; end
17
+ def user_interfaces; USER_INTERFACES; end
17
18
 
18
19
  def default_gui_mode
19
- return :graphical if [Aspera::Environment::OS_WINDOWS,Aspera::Environment::OS_X].include?(Aspera::Environment.os)
20
+ return :graphical if [Aspera::Environment::OS_WINDOWS, Aspera::Environment::OS_X].include?(Aspera::Environment.os)
20
21
  # unix family
21
- return :graphical if ENV.has_key?('DISPLAY') && !ENV['DISPLAY'].empty?
22
+ return :graphical if ENV.key?('DISPLAY') && !ENV['DISPLAY'].empty?
22
23
  return :text
23
24
  end
24
25
 
25
26
  # command must be non blocking
26
27
  def uri_graphical(uri)
27
28
  case Aspera::Environment.os
28
- when Aspera::Environment::OS_X
29
- return system('open',uri.to_s)
30
- when Aspera::Environment::OS_WINDOWS
31
- return system('start explorer "' + uri.to_s + '"')
32
- when Aspera::Environment::OS_LINUX
33
- return system("xdg-open '#{uri}'")
29
+ when Aspera::Environment::OS_X then return system('open', uri.to_s)
30
+ when Aspera::Environment::OS_WINDOWS then return system('start', 'explorer', '"' + uri.to_s + '"')
31
+ when Aspera::Environment::OS_LINUX then return system('xdg-open', uri.to_s)
34
32
  else
35
33
  raise "no graphical open method for #{Aspera::Environment.os}"
36
34
  end
37
35
  end
38
- end
36
+
37
+ def editor(file_path)
38
+ if ENV.key?('EDITOR')
39
+ system(ENV['EDITOR'], file_path.to_s)
40
+ elsif Aspera::Environment.os.eql?(Aspera::Environment::OS_WINDOWS)
41
+ system('notepad.exe', '"' + file_path.to_s + '"')
42
+ else
43
+ uri_graphical(file_path.to_s)
44
+ end
45
+ end
46
+ end # self
39
47
 
40
48
  attr_accessor :url_method
41
49
 
@@ -56,7 +64,7 @@ module Aspera
56
64
  puts "USER ACTION: open this:\n" + the_url.to_s.red + "\n"
57
65
  end
58
66
  else
59
- raise StandardError,"unsupported url open method: #{@url_method}"
67
+ raise StandardError, "unsupported url open method: #{@url_method}"
60
68
  end
61
69
  end
62
70
  end # OpenApplication
@@ -14,7 +14,7 @@ module Aspera
14
14
  # @param :format Optional dump method (default to JSON)
15
15
  # @param :merge Optional merge data from file to current data
16
16
  def initialize(options)
17
- Log.log.debug("persistency: #{options}")
17
+ Log.log.debug{"persistency: #{options}"}
18
18
  raise 'options shall be Hash' unless options.is_a?(Hash)
19
19
  raise 'mandatory :manager' if options[:manager].nil?
20
20
  raise 'mandatory :data' if options[:data].nil?
@@ -27,16 +27,16 @@ module Aspera
27
27
  @delete_condition = options[:delete] || lambda{|d|d.empty?}
28
28
  @persist_format = options[:format] || lambda {|h| JSON.generate(h)}
29
29
  persist_parse = options[:parse] || lambda {|t| JSON.parse(t)}
30
- persist_merge = options[:merge] || lambda {|current,file| current.concat(file).uniq rescue current}
30
+ persist_merge = options[:merge] || lambda {|current, file| current.concat(file).uniq rescue current}
31
31
  value = @manager.get(@object_id)
32
- persist_merge.call(@persisted_object,persist_parse.call(value)) unless value.nil?
32
+ persist_merge.call(@persisted_object, persist_parse.call(value)) unless value.nil?
33
33
  end
34
34
 
35
35
  def save
36
36
  if @delete_condition.call(@persisted_object)
37
37
  @manager.delete(@object_id)
38
38
  else
39
- @manager.put(@object_id,@persist_format.call(@persisted_object))
39
+ @manager.put(@object_id, @persist_format.call(@persisted_object))
40
40
  end
41
41
  end
42
42
 
@@ -14,17 +14,17 @@ module Aspera
14
14
  def initialize(folder)
15
15
  @cache = {}
16
16
  @folder = folder
17
- Log.log.debug("persistency folder: #{@folder}")
17
+ Log.log.debug{"persistency folder: #{@folder}"}
18
18
  end
19
19
 
20
20
  # @return String or nil string on existing persist, else nil
21
21
  def get(object_id)
22
- Log.log.debug("persistency get: #{object_id}")
23
- if @cache.has_key?(object_id)
22
+ Log.log.debug{"persistency get: #{object_id}"}
23
+ if @cache.key?(object_id)
24
24
  Log.log.debug('got from memory cache')
25
25
  else
26
26
  persist_filepath = id_to_filepath(object_id)
27
- Log.log.debug("persistency = #{persist_filepath}")
27
+ Log.log.debug{"persistency = #{persist_filepath}"}
28
28
  if File.exist?(persist_filepath)
29
29
  Log.log.debug('got from file cache')
30
30
  @cache[object_id] = File.read(persist_filepath)
@@ -33,32 +33,32 @@ module Aspera
33
33
  return @cache[object_id]
34
34
  end
35
35
 
36
- def put(object_id,value)
36
+ def put(object_id, value)
37
37
  raise 'value: only String supported' unless value.is_a?(String)
38
38
  persist_filepath = id_to_filepath(object_id)
39
- Log.log.debug("persistency saving: #{persist_filepath}")
39
+ Log.log.debug{"persistency saving: #{persist_filepath}"}
40
40
  File.delete(persist_filepath) if File.exist?(persist_filepath)
41
- File.write(persist_filepath,value)
41
+ File.write(persist_filepath, value)
42
42
  Environment.restrict_file_access(persist_filepath)
43
43
  @cache[object_id] = value
44
44
  end
45
45
 
46
46
  def delete(object_id)
47
47
  persist_filepath = id_to_filepath(object_id)
48
- Log.log.debug("persistency deleting: #{persist_filepath}")
48
+ Log.log.debug{"persistency deleting: #{persist_filepath}"}
49
49
  File.delete(persist_filepath) if File.exist?(persist_filepath)
50
50
  @cache.delete(object_id)
51
51
  end
52
52
 
53
- def garbage_collect(persist_category,max_age_seconds=nil)
54
- garbage_files = Dir[File.join(@folder,persist_category + '*' + FILE_SUFFIX)]
53
+ def garbage_collect(persist_category, max_age_seconds=nil)
54
+ garbage_files = Dir[File.join(@folder, persist_category + '*' + FILE_SUFFIX)]
55
55
  if !max_age_seconds.nil?
56
56
  current_time = Time.now
57
57
  garbage_files.select! { |filepath| (current_time - File.stat(filepath).mtime).to_i > max_age_seconds}
58
58
  end
59
59
  garbage_files.each do |filepath|
60
60
  File.delete(filepath)
61
- Log.log.debug("persistency deleted expired: #{filepath}")
61
+ Log.log.debug{"persistency deleted expired: #{filepath}"}
62
62
  end
63
63
  return garbage_files
64
64
  end
@@ -70,8 +70,8 @@ module Aspera
70
70
  raise 'object_id: only String supported' unless object_id.is_a?(String)
71
71
  FileUtils.mkdir_p(@folder)
72
72
  Environment.restrict_file_access(@folder)
73
- return File.join(@folder,"#{object_id}#{FILE_SUFFIX}")
74
- #.gsub(/[^a-z]+/,FILE_FIELD_SEPARATOR)
73
+ return File.join(@folder, "#{object_id}#{FILE_SUFFIX}")
74
+ # .gsub(/[^a-z]+/,FILE_FIELD_SEPARATOR)
75
75
  end
76
76
  end # PersistencyFolder
77
77
  end # Aspera