aspera-cli 4.10.0 → 4.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +19 -0
  4. data/CHANGELOG.md +528 -0
  5. data/CONTRIBUTING.md +143 -0
  6. data/README.md +977 -589
  7. data/bin/ascli +4 -4
  8. data/bin/asession +12 -12
  9. data/docs/test_env.conf +29 -19
  10. data/examples/aoc.rb +6 -6
  11. data/examples/dascli +18 -16
  12. data/examples/faspex4.rb +15 -15
  13. data/examples/node.rb +12 -12
  14. data/examples/proxy.pac +2 -2
  15. data/examples/server.rb +12 -12
  16. data/lib/aspera/aoc.rb +344 -272
  17. data/lib/aspera/ascmd.rb +56 -54
  18. data/lib/aspera/ats_api.rb +4 -4
  19. data/lib/aspera/cli/basic_auth_plugin.rb +15 -12
  20. data/lib/aspera/cli/extended_value.rb +9 -9
  21. data/lib/aspera/cli/{formater.rb → formatter.rb} +69 -69
  22. data/lib/aspera/cli/listener/line_dump.rb +1 -1
  23. data/lib/aspera/cli/listener/logger.rb +1 -1
  24. data/lib/aspera/cli/listener/progress.rb +5 -6
  25. data/lib/aspera/cli/listener/progress_multi.rb +16 -21
  26. data/lib/aspera/cli/main.rb +72 -73
  27. data/lib/aspera/cli/manager.rb +112 -112
  28. data/lib/aspera/cli/plugin.rb +68 -48
  29. data/lib/aspera/cli/plugins/alee.rb +4 -4
  30. data/lib/aspera/cli/plugins/aoc.rb +322 -720
  31. data/lib/aspera/cli/plugins/ats.rb +50 -52
  32. data/lib/aspera/cli/plugins/bss.rb +10 -10
  33. data/lib/aspera/cli/plugins/config.rb +514 -410
  34. data/lib/aspera/cli/plugins/console.rb +12 -12
  35. data/lib/aspera/cli/plugins/cos.rb +18 -20
  36. data/lib/aspera/cli/plugins/faspex.rb +134 -136
  37. data/lib/aspera/cli/plugins/faspex5.rb +235 -70
  38. data/lib/aspera/cli/plugins/node.rb +378 -309
  39. data/lib/aspera/cli/plugins/orchestrator.rb +52 -49
  40. data/lib/aspera/cli/plugins/preview.rb +129 -120
  41. data/lib/aspera/cli/plugins/server.rb +137 -83
  42. data/lib/aspera/cli/plugins/shares.rb +77 -52
  43. data/lib/aspera/cli/plugins/sync.rb +13 -33
  44. data/lib/aspera/cli/transfer_agent.rb +61 -61
  45. data/lib/aspera/cli/version.rb +2 -1
  46. data/lib/aspera/colors.rb +3 -3
  47. data/lib/aspera/command_line_builder.rb +78 -74
  48. data/lib/aspera/cos_node.rb +31 -29
  49. data/lib/aspera/data_repository.rb +1 -1
  50. data/lib/aspera/environment.rb +30 -28
  51. data/lib/aspera/fasp/agent_base.rb +17 -15
  52. data/lib/aspera/fasp/agent_connect.rb +34 -32
  53. data/lib/aspera/fasp/agent_direct.rb +70 -73
  54. data/lib/aspera/fasp/agent_httpgw.rb +79 -74
  55. data/lib/aspera/fasp/agent_node.rb +26 -26
  56. data/lib/aspera/fasp/agent_trsdk.rb +20 -20
  57. data/lib/aspera/fasp/error.rb +3 -2
  58. data/lib/aspera/fasp/error_info.rb +11 -8
  59. data/lib/aspera/fasp/installation.rb +80 -80
  60. data/lib/aspera/fasp/listener.rb +2 -2
  61. data/lib/aspera/fasp/parameters.rb +103 -92
  62. data/lib/aspera/fasp/parameters.yaml +313 -214
  63. data/lib/aspera/fasp/resume_policy.rb +10 -10
  64. data/lib/aspera/fasp/transfer_spec.rb +22 -2
  65. data/lib/aspera/fasp/uri.rb +7 -7
  66. data/lib/aspera/faspex_gw.rb +80 -159
  67. data/lib/aspera/faspex_postproc.rb +77 -0
  68. data/lib/aspera/hash_ext.rb +3 -3
  69. data/lib/aspera/id_generator.rb +5 -5
  70. data/lib/aspera/keychain/encrypted_hash.rb +23 -28
  71. data/lib/aspera/keychain/macos_security.rb +21 -20
  72. data/lib/aspera/log.rb +13 -13
  73. data/lib/aspera/nagios.rb +24 -23
  74. data/lib/aspera/node.rb +217 -38
  75. data/lib/aspera/oauth.rb +78 -74
  76. data/lib/aspera/open_application.rb +19 -11
  77. data/lib/aspera/persistency_action_once.rb +4 -4
  78. data/lib/aspera/persistency_folder.rb +13 -13
  79. data/lib/aspera/preview/file_types.rb +8 -8
  80. data/lib/aspera/preview/generator.rb +67 -67
  81. data/lib/aspera/preview/utils.rb +27 -27
  82. data/lib/aspera/proxy_auto_config.js +63 -63
  83. data/lib/aspera/proxy_auto_config.rb +19 -19
  84. data/lib/aspera/rest.rb +65 -67
  85. data/lib/aspera/rest_call_error.rb +2 -1
  86. data/lib/aspera/rest_error_analyzer.rb +22 -21
  87. data/lib/aspera/rest_errors_aspera.rb +16 -16
  88. data/lib/aspera/secret_hider.rb +17 -14
  89. data/lib/aspera/ssh.rb +15 -14
  90. data/lib/aspera/sync.rb +177 -62
  91. data/lib/aspera/temp_file_manager.rb +2 -2
  92. data/lib/aspera/uri_reader.rb +4 -4
  93. data/lib/aspera/web_auth.rb +13 -64
  94. data/lib/aspera/web_server_simple.rb +76 -0
  95. data.tar.gz.sig +0 -0
  96. metadata +11 -6
  97. metadata.gz.sig +0 -0
@@ -10,10 +10,10 @@ module Aspera
10
10
  # keychain based on macOS keychain, using `security` cmmand line
11
11
  class Keychain
12
12
  DOMAINS = %i[user system common dynamic].freeze
13
- LIST_OPTIONS={
13
+ LIST_OPTIONS = {
14
14
  domain: :c
15
15
  }
16
- ADD_PASS_OPTIONS={
16
+ ADD_PASS_OPTIONS = {
17
17
  account: :a,
18
18
  creator: :c,
19
19
  type: :C,
@@ -32,7 +32,7 @@ module Aspera
32
32
  getpass: :g
33
33
  }.freeze
34
34
  class << self
35
- def execute(command,options=nil,supported=nil,lastopt=nil)
35
+ def execute(command, options=nil, supported=nil, lastopt=nil)
36
36
  url = options&.delete(:url)
37
37
  if !url.nil?
38
38
  uri = URI.parse(url)
@@ -40,20 +40,20 @@ module Aspera
40
40
  options[:protocol] = 'htps'
41
41
  raise 'host required in URL' if uri.host.nil?
42
42
  options[:server] = uri.host
43
- options[:path] = uri.path unless ['','/'].include?(uri.path)
43
+ options[:path] = uri.path unless ['', '/'].include?(uri.path)
44
44
  options[:port] = uri.port unless uri.port.eql?(443) && !url.include?(':443/')
45
45
  end
46
- cmd=['security',command]
47
- options&.each do |k,v|
48
- raise "unknown option: #{k}" unless supported.has_key?(k)
46
+ cmd = ['security', command]
47
+ options&.each do |k, v|
48
+ raise "unknown option: #{k}" unless supported.key?(k)
49
49
  next if v.nil?
50
50
  cmd.push("-#{supported[k]}")
51
51
  cmd.push(v.shellescape) unless v.empty?
52
52
  end
53
53
  cmd.push(lastopt) unless lastopt.nil?
54
- Log.log.debug("executing>>#{cmd.join(' ')}")
55
- result=%x(#{cmd.join(' ')} 2>&1)
56
- Log.log.debug("result>>[#{result}]")
54
+ Log.log.debug{"executing>>#{cmd.join(' ')}"}
55
+ result = %x(#{cmd.join(' ')} 2>&1)
56
+ Log.log.debug{"result>>[#{result}]"}
57
57
  return result
58
58
  end
59
59
 
@@ -70,8 +70,8 @@ module Aspera
70
70
  end
71
71
 
72
72
  def list(options={})
73
- raise ArgumentError,"Invalid domain #{options[:domain]}, expected one of: #{DOMAINS}" unless options[:domain].nil? || DOMAINS.include?(options[:domain])
74
- keychains(execute('list-keychains',options,LIST_OPTIONS))
73
+ raise ArgumentError, "Invalid domain #{options[:domain]}, expected one of: #{DOMAINS}" unless options[:domain].nil? || DOMAINS.include?(options[:domain])
74
+ keychains(execute('list-keychains', options, LIST_OPTIONS))
75
75
  end
76
76
 
77
77
  def by_name(name)
@@ -88,15 +88,15 @@ module Aspera
88
88
  [string].pack('H*').force_encoding('UTF-8')
89
89
  end
90
90
 
91
- def password(operation,passtype,options)
91
+ def password(operation, passtype, options)
92
92
  raise "wrong operation: #{operation}" unless %i[add find delete].include?(operation)
93
93
  raise "wrong passtype: #{passtype}" unless %i[generic internet].include?(passtype)
94
94
  raise 'options shall be Hash' unless options.is_a?(Hash)
95
- missing=(operation.eql?(:add) ? %i[account service password] : %i[label])-options.keys
95
+ missing = (operation.eql?(:add) ? %i[account service password] : %i[label]) - options.keys
96
96
  raise "missing options: #{missing}" unless missing.empty?
97
- options[:getpass]='' if operation.eql?(:find)
98
- output=self.class.execute("#{operation}-#{passtype}-password",options,ADD_PASS_OPTIONS,@path)
99
- raise output.gsub(/^.*: /,'') if output.start_with?('security: ')
97
+ options[:getpass] = '' if operation.eql?(:find)
98
+ output = self.class.execute("#{operation}-#{passtype}-password", options, ADD_PASS_OPTIONS, @path)
99
+ raise output.gsub(/^.*: /, '') if output.start_with?('security: ')
100
100
  return nil unless operation.eql?(:find)
101
101
  attributes = {}
102
102
  output.split("\n").each do |line|
@@ -121,7 +121,7 @@ module Aspera
121
121
  end
122
122
 
123
123
  class MacosSystem
124
- def initialize(name=nil,password=nil)
124
+ def initialize(name=nil, _password=nil)
125
125
  @keychain = name.nil? ? MacosSecurity::Keychain.default_keychain : MacosSecurity::Keychain.by_name(name)
126
126
  raise "no such keychain #{name}" if @keychain.nil?
127
127
  end
@@ -130,7 +130,8 @@ module Aspera
130
130
  raise 'options shall be Hash' unless options.is_a?(Hash)
131
131
  unsupported = options.keys - %i[label username password url description]
132
132
  raise "unsupported options: #{unsupported}" unless unsupported.empty?
133
- @keychain.password(:add,:generic, service: options[:label],
133
+ @keychain.password(
134
+ :add, :generic, service: options[:label],
134
135
  account: options[:username] || 'none', password: options[:password], comment: options[:description])
135
136
  end
136
137
 
@@ -138,7 +139,7 @@ module Aspera
138
139
  raise 'options shall be Hash' unless options.is_a?(Hash)
139
140
  unsupported = options.keys - %i[label]
140
141
  raise "unsupported options: #{unsupported}" unless unsupported.empty?
141
- info = @keychain.password(:find,:generic,label: options[:label])
142
+ info = @keychain.password(:find, :generic, label: options[:label])
142
143
  raise 'not found' if info.nil?
143
144
  result = options.clone
144
145
  result[:secret] = info['password']
data/lib/aspera/log.rb CHANGED
@@ -11,28 +11,27 @@ module Aspera
11
11
  # Singleton object for logging
12
12
  class Log
13
13
  include Singleton
14
+ # where logs are sent to
15
+ LOG_TYPES = %i[stderr stdout syslog].freeze
14
16
  # class methods
15
17
  class << self
16
18
  # levels are :debug,:info,:warn,:error,fatal,:unknown
17
- def levels; Logger::Severity.constants.sort{|a,b|Logger::Severity.const_get(a) <=> Logger::Severity.const_get(b)}.map{|c|c.downcase.to_sym};end
18
-
19
- # where logs are sent to
20
- def logtypes; %i[stderr stdout syslog];end
19
+ def levels; Logger::Severity.constants.sort{|a, b|Logger::Severity.const_get(a) <=> Logger::Severity.const_get(b)}.map{|c|c.downcase.to_sym}; end
21
20
 
22
21
  # get the logger object of singleton
23
- def log; instance.logger;end
22
+ def log; instance.logger; end
24
23
 
25
24
  # dump object in debug mode
26
25
  # @param name string or symbol
27
26
  # @param format either pp or json format
28
- def dump(name,object,format=:json)
27
+ def dump(name, object, format=:json)
29
28
  log.debug do
30
29
  result =
31
30
  case format
32
31
  when :json
33
- JSON.pretty_generate(object) rescue PP.pp(object,+'')
32
+ JSON.pretty_generate(object) rescue PP.pp(object, +'')
34
33
  when :ruby
35
- PP.pp(object,+'')
34
+ PP.pp(object, +'')
36
35
  else
37
36
  raise 'wrong parameter, expect pp or json'
38
37
  end
@@ -68,12 +67,13 @@ module Aspera
68
67
  end
69
68
 
70
69
  # change underlying logger, but keep log level
71
- def logger_type=(new_logtype)
70
+ def logger_type=(new_log_type)
72
71
  current_severity_integer = @logger.level unless @logger.nil?
73
- current_severity_integer = ENV['AS_LOG_LEVEL'] if current_severity_integer.nil? && ENV.has_key?('AS_LOG_LEVEL')
72
+ current_severity_integer = ENV['AS_LOG_LEVEL'] if current_severity_integer.nil? && ENV.key?('AS_LOG_LEVEL')
74
73
  current_severity_integer = Logger::Severity::WARN if current_severity_integer.nil?
75
- case new_logtype
74
+ case new_log_type
76
75
  when :stderr
76
+ # typed: Logger
77
77
  @logger = Logger.new($stderr)
78
78
  when :stdout
79
79
  @logger = Logger.new($stdout)
@@ -81,10 +81,10 @@ module Aspera
81
81
  require 'syslog/logger'
82
82
  @logger = Syslog::Logger.new(@program_name, Syslog::LOG_LOCAL2)
83
83
  else
84
- raise "unknown log type: #{new_logtype.class} #{new_logtype}"
84
+ raise "unknown log type: #{new_log_type.class} #{new_log_type}"
85
85
  end
86
86
  @logger.level = current_severity_integer
87
- @logger_type = new_logtype
87
+ @logger_type = new_log_type
88
88
  # update formatter with password hiding
89
89
  @logger.formatter = SecretHider.log_formatter(@logger.formatter)
90
90
  end
data/lib/aspera/nagios.rb CHANGED
@@ -10,32 +10,32 @@ module Aspera
10
10
  # date offset levels
11
11
  DATE_WARN_OFFSET = 2
12
12
  DATE_CRIT_OFFSET = 5
13
- private_constant :LEVELS,:ADD_PREFIX,:DATE_WARN_OFFSET,:DATE_CRIT_OFFSET
13
+ private_constant :LEVELS, :ADD_PREFIX, :DATE_WARN_OFFSET, :DATE_CRIT_OFFSET
14
14
 
15
15
  # add methods to add nagios error levels, each take component name and message
16
16
  LEVELS.each_index do |code|
17
17
  name = "#{ADD_PREFIX}#{LEVELS[code]}".to_sym
18
- define_method(name){|comp,msg|@data.push({code: code,comp: comp,msg: msg})}
18
+ define_method(name){|comp, msg|@data.push({code: code, comp: comp, msg: msg})}
19
19
  end
20
20
 
21
21
  class << self
22
22
  # process results of a analysis and display status and exit with code
23
23
  def process(data)
24
24
  raise 'INTERNAL ERROR, result must be list and not empty' unless data.is_a?(Array) && !data.empty?
25
- %w[status component message].each{|c|raise "INTERNAL ERROR, result must have #{c}" unless data.first.has_key?(c)}
25
+ %w[status component message].each{|c|raise "INTERNAL ERROR, result must have #{c}" unless data.first.key?(c)}
26
26
  res_errors = data.reject{|s|s['status'].eql?('ok')}
27
27
  # keep only errors in case of problem, other ok are assumed so
28
28
  data = res_errors unless res_errors.empty?
29
29
  # first is most critical
30
- data.sort!{|a,b|LEVELS.index(a['status'].to_sym) <=> LEVELS.index(b['status'].to_sym)}
30
+ data.sort!{|a, b|LEVELS.index(a['status'].to_sym) <=> LEVELS.index(b['status'].to_sym)}
31
31
  # build message: if multiple components: concatenate
32
- #message = data.map{|i|"#{i['component']}:#{i['message']}"}.join(', ').gsub("\n",' ')
33
- message = data.
34
- map{|i|i['component']}.
35
- uniq.
36
- map{|comp|comp + ':' + data.select{|d|d['component'].eql?(comp)}.map{|d|d['message']}.join(',')}.
37
- join(', ').
38
- tr("\n",' ')
32
+ # message = data.map{|i|"#{i['component']}:#{i['message']}"}.join(', ').gsub("\n",' ')
33
+ message = data
34
+ .map{|i|i['component']}
35
+ .uniq
36
+ .map{|comp|comp + ':' + data.select{|d|d['component'].eql?(comp)}.map{|d|d['message']}.join(',')}
37
+ .join(', ')
38
+ .tr("\n", ' ')
39
39
  status = data.first['status'].upcase
40
40
  # display status for nagios
41
41
  puts("#{status} - [#{message}]\n")
@@ -45,36 +45,37 @@ module Aspera
45
45
  end
46
46
 
47
47
  attr_reader :data
48
+
48
49
  def initialize
49
50
  @data = []
50
51
  end
51
52
 
52
- # comparte remote time with local time
53
+ # compare remote time with local time
53
54
  def check_time_offset(remote_date, component)
54
55
  # check date if specified : 2015-10-13T07:32:01Z
55
- rtime = DateTime.strptime(remote_date)
56
- diff_time = (rtime - DateTime.now).abs
57
- diff_disp = diff_time.round(-2)
58
- Log.log.debug("DATE: #{remote_date} #{rtime} diff=#{diff_disp}")
59
- msg = "offset #{diff_disp} sec"
56
+ remote_time = DateTime.strptime(remote_date)
57
+ diff_time = (remote_time - DateTime.now).abs
58
+ diff_rounded = diff_time.round(-2)
59
+ Log.log.debug{"DATE: #{remote_date} #{remote_time} diff=#{diff_rounded}"}
60
+ msg = "offset #{diff_rounded} sec"
60
61
  if diff_time >= DATE_CRIT_OFFSET
61
- add_critical(component,msg)
62
+ add_critical(component, msg)
62
63
  elsif diff_time >= DATE_WARN_OFFSET
63
- add_warning(component,msg)
64
+ add_warning(component, msg)
64
65
  else
65
- add_ok(component,msg)
66
+ add_ok(component, msg)
66
67
  end
67
68
  end
68
69
 
69
70
  def check_product_version(component, _product, version)
70
- add_ok(component,"version #{version}")
71
- # TODO check on database if latest version
71
+ add_ok(component, "version #{version}")
72
+ # TODO: check on database if latest version
72
73
  end
73
74
 
74
75
  # translate for display
75
76
  def result
76
77
  raise 'missing result' if @data.empty?
77
- {type: :object_list,data: @data.map{|i|{'status' => LEVELS[i[:code]].to_s,'component' => i[:comp],'message' => i[:msg]}}}
78
+ {type: :object_list, data: @data.map{|i|{'status' => LEVELS[i[:code]].to_s, 'component' => i[:comp], 'message' => i[:msg]}}}
78
79
  end
79
80
  end
80
81
  end
data/lib/aspera/node.rb CHANGED
@@ -9,27 +9,28 @@ require 'zlib'
9
9
  require 'base64'
10
10
 
11
11
  module Aspera
12
- # Provides additional functions using node API.
13
- class Node < Rest
12
+ # Provides additional functions using node API with gen4 extensions (access keys)
13
+ class Node < Aspera::Rest
14
14
  # permissions
15
15
  ACCESS_LEVELS = %w[delete list mkdir preview read rename write].freeze
16
16
  # prefix for ruby code for filter
17
17
  MATCH_EXEC_PREFIX = 'exec:'
18
+ HEADER_X_ASPERA_ACCESS_KEY = 'X-Aspera-AccessKey'
19
+ PATH_SEPARATOR = '/'
18
20
 
19
21
  # register node special token decoder
20
22
  Oauth.register_decoder(lambda{|token|JSON.parse(Zlib::Inflate.inflate(Base64.decode64(token)).partition('==SIGNATURE==').first)})
21
23
 
24
+ # class instance variable, access with accessors on class
25
+ @use_standard_ports = true
26
+
22
27
  class << self
23
- def set_ak_basic_token(ts,ak,secret)
24
- Log.log.warn("Expected transfer user: #{Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER}, "\
25
- "but have #{ts['remote_user']}") unless ts['remote_user'].eql?(Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER)
26
- ts['token'] = Rest.basic_creds(ak,secret)
27
- end
28
+ attr_accessor :use_standard_ports
28
29
 
29
30
  # for access keys: provide expression to match entry in folder
30
31
  # if no prefix: regex
31
32
  # if prefix: ruby code
32
- # if filder is nil, then always match
33
+ # if expression is nil, then always match
33
34
  def file_matcher(match_expression)
34
35
  match_expression ||= "#{MATCH_EXEC_PREFIX}true"
35
36
  if match_expression.start_with?(MATCH_EXEC_PREFIX)
@@ -39,49 +40,227 @@ module Aspera
39
40
  end
40
41
  end
41
42
 
42
- # def initialize(rest_params)
43
- # super(rest_params)
44
- # end
45
-
46
- # recursively crawl in a folder.
47
- # subfolders a processed if the processing method returns true
48
- # @param processor must provide a method to process each entry
49
- # @param opt options
50
- # - top_file_id file id to start at (default = access key root file id)
51
- # - top_file_path path of top folder (default = /)
52
- # - method processing method (default= process_entry)
53
- def crawl(processor,opt={})
54
- Log.log.debug("crawl1 #{opt}")
55
- # not possible with bearer token
56
- opt[:top_file_id] ||= read('access_keys/self')[:data]['root_file_id']
57
- opt[:top_file_path] ||= '/'
58
- opt[:method] ||= :process_entry
59
- raise "processor must have #{opt[:method]}" unless processor.respond_to?(opt[:method])
60
- Log.log.debug("crawl #{opt}")
61
- #top_info=read("files/#{opt[:top_file_id]}")[:data]
62
- folders_to_explore = [{id: opt[:top_file_id], relpath: opt[:top_file_path]}]
63
- Log.dump(:folders_to_explore,folders_to_explore)
64
- while !folders_to_explore.empty?
43
+ REQUIRED_APP_INFO_FIELDS = %i[node_info app api workspace_info].freeze
44
+ REQUIRED_APP_API_METHODS = %i[node_api_from add_ts_tags].freeze
45
+ private_constant :REQUIRED_APP_INFO_FIELDS, :REQUIRED_APP_API_METHODS
46
+
47
+ attr_reader :app_info
48
+
49
+ # @param params [Hash] Rest parameters
50
+ # @param app_info [Hash,NilClass] special processing for AoC
51
+ def initialize(params:, app_info: nil, add_tspec: nil)
52
+ super(params)
53
+ @app_info = app_info
54
+ # this is added to transfer spec, for instance to add tags (COS)
55
+ @add_tspec = add_tspec
56
+ if !@app_info.nil?
57
+ REQUIRED_APP_INFO_FIELDS.each do |field|
58
+ raise "INTERNAL ERROR: app_info lacks field #{field}" unless @app_info.key?(field)
59
+ end
60
+ REQUIRED_APP_API_METHODS.each do |method|
61
+ raise "INTERNAL ERROR: #{@app_info[:api].class} lacks method #{method}" unless @app_info[:api].respond_to?(method)
62
+ end
63
+ end
64
+ end
65
+
66
+ # update transfer spec with special additional tags
67
+ def add_tspec_info(tspec)
68
+ tspec.deep_merge!(@add_tspec) unless @add_tspec.nil?
69
+ return tspec
70
+ end
71
+
72
+ # @returns [Aspera::Node] a Node or nil
73
+ def node_id_to_node(node_id)
74
+ return self if !@app_info.nil? && @app_info[:node_info]['id'].eql?(node_id)
75
+ return @app_info[:api].node_api_from(node_id: node_id, workspace_info: @app_info[workspace_info]) unless @app_info.nil?
76
+ Log.log.warn{"cannot resolve link with node id #{node_id}"}
77
+ return nil
78
+ end
79
+
80
+ # recursively browse in a folder (with non-recursive method)
81
+ # sub folders are processed if the processing method returns true
82
+ # @param state [Object] state object sent to processing method
83
+ # @param method [Symbol] processing method name
84
+ # @param top_file_id [String] file id to start at (default = access key root file id)
85
+ # @param top_file_path [String] path of top folder (default = /)
86
+ def process_folder_tree(state:, method:, top_file_id:, top_file_path: '/')
87
+ raise 'INTERNAL ERROR: top_file_path not set' if top_file_path.nil?
88
+ raise "INTERNAL ERROR: Missing method #{method}" unless respond_to?(method)
89
+ folders_to_explore = [{id: top_file_id, path: top_file_path}]
90
+ Log.dump(:folders_to_explore, folders_to_explore)
91
+ until folders_to_explore.empty?
65
92
  current_item = folders_to_explore.shift
66
- Log.log.debug("searching #{current_item[:relpath]}".bg_green)
93
+ Log.log.debug{"searching #{current_item[:path]}".bg_green}
67
94
  # get folder content
68
95
  folder_contents =
69
96
  begin
70
97
  read("files/#{current_item[:id]}/files")[:data]
71
98
  rescue StandardError => e
72
- Log.log.warn("#{current_item[:relpath]}: #{e.class} #{e.message}")
99
+ Log.log.warn{"#{current_item[:path]}: #{e.class} #{e.message}"}
73
100
  []
74
101
  end
75
- Log.dump(:folder_contents,folder_contents)
102
+ Log.dump(:folder_contents, folder_contents)
76
103
  folder_contents.each do |entry|
77
- relative_path = File.join(current_item[:relpath],entry['name'])
78
- Log.log.debug("looking #{relative_path}".bg_green)
104
+ relative_path = File.join(current_item[:path], entry['name'])
105
+ Log.log.debug{"looking #{relative_path}".bg_green}
106
+ # continue only if method returns true
107
+ next unless send(method, entry, relative_path, state)
79
108
  # entry type is file, folder or link
80
- if processor.send(opt[:method],entry,relative_path) && entry['type'].eql?('folder')
81
- folders_to_explore.push({id: entry['id'],relpath: relative_path})
109
+ case entry['type']
110
+ when 'folder'
111
+ folders_to_explore.push({id: entry['id'], path: relative_path})
112
+ when 'link'
113
+ node_id_to_node(entry['target_node_id'])&.process_folder_tree(
114
+ state: state,
115
+ method: method,
116
+ top_file_id: entry['target_id'],
117
+ top_file_path: relative_path)
82
118
  end
83
119
  end
84
120
  end
121
+ end # process_folder_tree
122
+
123
+ # processing method to resolve a file path to id
124
+ # @returns true if processing need to continue
125
+ def process_resolve_node_path(entry, _path, state)
126
+ # stop digging here if not in right path
127
+ return false unless entry['name'].eql?(state[:path].first)
128
+ # ok it matches, so we remove the match
129
+ state[:path].shift
130
+ case entry['type']
131
+ when 'file'
132
+ # file must be terminal
133
+ raise "#{entry['name']} is a file, expecting folder to find: #{state[:path]}" unless state[:path].empty?
134
+ # it's terminal, we found it
135
+ state[:result] = {api: self, file_id: entry['id']}
136
+ return false
137
+ when 'folder'
138
+ if state[:path].empty?
139
+ # we found it
140
+ state[:result] = {api: self, file_id: entry['id']}
141
+ return false
142
+ end
143
+ when 'link'
144
+ if state[:path].empty?
145
+ # we found it
146
+ other_node = node_id_to_node(entry['target_node_id'])
147
+ raise 'cannot resolve link' if other_node.nil?
148
+ state[:result] = {api: other_node, file_id: entry['target_id']}
149
+ return false
150
+ end
151
+ else
152
+ Log.log.warn{"Unknown element type: #{entry['type']}"}
153
+ end
154
+ # continue to dig folder
155
+ return true
156
+ end
157
+
158
+ # Navigate the path from given file id
159
+ # @param top_file_id [String] id initial file id
160
+ # @param path [String] file path
161
+ # @return [Hash] {.api,.file_id}
162
+ def resolve_api_fid(top_file_id, path)
163
+ raise 'file id shall be String' unless top_file_id.is_a?(String)
164
+ path_elements = path.split(PATH_SEPARATOR).reject(&:empty?)
165
+ return {api: self, file_id: top_file_id} if path_elements.empty?
166
+ resolve_state = {path: path_elements, result: nil}
167
+ process_folder_tree(state: resolve_state, method: :process_resolve_node_path, top_file_id: top_file_id)
168
+ raise "entry not found: #{resolve_state[:path]}" if resolve_state[:result].nil?
169
+ return resolve_state[:result]
170
+ end
171
+
172
+ # add entry to list if test block is success
173
+ # @return [TrueClass,FalseClass]
174
+ def process_find_files(entry, path, state)
175
+ begin
176
+ # add to result if match filter
177
+ state[:found].push(entry.merge({'path' => path})) if state[:test_block].call(entry)
178
+ # process link
179
+ if entry[:type].eql?('link')
180
+ other_node = node_id_to_node(entry['target_node_id'])
181
+ other_node.process_folder_tree(state: state, method: process_find_files, top_file_id: entry['target_id'], top_file_path: path)
182
+ end
183
+ rescue StandardError => e
184
+ Log.log.error{"#{path}: #{e.message}"}
185
+ end
186
+ # process all folders
187
+ return true
188
+ end
189
+
190
+ def find_files(top_file_id, test_block)
191
+ Log.log.debug{"find_files: file id=#{top_file_id}"}
192
+ find_state = {found: [], test_block: test_block}
193
+ process_folder_tree(state: find_state, method: :process_find_files, top_file_id: top_file_id)
194
+ return find_state[:found]
195
+ end
196
+
197
+ def refreshed_transfer_token
198
+ return oauth_token(force_refresh: true)
199
+ end
200
+
201
+ # Create transfer spec for gen4
202
+ def transfer_spec_gen4(file_id, direction, ts_merge=nil)
203
+ ak_name = nil
204
+ ak_token = nil
205
+ case params[:auth][:type]
206
+ when :basic
207
+ ak_name = params[:auth][:username]
208
+ when :oauth2
209
+ ak_name = params[:headers][HEADER_X_ASPERA_ACCESS_KEY]
210
+ # TODO: token_generation_lambda = lambda{|do_refresh|oauth_token(force_refresh: do_refresh)}
211
+ # get bearer token, possibly use cache
212
+ ak_token = oauth_token(force_refresh: false)
213
+ else raise "Unsupported auth method for node gen4: #{params[:auth][:type]}"
214
+ end
215
+ transfer_spec = {
216
+ 'direction' => direction,
217
+ 'token' => ak_token,
218
+ 'tags' => {
219
+ 'aspera' => {
220
+ 'node' => {
221
+ 'access_key' => ak_name,
222
+ 'file_id' => file_id
223
+ } # node
224
+ } # aspera
225
+ } # tags
226
+ }
227
+ # add specials tags (cos)
228
+ add_tspec_info(transfer_spec)
229
+ transfer_spec.deep_merge!(ts_merge) unless ts_merge.nil?
230
+ # add application specific tags (AoC)
231
+ the_app = app_info
232
+ the_app[:api].add_ts_tags(transfer_spec: transfer_spec, app_info: the_app) unless the_app.nil?
233
+ # add basic token
234
+ if transfer_spec['token'].nil?
235
+ ts_basic_token(transfer_spec)
236
+ end
237
+ # add remote host info
238
+ if self.class.use_standard_ports
239
+ # get default TCP/UDP ports and transfer user
240
+ transfer_spec.merge!(Fasp::TransferSpec::AK_TSPEC_BASE)
241
+ # by default: same address as node API
242
+ transfer_spec['remote_host'] = URI.parse(params[:base_url]).host
243
+ if !@app_info.nil? && !@app_info[:node_info]['transfer_url'].nil? && !@app_info[:node_info]['transfer_url'].empty?
244
+ transfer_spec['remote_host'] = @app_info[:node_info]['transfer_url']
245
+ end
246
+ else
247
+ # retrieve values from API
248
+ std_t_spec = create(
249
+ 'files/download_setup',
250
+ {transfer_requests: [{ transfer_request: {paths: [{'source' => '/'}] } }] }
251
+ )[:data]['transfer_specs'].first['transfer_spec']
252
+ # copy some parts
253
+ %w[remote_host remote_user ssh_port fasp_port wss_enabled wss_port].each {|i| transfer_spec[i] = std_t_spec[i] if std_t_spec.key?(i)}
254
+ end
255
+ return transfer_spec
256
+ end
257
+
258
+ # set basic token in transfer spec
259
+ def ts_basic_token(ts)
260
+ Log.log.warn{"Expected transfer user: #{Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER}, but have #{ts['remote_user']}"} \
261
+ unless ts['remote_user'].eql?(Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER)
262
+ raise 'ERROR: no secret in node object' unless params[:auth][:password]
263
+ ts['token'] = Rest.basic_creds(params[:auth][:username], params[:auth][:password])
85
264
  end
86
265
  end
87
266
  end