aspera-cli 4.10.0 → 4.12.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (97) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +19 -0
  4. data/CHANGELOG.md +528 -0
  5. data/CONTRIBUTING.md +143 -0
  6. data/README.md +977 -589
  7. data/bin/ascli +4 -4
  8. data/bin/asession +12 -12
  9. data/docs/test_env.conf +29 -19
  10. data/examples/aoc.rb +6 -6
  11. data/examples/dascli +18 -16
  12. data/examples/faspex4.rb +15 -15
  13. data/examples/node.rb +12 -12
  14. data/examples/proxy.pac +2 -2
  15. data/examples/server.rb +12 -12
  16. data/lib/aspera/aoc.rb +344 -272
  17. data/lib/aspera/ascmd.rb +56 -54
  18. data/lib/aspera/ats_api.rb +4 -4
  19. data/lib/aspera/cli/basic_auth_plugin.rb +15 -12
  20. data/lib/aspera/cli/extended_value.rb +9 -9
  21. data/lib/aspera/cli/{formater.rb → formatter.rb} +69 -69
  22. data/lib/aspera/cli/listener/line_dump.rb +1 -1
  23. data/lib/aspera/cli/listener/logger.rb +1 -1
  24. data/lib/aspera/cli/listener/progress.rb +5 -6
  25. data/lib/aspera/cli/listener/progress_multi.rb +16 -21
  26. data/lib/aspera/cli/main.rb +72 -73
  27. data/lib/aspera/cli/manager.rb +112 -112
  28. data/lib/aspera/cli/plugin.rb +68 -48
  29. data/lib/aspera/cli/plugins/alee.rb +4 -4
  30. data/lib/aspera/cli/plugins/aoc.rb +322 -720
  31. data/lib/aspera/cli/plugins/ats.rb +50 -52
  32. data/lib/aspera/cli/plugins/bss.rb +10 -10
  33. data/lib/aspera/cli/plugins/config.rb +514 -410
  34. data/lib/aspera/cli/plugins/console.rb +12 -12
  35. data/lib/aspera/cli/plugins/cos.rb +18 -20
  36. data/lib/aspera/cli/plugins/faspex.rb +134 -136
  37. data/lib/aspera/cli/plugins/faspex5.rb +235 -70
  38. data/lib/aspera/cli/plugins/node.rb +378 -309
  39. data/lib/aspera/cli/plugins/orchestrator.rb +52 -49
  40. data/lib/aspera/cli/plugins/preview.rb +129 -120
  41. data/lib/aspera/cli/plugins/server.rb +137 -83
  42. data/lib/aspera/cli/plugins/shares.rb +77 -52
  43. data/lib/aspera/cli/plugins/sync.rb +13 -33
  44. data/lib/aspera/cli/transfer_agent.rb +61 -61
  45. data/lib/aspera/cli/version.rb +2 -1
  46. data/lib/aspera/colors.rb +3 -3
  47. data/lib/aspera/command_line_builder.rb +78 -74
  48. data/lib/aspera/cos_node.rb +31 -29
  49. data/lib/aspera/data_repository.rb +1 -1
  50. data/lib/aspera/environment.rb +30 -28
  51. data/lib/aspera/fasp/agent_base.rb +17 -15
  52. data/lib/aspera/fasp/agent_connect.rb +34 -32
  53. data/lib/aspera/fasp/agent_direct.rb +70 -73
  54. data/lib/aspera/fasp/agent_httpgw.rb +79 -74
  55. data/lib/aspera/fasp/agent_node.rb +26 -26
  56. data/lib/aspera/fasp/agent_trsdk.rb +20 -20
  57. data/lib/aspera/fasp/error.rb +3 -2
  58. data/lib/aspera/fasp/error_info.rb +11 -8
  59. data/lib/aspera/fasp/installation.rb +80 -80
  60. data/lib/aspera/fasp/listener.rb +2 -2
  61. data/lib/aspera/fasp/parameters.rb +103 -92
  62. data/lib/aspera/fasp/parameters.yaml +313 -214
  63. data/lib/aspera/fasp/resume_policy.rb +10 -10
  64. data/lib/aspera/fasp/transfer_spec.rb +22 -2
  65. data/lib/aspera/fasp/uri.rb +7 -7
  66. data/lib/aspera/faspex_gw.rb +80 -159
  67. data/lib/aspera/faspex_postproc.rb +77 -0
  68. data/lib/aspera/hash_ext.rb +3 -3
  69. data/lib/aspera/id_generator.rb +5 -5
  70. data/lib/aspera/keychain/encrypted_hash.rb +23 -28
  71. data/lib/aspera/keychain/macos_security.rb +21 -20
  72. data/lib/aspera/log.rb +13 -13
  73. data/lib/aspera/nagios.rb +24 -23
  74. data/lib/aspera/node.rb +217 -38
  75. data/lib/aspera/oauth.rb +78 -74
  76. data/lib/aspera/open_application.rb +19 -11
  77. data/lib/aspera/persistency_action_once.rb +4 -4
  78. data/lib/aspera/persistency_folder.rb +13 -13
  79. data/lib/aspera/preview/file_types.rb +8 -8
  80. data/lib/aspera/preview/generator.rb +67 -67
  81. data/lib/aspera/preview/utils.rb +27 -27
  82. data/lib/aspera/proxy_auto_config.js +63 -63
  83. data/lib/aspera/proxy_auto_config.rb +19 -19
  84. data/lib/aspera/rest.rb +65 -67
  85. data/lib/aspera/rest_call_error.rb +2 -1
  86. data/lib/aspera/rest_error_analyzer.rb +22 -21
  87. data/lib/aspera/rest_errors_aspera.rb +16 -16
  88. data/lib/aspera/secret_hider.rb +17 -14
  89. data/lib/aspera/ssh.rb +15 -14
  90. data/lib/aspera/sync.rb +177 -62
  91. data/lib/aspera/temp_file_manager.rb +2 -2
  92. data/lib/aspera/uri_reader.rb +4 -4
  93. data/lib/aspera/web_auth.rb +13 -64
  94. data/lib/aspera/web_server_simple.rb +76 -0
  95. data.tar.gz.sig +0 -0
  96. metadata +11 -6
  97. metadata.gz.sig +0 -0
@@ -10,10 +10,10 @@ module Aspera
10
10
  # keychain based on macOS keychain, using `security` cmmand line
11
11
  class Keychain
12
12
  DOMAINS = %i[user system common dynamic].freeze
13
- LIST_OPTIONS={
13
+ LIST_OPTIONS = {
14
14
  domain: :c
15
15
  }
16
- ADD_PASS_OPTIONS={
16
+ ADD_PASS_OPTIONS = {
17
17
  account: :a,
18
18
  creator: :c,
19
19
  type: :C,
@@ -32,7 +32,7 @@ module Aspera
32
32
  getpass: :g
33
33
  }.freeze
34
34
  class << self
35
- def execute(command,options=nil,supported=nil,lastopt=nil)
35
+ def execute(command, options=nil, supported=nil, lastopt=nil)
36
36
  url = options&.delete(:url)
37
37
  if !url.nil?
38
38
  uri = URI.parse(url)
@@ -40,20 +40,20 @@ module Aspera
40
40
  options[:protocol] = 'htps'
41
41
  raise 'host required in URL' if uri.host.nil?
42
42
  options[:server] = uri.host
43
- options[:path] = uri.path unless ['','/'].include?(uri.path)
43
+ options[:path] = uri.path unless ['', '/'].include?(uri.path)
44
44
  options[:port] = uri.port unless uri.port.eql?(443) && !url.include?(':443/')
45
45
  end
46
- cmd=['security',command]
47
- options&.each do |k,v|
48
- raise "unknown option: #{k}" unless supported.has_key?(k)
46
+ cmd = ['security', command]
47
+ options&.each do |k, v|
48
+ raise "unknown option: #{k}" unless supported.key?(k)
49
49
  next if v.nil?
50
50
  cmd.push("-#{supported[k]}")
51
51
  cmd.push(v.shellescape) unless v.empty?
52
52
  end
53
53
  cmd.push(lastopt) unless lastopt.nil?
54
- Log.log.debug("executing>>#{cmd.join(' ')}")
55
- result=%x(#{cmd.join(' ')} 2>&1)
56
- Log.log.debug("result>>[#{result}]")
54
+ Log.log.debug{"executing>>#{cmd.join(' ')}"}
55
+ result = %x(#{cmd.join(' ')} 2>&1)
56
+ Log.log.debug{"result>>[#{result}]"}
57
57
  return result
58
58
  end
59
59
 
@@ -70,8 +70,8 @@ module Aspera
70
70
  end
71
71
 
72
72
  def list(options={})
73
- raise ArgumentError,"Invalid domain #{options[:domain]}, expected one of: #{DOMAINS}" unless options[:domain].nil? || DOMAINS.include?(options[:domain])
74
- keychains(execute('list-keychains',options,LIST_OPTIONS))
73
+ raise ArgumentError, "Invalid domain #{options[:domain]}, expected one of: #{DOMAINS}" unless options[:domain].nil? || DOMAINS.include?(options[:domain])
74
+ keychains(execute('list-keychains', options, LIST_OPTIONS))
75
75
  end
76
76
 
77
77
  def by_name(name)
@@ -88,15 +88,15 @@ module Aspera
88
88
  [string].pack('H*').force_encoding('UTF-8')
89
89
  end
90
90
 
91
- def password(operation,passtype,options)
91
+ def password(operation, passtype, options)
92
92
  raise "wrong operation: #{operation}" unless %i[add find delete].include?(operation)
93
93
  raise "wrong passtype: #{passtype}" unless %i[generic internet].include?(passtype)
94
94
  raise 'options shall be Hash' unless options.is_a?(Hash)
95
- missing=(operation.eql?(:add) ? %i[account service password] : %i[label])-options.keys
95
+ missing = (operation.eql?(:add) ? %i[account service password] : %i[label]) - options.keys
96
96
  raise "missing options: #{missing}" unless missing.empty?
97
- options[:getpass]='' if operation.eql?(:find)
98
- output=self.class.execute("#{operation}-#{passtype}-password",options,ADD_PASS_OPTIONS,@path)
99
- raise output.gsub(/^.*: /,'') if output.start_with?('security: ')
97
+ options[:getpass] = '' if operation.eql?(:find)
98
+ output = self.class.execute("#{operation}-#{passtype}-password", options, ADD_PASS_OPTIONS, @path)
99
+ raise output.gsub(/^.*: /, '') if output.start_with?('security: ')
100
100
  return nil unless operation.eql?(:find)
101
101
  attributes = {}
102
102
  output.split("\n").each do |line|
@@ -121,7 +121,7 @@ module Aspera
121
121
  end
122
122
 
123
123
  class MacosSystem
124
- def initialize(name=nil,password=nil)
124
+ def initialize(name=nil, _password=nil)
125
125
  @keychain = name.nil? ? MacosSecurity::Keychain.default_keychain : MacosSecurity::Keychain.by_name(name)
126
126
  raise "no such keychain #{name}" if @keychain.nil?
127
127
  end
@@ -130,7 +130,8 @@ module Aspera
130
130
  raise 'options shall be Hash' unless options.is_a?(Hash)
131
131
  unsupported = options.keys - %i[label username password url description]
132
132
  raise "unsupported options: #{unsupported}" unless unsupported.empty?
133
- @keychain.password(:add,:generic, service: options[:label],
133
+ @keychain.password(
134
+ :add, :generic, service: options[:label],
134
135
  account: options[:username] || 'none', password: options[:password], comment: options[:description])
135
136
  end
136
137
 
@@ -138,7 +139,7 @@ module Aspera
138
139
  raise 'options shall be Hash' unless options.is_a?(Hash)
139
140
  unsupported = options.keys - %i[label]
140
141
  raise "unsupported options: #{unsupported}" unless unsupported.empty?
141
- info = @keychain.password(:find,:generic,label: options[:label])
142
+ info = @keychain.password(:find, :generic, label: options[:label])
142
143
  raise 'not found' if info.nil?
143
144
  result = options.clone
144
145
  result[:secret] = info['password']
data/lib/aspera/log.rb CHANGED
@@ -11,28 +11,27 @@ module Aspera
11
11
  # Singleton object for logging
12
12
  class Log
13
13
  include Singleton
14
+ # where logs are sent to
15
+ LOG_TYPES = %i[stderr stdout syslog].freeze
14
16
  # class methods
15
17
  class << self
16
18
  # levels are :debug,:info,:warn,:error,fatal,:unknown
17
- def levels; Logger::Severity.constants.sort{|a,b|Logger::Severity.const_get(a) <=> Logger::Severity.const_get(b)}.map{|c|c.downcase.to_sym};end
18
-
19
- # where logs are sent to
20
- def logtypes; %i[stderr stdout syslog];end
19
+ def levels; Logger::Severity.constants.sort{|a, b|Logger::Severity.const_get(a) <=> Logger::Severity.const_get(b)}.map{|c|c.downcase.to_sym}; end
21
20
 
22
21
  # get the logger object of singleton
23
- def log; instance.logger;end
22
+ def log; instance.logger; end
24
23
 
25
24
  # dump object in debug mode
26
25
  # @param name string or symbol
27
26
  # @param format either pp or json format
28
- def dump(name,object,format=:json)
27
+ def dump(name, object, format=:json)
29
28
  log.debug do
30
29
  result =
31
30
  case format
32
31
  when :json
33
- JSON.pretty_generate(object) rescue PP.pp(object,+'')
32
+ JSON.pretty_generate(object) rescue PP.pp(object, +'')
34
33
  when :ruby
35
- PP.pp(object,+'')
34
+ PP.pp(object, +'')
36
35
  else
37
36
  raise 'wrong parameter, expect pp or json'
38
37
  end
@@ -68,12 +67,13 @@ module Aspera
68
67
  end
69
68
 
70
69
  # change underlying logger, but keep log level
71
- def logger_type=(new_logtype)
70
+ def logger_type=(new_log_type)
72
71
  current_severity_integer = @logger.level unless @logger.nil?
73
- current_severity_integer = ENV['AS_LOG_LEVEL'] if current_severity_integer.nil? && ENV.has_key?('AS_LOG_LEVEL')
72
+ current_severity_integer = ENV['AS_LOG_LEVEL'] if current_severity_integer.nil? && ENV.key?('AS_LOG_LEVEL')
74
73
  current_severity_integer = Logger::Severity::WARN if current_severity_integer.nil?
75
- case new_logtype
74
+ case new_log_type
76
75
  when :stderr
76
+ # typed: Logger
77
77
  @logger = Logger.new($stderr)
78
78
  when :stdout
79
79
  @logger = Logger.new($stdout)
@@ -81,10 +81,10 @@ module Aspera
81
81
  require 'syslog/logger'
82
82
  @logger = Syslog::Logger.new(@program_name, Syslog::LOG_LOCAL2)
83
83
  else
84
- raise "unknown log type: #{new_logtype.class} #{new_logtype}"
84
+ raise "unknown log type: #{new_log_type.class} #{new_log_type}"
85
85
  end
86
86
  @logger.level = current_severity_integer
87
- @logger_type = new_logtype
87
+ @logger_type = new_log_type
88
88
  # update formatter with password hiding
89
89
  @logger.formatter = SecretHider.log_formatter(@logger.formatter)
90
90
  end
data/lib/aspera/nagios.rb CHANGED
@@ -10,32 +10,32 @@ module Aspera
10
10
  # date offset levels
11
11
  DATE_WARN_OFFSET = 2
12
12
  DATE_CRIT_OFFSET = 5
13
- private_constant :LEVELS,:ADD_PREFIX,:DATE_WARN_OFFSET,:DATE_CRIT_OFFSET
13
+ private_constant :LEVELS, :ADD_PREFIX, :DATE_WARN_OFFSET, :DATE_CRIT_OFFSET
14
14
 
15
15
  # add methods to add nagios error levels, each take component name and message
16
16
  LEVELS.each_index do |code|
17
17
  name = "#{ADD_PREFIX}#{LEVELS[code]}".to_sym
18
- define_method(name){|comp,msg|@data.push({code: code,comp: comp,msg: msg})}
18
+ define_method(name){|comp, msg|@data.push({code: code, comp: comp, msg: msg})}
19
19
  end
20
20
 
21
21
  class << self
22
22
  # process results of a analysis and display status and exit with code
23
23
  def process(data)
24
24
  raise 'INTERNAL ERROR, result must be list and not empty' unless data.is_a?(Array) && !data.empty?
25
- %w[status component message].each{|c|raise "INTERNAL ERROR, result must have #{c}" unless data.first.has_key?(c)}
25
+ %w[status component message].each{|c|raise "INTERNAL ERROR, result must have #{c}" unless data.first.key?(c)}
26
26
  res_errors = data.reject{|s|s['status'].eql?('ok')}
27
27
  # keep only errors in case of problem, other ok are assumed so
28
28
  data = res_errors unless res_errors.empty?
29
29
  # first is most critical
30
- data.sort!{|a,b|LEVELS.index(a['status'].to_sym) <=> LEVELS.index(b['status'].to_sym)}
30
+ data.sort!{|a, b|LEVELS.index(a['status'].to_sym) <=> LEVELS.index(b['status'].to_sym)}
31
31
  # build message: if multiple components: concatenate
32
- #message = data.map{|i|"#{i['component']}:#{i['message']}"}.join(', ').gsub("\n",' ')
33
- message = data.
34
- map{|i|i['component']}.
35
- uniq.
36
- map{|comp|comp + ':' + data.select{|d|d['component'].eql?(comp)}.map{|d|d['message']}.join(',')}.
37
- join(', ').
38
- tr("\n",' ')
32
+ # message = data.map{|i|"#{i['component']}:#{i['message']}"}.join(', ').gsub("\n",' ')
33
+ message = data
34
+ .map{|i|i['component']}
35
+ .uniq
36
+ .map{|comp|comp + ':' + data.select{|d|d['component'].eql?(comp)}.map{|d|d['message']}.join(',')}
37
+ .join(', ')
38
+ .tr("\n", ' ')
39
39
  status = data.first['status'].upcase
40
40
  # display status for nagios
41
41
  puts("#{status} - [#{message}]\n")
@@ -45,36 +45,37 @@ module Aspera
45
45
  end
46
46
 
47
47
  attr_reader :data
48
+
48
49
  def initialize
49
50
  @data = []
50
51
  end
51
52
 
52
- # comparte remote time with local time
53
+ # compare remote time with local time
53
54
  def check_time_offset(remote_date, component)
54
55
  # check date if specified : 2015-10-13T07:32:01Z
55
- rtime = DateTime.strptime(remote_date)
56
- diff_time = (rtime - DateTime.now).abs
57
- diff_disp = diff_time.round(-2)
58
- Log.log.debug("DATE: #{remote_date} #{rtime} diff=#{diff_disp}")
59
- msg = "offset #{diff_disp} sec"
56
+ remote_time = DateTime.strptime(remote_date)
57
+ diff_time = (remote_time - DateTime.now).abs
58
+ diff_rounded = diff_time.round(-2)
59
+ Log.log.debug{"DATE: #{remote_date} #{remote_time} diff=#{diff_rounded}"}
60
+ msg = "offset #{diff_rounded} sec"
60
61
  if diff_time >= DATE_CRIT_OFFSET
61
- add_critical(component,msg)
62
+ add_critical(component, msg)
62
63
  elsif diff_time >= DATE_WARN_OFFSET
63
- add_warning(component,msg)
64
+ add_warning(component, msg)
64
65
  else
65
- add_ok(component,msg)
66
+ add_ok(component, msg)
66
67
  end
67
68
  end
68
69
 
69
70
  def check_product_version(component, _product, version)
70
- add_ok(component,"version #{version}")
71
- # TODO check on database if latest version
71
+ add_ok(component, "version #{version}")
72
+ # TODO: check on database if latest version
72
73
  end
73
74
 
74
75
  # translate for display
75
76
  def result
76
77
  raise 'missing result' if @data.empty?
77
- {type: :object_list,data: @data.map{|i|{'status' => LEVELS[i[:code]].to_s,'component' => i[:comp],'message' => i[:msg]}}}
78
+ {type: :object_list, data: @data.map{|i|{'status' => LEVELS[i[:code]].to_s, 'component' => i[:comp], 'message' => i[:msg]}}}
78
79
  end
79
80
  end
80
81
  end
data/lib/aspera/node.rb CHANGED
@@ -9,27 +9,28 @@ require 'zlib'
9
9
  require 'base64'
10
10
 
11
11
  module Aspera
12
- # Provides additional functions using node API.
13
- class Node < Rest
12
+ # Provides additional functions using node API with gen4 extensions (access keys)
13
+ class Node < Aspera::Rest
14
14
  # permissions
15
15
  ACCESS_LEVELS = %w[delete list mkdir preview read rename write].freeze
16
16
  # prefix for ruby code for filter
17
17
  MATCH_EXEC_PREFIX = 'exec:'
18
+ HEADER_X_ASPERA_ACCESS_KEY = 'X-Aspera-AccessKey'
19
+ PATH_SEPARATOR = '/'
18
20
 
19
21
  # register node special token decoder
20
22
  Oauth.register_decoder(lambda{|token|JSON.parse(Zlib::Inflate.inflate(Base64.decode64(token)).partition('==SIGNATURE==').first)})
21
23
 
24
+ # class instance variable, access with accessors on class
25
+ @use_standard_ports = true
26
+
22
27
  class << self
23
- def set_ak_basic_token(ts,ak,secret)
24
- Log.log.warn("Expected transfer user: #{Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER}, "\
25
- "but have #{ts['remote_user']}") unless ts['remote_user'].eql?(Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER)
26
- ts['token'] = Rest.basic_creds(ak,secret)
27
- end
28
+ attr_accessor :use_standard_ports
28
29
 
29
30
  # for access keys: provide expression to match entry in folder
30
31
  # if no prefix: regex
31
32
  # if prefix: ruby code
32
- # if filder is nil, then always match
33
+ # if expression is nil, then always match
33
34
  def file_matcher(match_expression)
34
35
  match_expression ||= "#{MATCH_EXEC_PREFIX}true"
35
36
  if match_expression.start_with?(MATCH_EXEC_PREFIX)
@@ -39,49 +40,227 @@ module Aspera
39
40
  end
40
41
  end
41
42
 
42
- # def initialize(rest_params)
43
- # super(rest_params)
44
- # end
45
-
46
- # recursively crawl in a folder.
47
- # subfolders a processed if the processing method returns true
48
- # @param processor must provide a method to process each entry
49
- # @param opt options
50
- # - top_file_id file id to start at (default = access key root file id)
51
- # - top_file_path path of top folder (default = /)
52
- # - method processing method (default= process_entry)
53
- def crawl(processor,opt={})
54
- Log.log.debug("crawl1 #{opt}")
55
- # not possible with bearer token
56
- opt[:top_file_id] ||= read('access_keys/self')[:data]['root_file_id']
57
- opt[:top_file_path] ||= '/'
58
- opt[:method] ||= :process_entry
59
- raise "processor must have #{opt[:method]}" unless processor.respond_to?(opt[:method])
60
- Log.log.debug("crawl #{opt}")
61
- #top_info=read("files/#{opt[:top_file_id]}")[:data]
62
- folders_to_explore = [{id: opt[:top_file_id], relpath: opt[:top_file_path]}]
63
- Log.dump(:folders_to_explore,folders_to_explore)
64
- while !folders_to_explore.empty?
43
+ REQUIRED_APP_INFO_FIELDS = %i[node_info app api workspace_info].freeze
44
+ REQUIRED_APP_API_METHODS = %i[node_api_from add_ts_tags].freeze
45
+ private_constant :REQUIRED_APP_INFO_FIELDS, :REQUIRED_APP_API_METHODS
46
+
47
+ attr_reader :app_info
48
+
49
+ # @param params [Hash] Rest parameters
50
+ # @param app_info [Hash,NilClass] special processing for AoC
51
+ def initialize(params:, app_info: nil, add_tspec: nil)
52
+ super(params)
53
+ @app_info = app_info
54
+ # this is added to transfer spec, for instance to add tags (COS)
55
+ @add_tspec = add_tspec
56
+ if !@app_info.nil?
57
+ REQUIRED_APP_INFO_FIELDS.each do |field|
58
+ raise "INTERNAL ERROR: app_info lacks field #{field}" unless @app_info.key?(field)
59
+ end
60
+ REQUIRED_APP_API_METHODS.each do |method|
61
+ raise "INTERNAL ERROR: #{@app_info[:api].class} lacks method #{method}" unless @app_info[:api].respond_to?(method)
62
+ end
63
+ end
64
+ end
65
+
66
+ # update transfer spec with special additional tags
67
+ def add_tspec_info(tspec)
68
+ tspec.deep_merge!(@add_tspec) unless @add_tspec.nil?
69
+ return tspec
70
+ end
71
+
72
+ # @returns [Aspera::Node] a Node or nil
73
+ def node_id_to_node(node_id)
74
+ return self if !@app_info.nil? && @app_info[:node_info]['id'].eql?(node_id)
75
+ return @app_info[:api].node_api_from(node_id: node_id, workspace_info: @app_info[workspace_info]) unless @app_info.nil?
76
+ Log.log.warn{"cannot resolve link with node id #{node_id}"}
77
+ return nil
78
+ end
79
+
80
+ # recursively browse in a folder (with non-recursive method)
81
+ # sub folders are processed if the processing method returns true
82
+ # @param state [Object] state object sent to processing method
83
+ # @param method [Symbol] processing method name
84
+ # @param top_file_id [String] file id to start at (default = access key root file id)
85
+ # @param top_file_path [String] path of top folder (default = /)
86
+ def process_folder_tree(state:, method:, top_file_id:, top_file_path: '/')
87
+ raise 'INTERNAL ERROR: top_file_path not set' if top_file_path.nil?
88
+ raise "INTERNAL ERROR: Missing method #{method}" unless respond_to?(method)
89
+ folders_to_explore = [{id: top_file_id, path: top_file_path}]
90
+ Log.dump(:folders_to_explore, folders_to_explore)
91
+ until folders_to_explore.empty?
65
92
  current_item = folders_to_explore.shift
66
- Log.log.debug("searching #{current_item[:relpath]}".bg_green)
93
+ Log.log.debug{"searching #{current_item[:path]}".bg_green}
67
94
  # get folder content
68
95
  folder_contents =
69
96
  begin
70
97
  read("files/#{current_item[:id]}/files")[:data]
71
98
  rescue StandardError => e
72
- Log.log.warn("#{current_item[:relpath]}: #{e.class} #{e.message}")
99
+ Log.log.warn{"#{current_item[:path]}: #{e.class} #{e.message}"}
73
100
  []
74
101
  end
75
- Log.dump(:folder_contents,folder_contents)
102
+ Log.dump(:folder_contents, folder_contents)
76
103
  folder_contents.each do |entry|
77
- relative_path = File.join(current_item[:relpath],entry['name'])
78
- Log.log.debug("looking #{relative_path}".bg_green)
104
+ relative_path = File.join(current_item[:path], entry['name'])
105
+ Log.log.debug{"looking #{relative_path}".bg_green}
106
+ # continue only if method returns true
107
+ next unless send(method, entry, relative_path, state)
79
108
  # entry type is file, folder or link
80
- if processor.send(opt[:method],entry,relative_path) && entry['type'].eql?('folder')
81
- folders_to_explore.push({id: entry['id'],relpath: relative_path})
109
+ case entry['type']
110
+ when 'folder'
111
+ folders_to_explore.push({id: entry['id'], path: relative_path})
112
+ when 'link'
113
+ node_id_to_node(entry['target_node_id'])&.process_folder_tree(
114
+ state: state,
115
+ method: method,
116
+ top_file_id: entry['target_id'],
117
+ top_file_path: relative_path)
82
118
  end
83
119
  end
84
120
  end
121
+ end # process_folder_tree
122
+
123
+ # processing method to resolve a file path to id
124
+ # @returns true if processing need to continue
125
+ def process_resolve_node_path(entry, _path, state)
126
+ # stop digging here if not in right path
127
+ return false unless entry['name'].eql?(state[:path].first)
128
+ # ok it matches, so we remove the match
129
+ state[:path].shift
130
+ case entry['type']
131
+ when 'file'
132
+ # file must be terminal
133
+ raise "#{entry['name']} is a file, expecting folder to find: #{state[:path]}" unless state[:path].empty?
134
+ # it's terminal, we found it
135
+ state[:result] = {api: self, file_id: entry['id']}
136
+ return false
137
+ when 'folder'
138
+ if state[:path].empty?
139
+ # we found it
140
+ state[:result] = {api: self, file_id: entry['id']}
141
+ return false
142
+ end
143
+ when 'link'
144
+ if state[:path].empty?
145
+ # we found it
146
+ other_node = node_id_to_node(entry['target_node_id'])
147
+ raise 'cannot resolve link' if other_node.nil?
148
+ state[:result] = {api: other_node, file_id: entry['target_id']}
149
+ return false
150
+ end
151
+ else
152
+ Log.log.warn{"Unknown element type: #{entry['type']}"}
153
+ end
154
+ # continue to dig folder
155
+ return true
156
+ end
157
+
158
+ # Navigate the path from given file id
159
+ # @param top_file_id [String] id initial file id
160
+ # @param path [String] file path
161
+ # @return [Hash] {.api,.file_id}
162
+ def resolve_api_fid(top_file_id, path)
163
+ raise 'file id shall be String' unless top_file_id.is_a?(String)
164
+ path_elements = path.split(PATH_SEPARATOR).reject(&:empty?)
165
+ return {api: self, file_id: top_file_id} if path_elements.empty?
166
+ resolve_state = {path: path_elements, result: nil}
167
+ process_folder_tree(state: resolve_state, method: :process_resolve_node_path, top_file_id: top_file_id)
168
+ raise "entry not found: #{resolve_state[:path]}" if resolve_state[:result].nil?
169
+ return resolve_state[:result]
170
+ end
171
+
172
+ # add entry to list if test block is success
173
+ # @return [TrueClass,FalseClass]
174
+ def process_find_files(entry, path, state)
175
+ begin
176
+ # add to result if match filter
177
+ state[:found].push(entry.merge({'path' => path})) if state[:test_block].call(entry)
178
+ # process link
179
+ if entry[:type].eql?('link')
180
+ other_node = node_id_to_node(entry['target_node_id'])
181
+ other_node.process_folder_tree(state: state, method: process_find_files, top_file_id: entry['target_id'], top_file_path: path)
182
+ end
183
+ rescue StandardError => e
184
+ Log.log.error{"#{path}: #{e.message}"}
185
+ end
186
+ # process all folders
187
+ return true
188
+ end
189
+
190
+ def find_files(top_file_id, test_block)
191
+ Log.log.debug{"find_files: file id=#{top_file_id}"}
192
+ find_state = {found: [], test_block: test_block}
193
+ process_folder_tree(state: find_state, method: :process_find_files, top_file_id: top_file_id)
194
+ return find_state[:found]
195
+ end
196
+
197
+ def refreshed_transfer_token
198
+ return oauth_token(force_refresh: true)
199
+ end
200
+
201
+ # Create transfer spec for gen4
202
+ def transfer_spec_gen4(file_id, direction, ts_merge=nil)
203
+ ak_name = nil
204
+ ak_token = nil
205
+ case params[:auth][:type]
206
+ when :basic
207
+ ak_name = params[:auth][:username]
208
+ when :oauth2
209
+ ak_name = params[:headers][HEADER_X_ASPERA_ACCESS_KEY]
210
+ # TODO: token_generation_lambda = lambda{|do_refresh|oauth_token(force_refresh: do_refresh)}
211
+ # get bearer token, possibly use cache
212
+ ak_token = oauth_token(force_refresh: false)
213
+ else raise "Unsupported auth method for node gen4: #{params[:auth][:type]}"
214
+ end
215
+ transfer_spec = {
216
+ 'direction' => direction,
217
+ 'token' => ak_token,
218
+ 'tags' => {
219
+ 'aspera' => {
220
+ 'node' => {
221
+ 'access_key' => ak_name,
222
+ 'file_id' => file_id
223
+ } # node
224
+ } # aspera
225
+ } # tags
226
+ }
227
+ # add specials tags (cos)
228
+ add_tspec_info(transfer_spec)
229
+ transfer_spec.deep_merge!(ts_merge) unless ts_merge.nil?
230
+ # add application specific tags (AoC)
231
+ the_app = app_info
232
+ the_app[:api].add_ts_tags(transfer_spec: transfer_spec, app_info: the_app) unless the_app.nil?
233
+ # add basic token
234
+ if transfer_spec['token'].nil?
235
+ ts_basic_token(transfer_spec)
236
+ end
237
+ # add remote host info
238
+ if self.class.use_standard_ports
239
+ # get default TCP/UDP ports and transfer user
240
+ transfer_spec.merge!(Fasp::TransferSpec::AK_TSPEC_BASE)
241
+ # by default: same address as node API
242
+ transfer_spec['remote_host'] = URI.parse(params[:base_url]).host
243
+ if !@app_info.nil? && !@app_info[:node_info]['transfer_url'].nil? && !@app_info[:node_info]['transfer_url'].empty?
244
+ transfer_spec['remote_host'] = @app_info[:node_info]['transfer_url']
245
+ end
246
+ else
247
+ # retrieve values from API
248
+ std_t_spec = create(
249
+ 'files/download_setup',
250
+ {transfer_requests: [{ transfer_request: {paths: [{'source' => '/'}] } }] }
251
+ )[:data]['transfer_specs'].first['transfer_spec']
252
+ # copy some parts
253
+ %w[remote_host remote_user ssh_port fasp_port wss_enabled wss_port].each {|i| transfer_spec[i] = std_t_spec[i] if std_t_spec.key?(i)}
254
+ end
255
+ return transfer_spec
256
+ end
257
+
258
+ # set basic token in transfer spec
259
+ def ts_basic_token(ts)
260
+ Log.log.warn{"Expected transfer user: #{Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER}, but have #{ts['remote_user']}"} \
261
+ unless ts['remote_user'].eql?(Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER)
262
+ raise 'ERROR: no secret in node object' unless params[:auth][:password]
263
+ ts['token'] = Rest.basic_creds(params[:auth][:username], params[:auth][:password])
85
264
  end
86
265
  end
87
266
  end