aspera-cli 4.14.0 → 4.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +54 -3
  4. data/CONTRIBUTING.md +7 -7
  5. data/README.md +1457 -880
  6. data/bin/ascli +18 -9
  7. data/bin/asession +12 -14
  8. data/examples/proxy.pac +1 -1
  9. data/lib/aspera/aoc.rb +198 -127
  10. data/lib/aspera/ascmd.rb +24 -14
  11. data/lib/aspera/cli/basic_auth_plugin.rb +9 -6
  12. data/lib/aspera/cli/error.rb +17 -0
  13. data/lib/aspera/cli/extended_value.rb +47 -12
  14. data/lib/aspera/cli/formatter.rb +260 -171
  15. data/lib/aspera/cli/hints.rb +80 -0
  16. data/lib/aspera/cli/main.rb +101 -147
  17. data/lib/aspera/cli/manager.rb +160 -124
  18. data/lib/aspera/cli/plugin.rb +70 -59
  19. data/lib/aspera/cli/plugins/alee.rb +0 -1
  20. data/lib/aspera/cli/plugins/aoc.rb +239 -273
  21. data/lib/aspera/cli/plugins/ats.rb +8 -5
  22. data/lib/aspera/cli/plugins/bss.rb +2 -2
  23. data/lib/aspera/cli/plugins/config.rb +516 -375
  24. data/lib/aspera/cli/plugins/console.rb +40 -0
  25. data/lib/aspera/cli/plugins/cos.rb +4 -5
  26. data/lib/aspera/cli/plugins/faspex.rb +99 -84
  27. data/lib/aspera/cli/plugins/faspex5.rb +179 -148
  28. data/lib/aspera/cli/plugins/node.rb +219 -153
  29. data/lib/aspera/cli/plugins/orchestrator.rb +52 -17
  30. data/lib/aspera/cli/plugins/preview.rb +46 -32
  31. data/lib/aspera/cli/plugins/server.rb +57 -17
  32. data/lib/aspera/cli/plugins/shares.rb +34 -12
  33. data/lib/aspera/cli/sync_actions.rb +68 -0
  34. data/lib/aspera/cli/transfer_agent.rb +45 -55
  35. data/lib/aspera/cli/transfer_progress.rb +74 -0
  36. data/lib/aspera/cli/version.rb +1 -1
  37. data/lib/aspera/colors.rb +3 -1
  38. data/lib/aspera/command_line_builder.rb +14 -11
  39. data/lib/aspera/cos_node.rb +3 -2
  40. data/lib/aspera/environment.rb +17 -6
  41. data/lib/aspera/fasp/agent_aspera.rb +126 -0
  42. data/lib/aspera/fasp/agent_base.rb +31 -77
  43. data/lib/aspera/fasp/agent_connect.rb +21 -22
  44. data/lib/aspera/fasp/agent_direct.rb +88 -102
  45. data/lib/aspera/fasp/agent_httpgw.rb +196 -192
  46. data/lib/aspera/fasp/agent_node.rb +41 -34
  47. data/lib/aspera/fasp/agent_trsdk.rb +75 -34
  48. data/lib/aspera/fasp/error_info.rb +2 -2
  49. data/lib/aspera/fasp/faux_file.rb +52 -0
  50. data/lib/aspera/fasp/installation.rb +43 -184
  51. data/lib/aspera/fasp/management.rb +244 -0
  52. data/lib/aspera/fasp/parameters.rb +59 -26
  53. data/lib/aspera/fasp/parameters.yaml +75 -8
  54. data/lib/aspera/fasp/products.rb +162 -0
  55. data/lib/aspera/fasp/transfer_spec.rb +1 -1
  56. data/lib/aspera/fasp/uri.rb +4 -4
  57. data/lib/aspera/faspex_gw.rb +2 -2
  58. data/lib/aspera/faspex_postproc.rb +2 -2
  59. data/lib/aspera/hash_ext.rb +2 -2
  60. data/lib/aspera/json_rpc.rb +49 -0
  61. data/lib/aspera/line_logger.rb +23 -0
  62. data/lib/aspera/log.rb +57 -16
  63. data/lib/aspera/node.rb +97 -14
  64. data/lib/aspera/oauth.rb +36 -18
  65. data/lib/aspera/open_application.rb +4 -4
  66. data/lib/aspera/persistency_folder.rb +2 -2
  67. data/lib/aspera/preview/file_types.rb +4 -2
  68. data/lib/aspera/preview/generator.rb +22 -35
  69. data/lib/aspera/preview/options.rb +2 -0
  70. data/lib/aspera/preview/terminal.rb +24 -13
  71. data/lib/aspera/preview/utils.rb +19 -26
  72. data/lib/aspera/rest.rb +103 -72
  73. data/lib/aspera/rest_call_error.rb +1 -1
  74. data/lib/aspera/rest_error_analyzer.rb +15 -14
  75. data/lib/aspera/rest_errors_aspera.rb +37 -34
  76. data/lib/aspera/secret_hider.rb +14 -16
  77. data/lib/aspera/ssh.rb +4 -1
  78. data/lib/aspera/sync.rb +128 -122
  79. data/lib/aspera/temp_file_manager.rb +10 -3
  80. data/lib/aspera/web_auth.rb +10 -7
  81. data/lib/aspera/web_server_simple.rb +9 -4
  82. data.tar.gz.sig +0 -0
  83. metadata +33 -15
  84. metadata.gz.sig +0 -0
  85. data/lib/aspera/cli/listener/line_dump.rb +0 -19
  86. data/lib/aspera/cli/listener/logger.rb +0 -22
  87. data/lib/aspera/cli/listener/progress.rb +0 -50
  88. data/lib/aspera/cli/listener/progress_multi.rb +0 -84
  89. data/lib/aspera/cli/plugins/sync.rb +0 -44
  90. data/lib/aspera/fasp/listener.rb +0 -13
@@ -0,0 +1,162 @@
1
+ # frozen_string_literal: true
2
+
3
+ # cspell:ignore LOCALAPPDATA
4
+ require 'aspera/environment'
5
+
6
+ module Aspera
7
+ module Fasp
8
+ # find Aspera standard products installation in standard paths
9
+ class Products
10
+ # known product names
11
+ CONNECT = 'IBM Aspera Connect'
12
+ ASPERA = 'IBM Aspera (Client)'
13
+ CLI_V1 = 'Aspera CLI (deprecated)'
14
+ DRIVE = 'Aspera Drive (deprecated)'
15
+ HSTS = 'IBM Aspera High-Speed Transfer Server'
16
+ # product information manifest: XML (part of aspera product)
17
+ INFO_META_FILE = 'product-info.mf'
18
+ BIN_SUBFOLDER = 'bin'
19
+ ETC_SUBFOLDER = 'etc'
20
+ VAR_RUN_SUBFOLDER = File.join('var', 'run')
21
+
22
+ @@found_products = nil # rubocop:disable Style/ClassVars
23
+ class << self
24
+ # @return product folders depending on OS fields
25
+ # :expected M app name is taken from the manifest if present, else defaults to this value
26
+ # :app_root M main folder for the application
27
+ # :log_root O location of log files (Linux uses syslog)
28
+ # :run_root O only for Connect Client, location of http port file
29
+ # :sub_bin O subfolder with executables, default : bin
30
+ def product_locations_on_current_os
31
+ result =
32
+ case Aspera::Environment.os
33
+ when Aspera::Environment::OS_WINDOWS then [{
34
+ expected: CONNECT,
35
+ app_root: File.join(ENV.fetch('LOCALAPPDATA', nil), 'Programs', 'Aspera', 'Aspera Connect'),
36
+ log_root: File.join(ENV.fetch('LOCALAPPDATA', nil), 'Aspera', 'Aspera Connect', 'var', 'log'),
37
+ run_root: File.join(ENV.fetch('LOCALAPPDATA', nil), 'Aspera', 'Aspera Connect')
38
+ }, {
39
+ expected: CLI_V1,
40
+ app_root: File.join('C:', 'Program Files', 'Aspera', 'cli'),
41
+ log_root: File.join('C:', 'Program Files', 'Aspera', 'cli', 'var', 'log')
42
+ }, {
43
+ expected: HSTS,
44
+ app_root: File.join('C:', 'Program Files', 'Aspera', 'Enterprise Server'),
45
+ log_root: File.join('C:', 'Program Files', 'Aspera', 'Enterprise Server', 'var', 'log')
46
+ }]
47
+ when Aspera::Environment::OS_X then [{
48
+ expected: CONNECT,
49
+ app_root: File.join(Dir.home, 'Applications', 'Aspera Connect.app'),
50
+ log_root: File.join(Dir.home, 'Library', 'Logs', 'Aspera_Connect'),
51
+ run_root: File.join(Dir.home, 'Library', 'Application Support', 'Aspera', 'Aspera Connect'),
52
+ sub_bin: File.join('Contents', 'Resources')
53
+ }, {
54
+ expected: CONNECT,
55
+ app_root: File.join('', 'Applications', 'Aspera Connect.app'),
56
+ log_root: File.join(Dir.home, 'Library', 'Logs', 'Aspera_Connect'),
57
+ run_root: File.join(Dir.home, 'Library', 'Application Support', 'Aspera', 'Aspera Connect'),
58
+ sub_bin: File.join('Contents', 'Resources')
59
+ }, {
60
+ expected: CLI_V1,
61
+ app_root: File.join(Dir.home, 'Applications', 'Aspera CLI'),
62
+ log_root: File.join(Dir.home, 'Library', 'Logs', 'Aspera')
63
+ }, {
64
+ expected: HSTS,
65
+ app_root: File.join('', 'Library', 'Aspera'),
66
+ log_root: File.join(Dir.home, 'Library', 'Logs', 'Aspera')
67
+ }, {
68
+ expected: DRIVE,
69
+ app_root: File.join('', 'Applications', 'Aspera Drive.app'),
70
+ log_root: File.join(Dir.home, 'Library', 'Logs', 'Aspera_Drive'),
71
+ sub_bin: File.join('Contents', 'Resources')
72
+ }, {
73
+ expected: ASPERA,
74
+ app_root: File.join('', 'Applications', 'IBM Aspera.app'),
75
+ log_root: File.join(Dir.home, 'Library', 'Logs', 'IBM Aspera'),
76
+ sub_bin: File.join('Contents', 'Resources', 'sdk', 'aspera', 'bin')
77
+ }]
78
+ else [{ # other: Linux and Unix family
79
+ expected: CONNECT,
80
+ app_root: File.join(Dir.home, '.aspera', 'connect'),
81
+ run_root: File.join(Dir.home, '.aspera', 'connect')
82
+ }, {
83
+ expected: CLI_V1,
84
+ app_root: File.join(Dir.home, '.aspera', 'cli')
85
+ }, {
86
+ expected: HSTS,
87
+ app_root: File.join('', 'opt', 'aspera')
88
+ }]
89
+ end
90
+ result # .each {|item| item.deep_do {|h, _k, _v, _m|h.freeze}}.freeze
91
+ end
92
+
93
+ # @return the list of installed products in format of product_locations_on_current_os
94
+ def installed_products
95
+ if @@found_products.nil?
96
+ scan_locations = product_locations_on_current_os.clone
97
+ # add SDK as first search path
98
+ scan_locations.unshift({
99
+ expected: 'SDK',
100
+ app_root: Installation.instance.sdk_folder,
101
+ sub_bin: ''
102
+ })
103
+ # search installed products: with ascp
104
+ @@found_products = scan_locations.select! do |item| # rubocop:disable Style/ClassVars
105
+ # skip if not main folder
106
+ next false unless Dir.exist?(item[:app_root])
107
+ Log.log.debug{"Found #{item[:app_root]}"}
108
+ sub_bin = item[:sub_bin] || BIN_SUBFOLDER
109
+ item[:ascp_path] = File.join(item[:app_root], sub_bin, ascp_filename)
110
+ # skip if no ascp
111
+ next false unless File.exist?(item[:ascp_path])
112
+ # read info from product info file if present
113
+ product_info_file = "#{item[:app_root]}/#{INFO_META_FILE}"
114
+ if File.exist?(product_info_file)
115
+ res_s = XmlSimple.xml_in(File.read(product_info_file), {'ForceArray' => false})
116
+ item[:name] = res_s['name']
117
+ item[:version] = res_s['version']
118
+ else
119
+ item[:name] = item[:expected]
120
+ end
121
+ true # select this version
122
+ end
123
+ end
124
+ return @@found_products
125
+ end
126
+
127
+ # filename for ascp with optional extension (Windows)
128
+ def ascp_filename
129
+ return 'ascp' + Environment.exe_extension
130
+ end
131
+
132
+ # @return folder paths for specified applications
133
+ # @param name Connect or CLI
134
+ def folders(name)
135
+ found = Products.installed_products.select{|i|i[:expected].eql?(name) || i[:name].eql?(name)}
136
+ raise "Product: #{name} not found, please install." if found.empty?
137
+ return found.first
138
+ end
139
+
140
+ # @return the file path of local connect where API's URI can be read
141
+ def connect_uri
142
+ connect = folders(CONNECT)
143
+ folder = File.join(connect[:run_root], VAR_RUN_SUBFOLDER)
144
+ ['', 's'].each do |ext|
145
+ uri_file = File.join(folder, "http#{ext}.uri")
146
+ Log.log.debug{"checking connect port file: #{uri_file}"}
147
+ if File.exist?(uri_file)
148
+ return File.open(uri_file, &:gets).strip
149
+ end
150
+ end
151
+ raise "no connect uri file found in #{folder}"
152
+ end
153
+
154
+ # @ return path to configuration file of aspera CLI
155
+ # def cli_conf_file
156
+ # connect = folders(PRODUCT_CLI_V1)
157
+ # return File.join(connect[:app_root], BIN_SUBFOLDER, '.aspera_cli_conf')
158
+ # end
159
+ end
160
+ end
161
+ end
162
+ end
@@ -41,7 +41,7 @@ module Aspera
41
41
  return case tspec['direction']
42
42
  when DIRECTION_SEND then :upload
43
43
  when DIRECTION_RECEIVE then :download
44
- else raise 'Error: upload or download only'
44
+ else raise "Error: upload or download only, not #{tspec['direction']} (#{tspec['direction'].class})"
45
45
  end
46
46
  end
47
47
  end
@@ -1,14 +1,16 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # cspell:words httpport targetrate minrate bwcap createpath lockpolicy lockminrate
3
+ # cspell:words httpport targetrate minrate bwcap createpath lockpolicy lockminrate faspe
4
4
 
5
5
  require 'aspera/log'
6
+ require 'aspera/rest'
6
7
  require 'aspera/command_line_builder'
7
8
 
8
9
  module Aspera
9
10
  module Fasp
10
11
  # translates a "faspe:" URI (used in Faspex 4) into transfer spec hash
11
12
  class Uri
13
+ SCHEME = 'faspe'
12
14
  def initialize(fasp_link)
13
15
  @fasp_uri = URI.parse(fasp_link.gsub(' ', '%20'))
14
16
  # TODO: check scheme is faspe
@@ -23,9 +25,7 @@ module Aspera
23
25
  # faspex does not encode trailing base64 padding, fix that to be able to decode properly
24
26
  fixed_query = @fasp_uri.query.gsub(/(=+)$/){|x|'%3D' * x.length}
25
27
 
26
- URI.decode_www_form(fixed_query).each do |i|
27
- name = i[0]
28
- value = i[1]
28
+ Rest.decode_query(fixed_query).each do |name, value|
29
29
  case name
30
30
  when 'cookie' then result_ts['cookie'] = value
31
31
  when 'token' then result_ts['token'] = value
@@ -68,9 +68,9 @@ module Aspera
68
68
  faspex_pkg_parameters = JSON.parse(request.body)
69
69
  Log.log.debug{"faspex pkg create parameters=#{faspex_pkg_parameters}"}
70
70
  faspex_package_create_result =
71
- if @app_api.is_a?(Aspera::AoC)
71
+ if @app_api.class.name.eql?('Aspera::AoC')
72
72
  faspex4_send_to_aoc(faspex_pkg_parameters)
73
- elsif @app_api.is_a?(Aspera::Rest)
73
+ elsif @app_api.class.name.eql?('Aspera::Rest')
74
74
  faspex4_send_to_faspex5(faspex_pkg_parameters)
75
75
  else
76
76
  raise "No such adapter: #{@app_api.class}"
@@ -13,7 +13,7 @@ module Aspera
13
13
  def initialize(server, parameters)
14
14
  raise 'parameters must be Hash' unless parameters.is_a?(Hash)
15
15
  @parameters = parameters.symbolize_keys
16
- Log.dump(:post_proc_parameters, @parameters)
16
+ Log.log.debug{Log.dump(:post_proc_parameters, @parameters)}
17
17
  raise "unexpected key in parameters config: only: #{ALLOWED_PARAMETERS.join(', ')}" if @parameters.keys.any?{|k|!ALLOWED_PARAMETERS.include?(k)}
18
18
  @parameters[:script_folder] ||= '.'
19
19
  @parameters[:fail_on_error] ||= false
@@ -44,7 +44,7 @@ module Aspera
44
44
  script_path = File.join(@parameters[:script_folder], script_file)
45
45
  Log.log.debug{"script=#{script_path}"}
46
46
  webhook_parameters = JSON.parse(request.body)
47
- Log.dump(:webhook_parameters, webhook_parameters)
47
+ Log.log.debug{Log.dump(:webhook_parameters, webhook_parameters)}
48
48
  # env expects only strings
49
49
  environment = webhook_parameters.each_with_object({}) { |(k, v), h| h[k] = v.to_s }
50
50
  post_proc_pid = Process.spawn(environment, [script_path, script_path])
@@ -24,8 +24,8 @@ end
24
24
  unless Hash.method_defined?(:transform_keys)
25
25
  class Hash
26
26
  def transform_keys
27
- return each_with_object({}){|(k, v), memo|memo[yield(k)] = v} if block_given?
28
- raise 'missing block'
27
+ raise 'missing block' unless block_given?
28
+ return each_with_object({}){|(k, v), memo|memo[yield(k)] = v}
29
29
  end
30
30
  end
31
31
  end
@@ -0,0 +1,49 @@
1
+ # frozen_string_literal: true
2
+
3
+ # cspell:ignore blankslate
4
+
5
+ require 'aspera/rest_error_analyzer'
6
+ require 'blankslate'
7
+
8
+ Aspera::RestErrorAnalyzer.instance.add_simple_handler(name: 'JSON RPC', path: %w[error message], always: true)
9
+
10
+ module Aspera
11
+ # a very simple JSON RPC client
12
+ class JsonRpcClient < BlankSlate
13
+ JSON_RPC_VERSION = '2.0'
14
+ reveal :instance_variable_get
15
+ reveal :inspect
16
+ reveal :to_s
17
+
18
+ def initialize(api, namespace = nil)
19
+ super()
20
+ @api = api
21
+ @namespace = namespace
22
+ @request_id = 0
23
+ end
24
+
25
+ def respond_to_missing?(sym, include_private = false)
26
+ true
27
+ end
28
+
29
+ def method_missing(method, *args, &block)
30
+ args = args.first if args.size == 1 && args.first.is_a?(Hash)
31
+ data = @api.create('', {
32
+ jsonrpc: JSON_RPC_VERSION,
33
+ method: "#{@namespace}#{method}",
34
+ params: args,
35
+ id: @request_id += 1
36
+ })[:data]
37
+ raise 'response shall be Hash' unless data.is_a?(Hash)
38
+ raise 'bad version in response' unless data['jsonrpc'] == JSON_RPC_VERSION
39
+ raise 'missing id in response' unless data.key?('id')
40
+ raise 'both error and response' if data.key?('error') && data.key?('result')
41
+ raise 'bad error response' unless
42
+ !data.key?('error') ||
43
+ data['error'].is_a?(Hash) &&
44
+ data['error']['code'].is_a?(Integer) &&
45
+ data['error']['message'].is_a?(String)
46
+ return data['result']
47
+ end
48
+ end
49
+ end
@@ -0,0 +1,23 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'aspera/log'
4
+
5
+ module Aspera
6
+ # used for logging http
7
+ class LineLogger
8
+ def initialize(level)
9
+ @level = level
10
+ @buffer = []
11
+ end
12
+
13
+ def <<(string)
14
+ return if string.nil? || string.empty?
15
+ if !string.end_with?("\n")
16
+ @buffer.push(string)
17
+ return
18
+ end
19
+ Log.log.send(@level, @buffer.join('') + string.chomp)
20
+ @buffer.clear
21
+ end
22
+ end
23
+ end
data/lib/aspera/log.rb CHANGED
@@ -2,17 +2,53 @@
2
2
 
3
3
  require 'aspera/colors'
4
4
  require 'aspera/secret_hider'
5
+ require 'aspera/environment'
5
6
  require 'logger'
6
7
  require 'pp'
7
8
  require 'json'
8
9
  require 'singleton'
9
10
 
11
+ # extend Ruby logger with trace levels
12
+ class Logger
13
+ TRACE_MAX = 2
14
+ # add custom level to logger severity
15
+ module Severity
16
+ 1.upto(TRACE_MAX).each { |level| const_set("TRACE#{level}", - level)}
17
+ end
18
+ # quick access to label
19
+ SEVERITY_LABEL = Severity.constants.each_with_object({}) { |name, hash| hash[Severity.const_get(name)] = name}
20
+ def format_severity(severity)
21
+ SEVERITY_LABEL[severity] || 'ANY'
22
+ end
23
+
24
+ # define methods for a given level
25
+ def self.make_methods(str_level) # rubocop:disable Style/ClassMethodsDefinitions
26
+ int_level = ::Logger.const_get(str_level.upcase)
27
+ str_level = str_level.downcase
28
+ Kernel.send('lave'.reverse, <<-EOM, nil, __FILE__, __LINE__ + 1)
29
+ def #{str_level}(message = nil, &block)
30
+ add(#{int_level}, message, &block)
31
+ end
32
+
33
+ def #{str_level}?
34
+ level <= #{int_level}
35
+ end
36
+
37
+ def #{str_level}!
38
+ self.level = #{int_level}
39
+ end
40
+ EOM
41
+ end
42
+ Logger::Severity.constants.each { |severity| make_methods(severity) }
43
+ end
44
+
10
45
  module Aspera
11
46
  # Singleton object for logging
12
47
  class Log
13
48
  include Singleton
14
49
  # where logs are sent to
15
50
  LOG_TYPES = %i[stderr stdout syslog].freeze
51
+ @@format = :json # rubocop:disable Style/ClassVars
16
52
  # class methods
17
53
  class << self
18
54
  # levels are :debug,:info,:warn,:error,fatal,:unknown
@@ -21,22 +57,20 @@ module Aspera
21
57
  # get the logger object of singleton
22
58
  def log; instance.logger; end
23
59
 
24
- # dump object in debug mode
60
+ # dump object suitable for Log.log.debug
25
61
  # @param name string or symbol
26
62
  # @param format either pp or json format
27
- def dump(name, object, format=:json)
28
- log.debug do
29
- result =
30
- case format
31
- when :json
32
- JSON.pretty_generate(object) rescue PP.pp(object, +'')
33
- when :ruby
34
- PP.pp(object, +'')
35
- else
36
- raise 'wrong parameter, expect pp or json'
37
- end
38
- "#{name.to_s.green} (#{format})=\n#{result}"
39
- end
63
+ def dump(name, object)
64
+ result =
65
+ case @@format
66
+ when :json
67
+ JSON.pretty_generate(object) rescue PP.pp(object, +'')
68
+ when :ruby
69
+ PP.pp(object, +'')
70
+ else
71
+ raise 'wrong parameter, expect ruby or json'
72
+ end
73
+ "#{name.to_s.green} (#{@@format})=\n#{result}"
40
74
  end
41
75
 
42
76
  # Capture the output of $stderr and log it at debug level
@@ -70,16 +104,23 @@ module Aspera
70
104
  # change underlying logger, but keep log level
71
105
  def logger_type=(new_log_type)
72
106
  current_severity_integer = @logger.level unless @logger.nil?
73
- current_severity_integer = ENV['AS_LOG_LEVEL'] if current_severity_integer.nil? && ENV.key?('AS_LOG_LEVEL')
107
+ current_severity_integer = ENV.fetch('AS_LOG_LEVEL', nil) if current_severity_integer.nil? && ENV.key?('AS_LOG_LEVEL')
74
108
  current_severity_integer = Logger::Severity::WARN if current_severity_integer.nil?
75
109
  case new_log_type
76
110
  when :stderr
77
- # typed: Logger
78
111
  @logger = Logger.new($stderr)
79
112
  when :stdout
80
113
  @logger = Logger.new($stdout)
81
114
  when :syslog
82
115
  require 'syslog/logger'
116
+ # the syslog class automatically creates methods from the severity names
117
+ # we just need to add the mapping (but syslog lowest is DEBUG)
118
+ 1.upto(Logger::TRACE_MAX).each do |level|
119
+ Syslog::Logger.const_get(:LEVEL_MAP)[Logger.const_get("TRACE#{level}")] = Syslog::LOG_DEBUG
120
+ end
121
+ Logger::Severity.constants.each do |severity|
122
+ Syslog::Logger.make_methods(severity.downcase)
123
+ end
83
124
  @logger = Syslog::Logger.new(@program_name, Syslog::LOG_LOCAL2)
84
125
  else
85
126
  raise "unknown log type: #{new_log_type.class} #{new_log_type}"
data/lib/aspera/node.rb CHANGED
@@ -1,5 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require 'aspera/cli/error'
3
4
  require 'aspera/fasp/transfer_spec'
4
5
  require 'aspera/rest'
5
6
  require 'aspera/oauth'
@@ -13,14 +14,22 @@ module Aspera
13
14
  class Node < Aspera::Rest
14
15
  # permissions
15
16
  ACCESS_LEVELS = %w[delete list mkdir preview read rename write].freeze
16
- # prefix for ruby code for filter
17
+ # prefix for ruby code for filter (deprecated)
17
18
  MATCH_EXEC_PREFIX = 'exec:'
19
+ MATCH_TYPES = [String, Proc, Regexp, NilClass].freeze
18
20
  HEADER_X_ASPERA_ACCESS_KEY = 'X-Aspera-AccessKey'
19
21
  PATH_SEPARATOR = '/'
20
22
  TS_FIELDS_TO_COPY = %w[remote_host remote_user ssh_port fasp_port wss_enabled wss_port].freeze
23
+ SCOPE_USER = 'user:all'
24
+ SCOPE_ADMIN = 'admin:all'
25
+ SCOPE_PREFIX = 'node.'
26
+ SCOPE_SEPARATOR = ':'
27
+ SIGNATURE_DELIMITER = '==SIGNATURE=='
28
+ BEARER_TOKEN_VALIDITY_DEFAULT = 86400
29
+ BEARER_TOKEN_SCOPE_DEFAULT = SCOPE_USER
21
30
 
22
31
  # register node special token decoder
23
- Oauth.register_decoder(lambda{|token|JSON.parse(Zlib::Inflate.inflate(Base64.decode64(token)).partition('==SIGNATURE==').first)})
32
+ Oauth.register_decoder(lambda{|token|Node.decode_bearer_token(token)})
24
33
 
25
34
  # class instance variable, access with accessors on class
26
35
  @use_standard_ports = true
@@ -28,16 +37,79 @@ module Aspera
28
37
  class << self
29
38
  attr_accessor :use_standard_ports
30
39
 
31
- # for access keys: provide expression to match entry in folder
32
- # if no prefix: regex
33
- # if prefix: ruby code
34
- # if expression is nil, then always match
40
+ # For access keys: provide expression to match entry in folder
35
41
  def file_matcher(match_expression)
36
- match_expression ||= "#{MATCH_EXEC_PREFIX}true"
37
- if match_expression.start_with?(MATCH_EXEC_PREFIX)
38
- return Environment.secure_eval("lambda{|f|#{match_expression[MATCH_EXEC_PREFIX.length..-1]}}")
42
+ case match_expression
43
+ when Proc then return match_expression
44
+ when Regexp then return ->(f){f['name'].match?(match_expression)}
45
+ when String
46
+ if match_expression.start_with?(MATCH_EXEC_PREFIX)
47
+ code = "->(f){#{match_expression[MATCH_EXEC_PREFIX.length..-1]}}"
48
+ Log.log.warn{"Use of prefix #{MATCH_EXEC_PREFIX} is deprecated (4.15), instead use: @ruby:'#{code}'"}
49
+ return Environment.secure_eval(code, __FILE__, __LINE__)
50
+ end
51
+ return lambda{|f|File.fnmatch(match_expression, f['name'], File::FNM_DOTMATCH)}
52
+ when NilClass then return ->(_){true}
53
+ else raise Cli::BadArgument, "Invalid match expression type: #{match_expression.class}"
54
+ end
55
+ end
56
+
57
+ def file_matcher_from_argument(options)
58
+ return file_matcher(options.get_next_argument('filter', type: MATCH_TYPES, mandatory: false))
59
+ end
60
+
61
+ # node API scopes
62
+ def token_scope(access_key, scope)
63
+ return [SCOPE_PREFIX, access_key, SCOPE_SEPARATOR, scope].join('')
64
+ end
65
+
66
+ def decode_scope(scope)
67
+ items = scope.split(SCOPE_SEPARATOR, 2)
68
+ raise "invalid scope: #{scope}" unless items.length.eql?(2)
69
+ raise "invalid scope: #{scope}" unless items[0].start_with?(SCOPE_PREFIX)
70
+ return {access_key: items[0][SCOPE_PREFIX.length..-1], scope: items[1]}
71
+ end
72
+
73
+ # Create an Aspera Node bearer token
74
+ # @param payload [String] JSON payload to be included in the token
75
+ # @param private_key [OpenSSL::PKey::RSA] Private key to sign the token
76
+ def bearer_token(access_key:, payload:, private_key:)
77
+ raise 'payload shall be Hash' unless payload.is_a?(Hash)
78
+ raise 'missing user_id' unless payload.key?('user_id')
79
+ raise 'user_id must be a String' unless payload['user_id'].is_a?(String)
80
+ raise 'user_id must not be empty' if payload['user_id'].empty?
81
+ raise 'private_key shall be OpenSSL::PKey::RSA' unless private_key.is_a?(OpenSSL::PKey::RSA)
82
+ # manage convenience parameters
83
+ expiration_sec = payload['_validity'] || BEARER_TOKEN_VALIDITY_DEFAULT
84
+ payload.delete('_validity')
85
+ scope = payload['_scope'] || BEARER_TOKEN_SCOPE_DEFAULT
86
+ payload.delete('_scope')
87
+ payload['scope'] ||= token_scope(access_key, scope)
88
+ payload['auth_type'] ||= 'access_key'
89
+ payload['expires_at'] ||= (Time.now + expiration_sec).utc.strftime('%FT%TZ')
90
+ payload_json = JSON.generate(payload)
91
+ return Base64.strict_encode64(Zlib::Deflate.deflate([
92
+ payload_json,
93
+ SIGNATURE_DELIMITER,
94
+ Base64.strict_encode64(private_key.sign(OpenSSL::Digest.new('sha512'), payload_json)).scan(/.{1,60}/).join("\n"),
95
+ ''
96
+ ].join("\n")))
97
+ end
98
+
99
+ def decode_bearer_token(token)
100
+ return JSON.parse(Zlib::Inflate.inflate(Base64.decode64(token)).partition(SIGNATURE_DELIMITER).first)
101
+ end
102
+
103
+ def bearer_headers(bearer_auth, access_key: nil)
104
+ # if username is not provided, use the access key from the token
105
+ if access_key.nil?
106
+ access_key = Aspera::Node.decode_scope(Aspera::Node.decode_bearer_token(Oauth.bearer_extract(bearer_auth))['scope'])[:access_key]
107
+ raise "internal error #{access_key}" if access_key.nil?
39
108
  end
40
- return lambda{|f|f['name'].match(/#{match_expression}/)}
109
+ return {
110
+ Aspera::Node::HEADER_X_ASPERA_ACCESS_KEY => access_key,
111
+ 'Authorization' => bearer_auth
112
+ }
41
113
  end
42
114
  end
43
115
 
@@ -52,6 +124,7 @@ module Aspera
52
124
  # @param params [Hash] Rest parameters
53
125
  # @param app_info [Hash,NilClass] special processing for AoC
54
126
  def initialize(params:, app_info: nil, add_tspec: nil)
127
+ # init Rest
55
128
  super(params)
56
129
  @app_info = app_info
57
130
  # this is added to transfer spec, for instance to add tags (COS)
@@ -90,13 +163,13 @@ module Aspera
90
163
  # @param state [Object] state object sent to processing method
91
164
  # @param top_file_id [String] file id to start at (default = access key root file id)
92
165
  # @param top_file_path [String] path of top folder (default = /)
93
- # @param block [Proc] processing method, args: entry, path, state
166
+ # @param block [Proc] processing method, arguments: entry, path, state
94
167
  def process_folder_tree(state:, top_file_id:, top_file_path: '/', &block)
95
168
  raise 'INTERNAL ERROR: top_file_path not set' if top_file_path.nil?
96
169
  raise 'INTERNAL ERROR: Missing block' unless block
97
170
  # start at top folder
98
171
  folders_to_explore = [{id: top_file_id, path: top_file_path}]
99
- Log.dump(:folders_to_explore, folders_to_explore)
172
+ Log.log.debug{Log.dump(:folders_to_explore, folders_to_explore)}
100
173
  until folders_to_explore.empty?
101
174
  current_item = folders_to_explore.shift
102
175
  Log.log.debug{"searching #{current_item[:path]}".bg_green}
@@ -108,7 +181,7 @@ module Aspera
108
181
  Log.log.warn{"#{current_item[:path]}: #{e.class} #{e.message}"}
109
182
  []
110
183
  end
111
- Log.dump(:folder_contents, folder_contents)
184
+ Log.log.debug{Log.dump(:folder_contents, folder_contents)}
112
185
  folder_contents.each do |entry|
113
186
  relative_path = File.join(current_item[:path], entry['name'])
114
187
  Log.log.debug{"process_folder_tree checking #{relative_path}"}
@@ -204,7 +277,7 @@ module Aspera
204
277
  when :basic
205
278
  ak_name = params[:auth][:username]
206
279
  raise 'ERROR: no secret in node object' unless params[:auth][:password]
207
- ak_token = Rest.basic_creds(params[:auth][:username], params[:auth][:password])
280
+ ak_token = Rest.basic_token(params[:auth][:username], params[:auth][:password])
208
281
  when :oauth2
209
282
  ak_name = params[:headers][HEADER_X_ASPERA_ACCESS_KEY]
210
283
  # TODO: token_generation_lambda = lambda{|do_refresh|oauth_token(force_refresh: do_refresh)}
@@ -235,9 +308,19 @@ module Aspera
235
308
  transfer_spec.merge!(Fasp::TransferSpec::AK_TSPEC_BASE)
236
309
  # by default: same address as node API
237
310
  transfer_spec['remote_host'] = URI.parse(params[:base_url]).host
311
+ # AoC allows specification of other url
238
312
  if !@app_info.nil? && !@app_info[:node_info]['transfer_url'].nil? && !@app_info[:node_info]['transfer_url'].empty?
239
313
  transfer_spec['remote_host'] = @app_info[:node_info]['transfer_url']
240
314
  end
315
+ info = read('info')[:data]
316
+ # get the transfer user from info on access key
317
+ transfer_spec['remote_user'] = info['transfer_user'] if info['transfer_user']
318
+ # get settings from name.value array to hash key.value
319
+ settings = info['settings']&.each_with_object({}){|i, h|h[i['name']] = i['value']}
320
+ # check WSS ports
321
+ %w[wss_enabled wss_port].each do |i|
322
+ transfer_spec[i] = settings[i] if settings.key?(i)
323
+ end if settings.is_a?(Hash)
241
324
  else
242
325
  # retrieve values from API (and keep a copy/cache)
243
326
  @std_t_spec_cache ||= create(