aspera-cli 4.15.0 → 4.17.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (108) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +29 -3
  4. data/CHANGELOG.md +375 -280
  5. data/CONTRIBUTING.md +71 -18
  6. data/README.md +1978 -1656
  7. data/bin/ascli +13 -31
  8. data/bin/asession +32 -22
  9. data/examples/dascli +2 -2
  10. data/lib/aspera/agent/alpha.rb +117 -0
  11. data/lib/aspera/agent/base.rb +61 -0
  12. data/lib/aspera/{fasp/agent_connect.rb → agent/connect.rb} +13 -11
  13. data/lib/aspera/{fasp/agent_direct.rb → agent/direct.rb} +116 -116
  14. data/lib/aspera/{fasp/agent_httpgw.rb → agent/httpgw.rb} +21 -19
  15. data/lib/aspera/{fasp/agent_node.rb → agent/node.rb} +21 -33
  16. data/lib/aspera/agent/trsdk.rb +188 -0
  17. data/lib/aspera/api/aoc.rb +586 -0
  18. data/lib/aspera/api/ats.rb +46 -0
  19. data/lib/aspera/api/cos_node.rb +95 -0
  20. data/lib/aspera/api/node.rb +344 -0
  21. data/lib/aspera/ascmd.rb +47 -14
  22. data/lib/aspera/{fasp → ascp}/installation.rb +54 -15
  23. data/lib/aspera/{fasp → ascp}/management.rb +14 -14
  24. data/lib/aspera/{fasp → ascp}/products.rb +1 -1
  25. data/lib/aspera/assert.rb +45 -0
  26. data/lib/aspera/cli/basic_auth_plugin.rb +11 -10
  27. data/lib/aspera/cli/extended_value.rb +5 -5
  28. data/lib/aspera/cli/formatter.rb +27 -14
  29. data/lib/aspera/cli/hints.rb +7 -6
  30. data/lib/aspera/cli/main.rb +49 -29
  31. data/lib/aspera/cli/manager.rb +46 -36
  32. data/lib/aspera/cli/plugin.rb +34 -20
  33. data/lib/aspera/cli/plugin_factory.rb +61 -0
  34. data/lib/aspera/cli/plugins/alee.rb +7 -7
  35. data/lib/aspera/cli/plugins/aoc.rb +168 -132
  36. data/lib/aspera/cli/plugins/ats.rb +33 -33
  37. data/lib/aspera/cli/plugins/bss.rb +3 -4
  38. data/lib/aspera/cli/plugins/config.rb +250 -272
  39. data/lib/aspera/cli/plugins/console.rb +8 -6
  40. data/lib/aspera/cli/plugins/cos.rb +20 -19
  41. data/lib/aspera/cli/plugins/faspex.rb +71 -60
  42. data/lib/aspera/cli/plugins/faspex5.rb +212 -133
  43. data/lib/aspera/cli/plugins/node.rb +83 -75
  44. data/lib/aspera/cli/plugins/orchestrator.rb +36 -44
  45. data/lib/aspera/cli/plugins/preview.rb +33 -31
  46. data/lib/aspera/cli/plugins/server.rb +33 -32
  47. data/lib/aspera/cli/plugins/shares.rb +39 -33
  48. data/lib/aspera/cli/sync_actions.rb +9 -9
  49. data/lib/aspera/cli/transfer_agent.rb +45 -25
  50. data/lib/aspera/cli/transfer_progress.rb +2 -3
  51. data/lib/aspera/cli/version.rb +1 -1
  52. data/lib/aspera/colors.rb +5 -0
  53. data/lib/aspera/command_line_builder.rb +16 -14
  54. data/lib/aspera/coverage.rb +21 -0
  55. data/lib/aspera/data_repository.rb +33 -2
  56. data/lib/aspera/environment.rb +5 -4
  57. data/lib/aspera/faspex_gw.rb +13 -11
  58. data/lib/aspera/faspex_postproc.rb +6 -5
  59. data/lib/aspera/id_generator.rb +4 -2
  60. data/lib/aspera/json_rpc.rb +10 -8
  61. data/lib/aspera/keychain/encrypted_hash.rb +46 -11
  62. data/lib/aspera/keychain/macos_security.rb +29 -22
  63. data/lib/aspera/log.rb +5 -4
  64. data/lib/aspera/nagios.rb +7 -2
  65. data/lib/aspera/node_simulator.rb +213 -0
  66. data/lib/aspera/oauth/base.rb +143 -0
  67. data/lib/aspera/oauth/factory.rb +124 -0
  68. data/lib/aspera/oauth/generic.rb +34 -0
  69. data/lib/aspera/oauth/jwt.rb +51 -0
  70. data/lib/aspera/oauth/url_json.rb +31 -0
  71. data/lib/aspera/oauth/web.rb +50 -0
  72. data/lib/aspera/oauth.rb +5 -328
  73. data/lib/aspera/open_application.rb +7 -7
  74. data/lib/aspera/persistency_action_once.rb +13 -14
  75. data/lib/aspera/persistency_folder.rb +3 -2
  76. data/lib/aspera/preview/file_types.rb +53 -267
  77. data/lib/aspera/preview/generator.rb +7 -5
  78. data/lib/aspera/preview/terminal.rb +17 -7
  79. data/lib/aspera/preview/utils.rb +8 -7
  80. data/lib/aspera/proxy_auto_config.rb +6 -3
  81. data/lib/aspera/rest.rb +187 -140
  82. data/lib/aspera/rest_error_analyzer.rb +1 -0
  83. data/lib/aspera/rest_errors_aspera.rb +5 -3
  84. data/lib/aspera/resumer.rb +77 -0
  85. data/lib/aspera/secret_hider.rb +5 -2
  86. data/lib/aspera/ssh.rb +15 -8
  87. data/lib/aspera/temp_file_manager.rb +1 -1
  88. data/lib/aspera/{fasp → transfer}/error.rb +3 -3
  89. data/lib/aspera/{fasp → transfer}/error_info.rb +1 -1
  90. data/lib/aspera/{fasp → transfer}/faux_file.rb +1 -1
  91. data/lib/aspera/{fasp → transfer}/parameters.rb +95 -120
  92. data/lib/aspera/{fasp/transfer_spec.rb → transfer/spec.rb} +23 -19
  93. data/lib/aspera/{fasp/parameters.yaml → transfer/spec.yaml} +4 -99
  94. data/lib/aspera/transfer/sync.rb +273 -0
  95. data/lib/aspera/{fasp → transfer}/uri.rb +10 -9
  96. data/lib/aspera/web_server_simple.rb +12 -3
  97. data.tar.gz.sig +0 -0
  98. metadata +92 -68
  99. metadata.gz.sig +0 -0
  100. data/lib/aspera/aoc.rb +0 -606
  101. data/lib/aspera/ats_api.rb +0 -47
  102. data/lib/aspera/cos_node.rb +0 -93
  103. data/lib/aspera/fasp/agent_aspera.rb +0 -126
  104. data/lib/aspera/fasp/agent_base.rb +0 -48
  105. data/lib/aspera/fasp/agent_trsdk.rb +0 -146
  106. data/lib/aspera/fasp/resume_policy.rb +0 -77
  107. data/lib/aspera/node.rb +0 -338
  108. data/lib/aspera/sync.rb +0 -219
@@ -1,7 +1,10 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/fasp/transfer_spec'
3
+ require 'aspera/agent/base'
4
+ require 'aspera/transfer/spec'
4
5
  require 'aspera/cli/info'
6
+ require 'aspera/log'
7
+ require 'aspera/assert'
5
8
 
6
9
  module Aspera
7
10
  module Cli
@@ -28,7 +31,7 @@ module Aspera
28
31
  :FILE_LIST_FROM_TRANSFER_SPEC,
29
32
  :FILE_LIST_OPTIONS,
30
33
  :DEFAULT_TRANSFER_NOTIFY_TEMPLATE
31
- TRANSFER_AGENTS = %i[direct node connect httpgw trsdk].freeze
34
+ TRANSFER_AGENTS = Agent::Base.agent_list.freeze
32
35
 
33
36
  class << self
34
37
  # @return :success if all sessions statuses returned by "start" are success
@@ -52,6 +55,8 @@ module Aspera
52
55
  @agent = nil
53
56
  # source/destination pair, like "paths" of transfer spec
54
57
  @transfer_paths = nil
58
+ # HTTPGW URL provided by webapp
59
+ @httpgw_url_lambda = nil
55
60
  @opt_mgr.declare(:ts, 'Override transfer spec values', types: Hash, handler: {o: self, m: :option_transfer_spec})
56
61
  @opt_mgr.declare(:to_folder, 'Destination folder for transferred files')
57
62
  @opt_mgr.declare(:sources, "How list of transferred files is provided (#{FILE_LIST_OPTIONS.join(',')})")
@@ -65,7 +70,7 @@ module Aspera
65
70
 
66
71
  # multiple option are merged
67
72
  def option_transfer_spec=(value)
68
- raise 'option ts shall be a Hash' unless value.is_a?(Hash)
73
+ Aspera.assert_type(value, Hash){'ts'}
69
74
  @transfer_spec_command_line.deep_merge!(value)
70
75
  end
71
76
 
@@ -92,11 +97,12 @@ module Aspera
92
97
  end
93
98
 
94
99
  # analyze options and create new agent if not already created or set
95
- def set_agent_by_options
96
- return nil unless @agent.nil?
100
+ # TODO: make a Factory pattern
101
+ def agent_instance
102
+ return @agent unless @agent.nil?
97
103
  agent_type = @opt_mgr.get_option(:transfer, mandatory: true)
98
104
  # agent plugin is loaded on demand to avoid loading unnecessary dependencies
99
- require "aspera/fasp/agent_#{agent_type}"
105
+ require "aspera/agent/#{agent_type}"
100
106
  # set keys as symbols
101
107
  agent_options = @opt_mgr.get_option(:transfer_info).symbolize_keys
102
108
  # special cases
@@ -110,14 +116,20 @@ module Aspera
110
116
  when :direct
111
117
  # by default do not display ascp native progress bar
112
118
  agent_options[:quiet] = true unless agent_options.key?(:quiet)
113
- agent_options[:check_ignore] = ->(host, port){@config.ignore_cert?(host, port)}
114
- agent_options[:trusted_certs] = @config.trusted_cert_locations(files_only: true) unless agent_options.key?(:trusted_certs)
119
+ agent_options[:check_ignore_cb] = ->(host, port){@config.ignore_cert?(host, port)}
120
+ agent_options[:trusted_certs] = @config.trusted_cert_locations unless agent_options.key?(:trusted_certs)
121
+ when :httpgw
122
+ unless agent_options.key?(:url) || @httpgw_url_lambda.nil?
123
+ Log.log.debug('retrieving HTTPGW URL from webapp')
124
+ agent_options[:url] = @httpgw_url_lambda.call
125
+ end
115
126
  end
116
127
  agent_options[:progress] = @config.progress_bar
117
128
  # get agent instance
118
- new_agent = Kernel.const_get("Aspera::Fasp::Agent#{agent_type.capitalize}").new(agent_options)
129
+ new_agent = Kernel.const_get("Aspera::Agent::#{agent_type.capitalize}").new(agent_options)
119
130
  self.agent_instance = new_agent
120
- return nil
131
+ Log.log.debug{"transfer agent is a #{@agent.class}"}
132
+ return @agent
121
133
  end
122
134
 
123
135
  # return destination folder for transfers
@@ -131,9 +143,9 @@ module Aspera
131
143
  return dest_folder unless dest_folder.nil?
132
144
  # default: / on remote, . on local
133
145
  case direction.to_s
134
- when Fasp::TransferSpec::DIRECTION_SEND then dest_folder = '/'
135
- when Fasp::TransferSpec::DIRECTION_RECEIVE then dest_folder = '.'
136
- else raise "wrong direction: #{direction}"
146
+ when Transfer::Spec::DIRECTION_SEND then dest_folder = '/'
147
+ when Transfer::Spec::DIRECTION_RECEIVE then dest_folder = '.'
148
+ else Aspera.error_unexpected_value(direction)
137
149
  end
138
150
  return dest_folder
139
151
  end
@@ -145,6 +157,11 @@ module Aspera
145
157
  end
146
158
  end
147
159
 
160
+ def httpgw_url_cb=(httpgw_url_proc)
161
+ Aspera.assert_type(httpgw_url_proc, Proc){'httpgw_url_cb'}
162
+ @httpgw_url_lambda = httpgw_url_proc
163
+ end
164
+
148
165
  # This is how the list of files to be transferred is specified
149
166
  # get paths suitable for transfer spec from command line
150
167
  # @return [Hash] {source: (mandatory), destination: (optional)}
@@ -167,7 +184,7 @@ module Aspera
167
184
  Log.log.debug('assume list provided in transfer spec')
168
185
  special_case_direct_with_list =
169
186
  @opt_mgr.get_option(:transfer, mandatory: true).eql?(:direct) &&
170
- Fasp::Parameters.ts_has_ascp_file_list(@transfer_spec_command_line, @opt_mgr.get_option(:transfer_info))
187
+ Transfer::Parameters.ascp_args_file_list?(@opt_mgr.get_option(:transfer_info)['ascp_args'])
171
188
  raise Cli::BadArgument, 'transfer spec on command line must have sources' if @transfer_paths.nil? && !special_case_direct_with_list
172
189
  # here we assume check of sources is made in transfer agent
173
190
  return @transfer_paths
@@ -181,14 +198,15 @@ module Aspera
181
198
  if !@transfer_paths.nil?
182
199
  Log.log.warn('--sources overrides paths from --ts')
183
200
  end
184
- case @opt_mgr.get_option(:src_type, mandatory: true)
201
+ source_type = @opt_mgr.get_option(:src_type, mandatory: true)
202
+ case source_type
185
203
  when :list
186
204
  # when providing a list, just specify source
187
205
  @transfer_paths = file_list.map{|i|{'source' => i}}
188
206
  when :pair
189
- raise Cli::BadArgument, "When using pair, provide an even number of paths: #{file_list.length}" unless file_list.length.even?
207
+ Aspera.assert(file_list.length.even?, exception_class: Cli::BadArgument){"When using pair, provide an even number of paths: #{file_list.length}"}
190
208
  @transfer_paths = file_list.each_slice(2).to_a.map{|s, d|{'source' => s, 'destination' => d}}
191
- else raise 'Unsupported src_type'
209
+ else Aspera.error_unexpected_value(source_type)
192
210
  end
193
211
  Log.log.debug{"paths=#{@transfer_paths}"}
194
212
  return @transfer_paths
@@ -199,14 +217,14 @@ module Aspera
199
217
  # @param rest_token [Rest] if oauth token regeneration supported
200
218
  def start(transfer_spec, rest_token: nil)
201
219
  # check parameters
202
- raise 'transfer_spec must be hash' unless transfer_spec.is_a?(Hash)
220
+ Aspera.assert_type(transfer_spec, Hash){'transfer_spec'}
203
221
  # process :src option
204
222
  case transfer_spec['direction']
205
- when Fasp::TransferSpec::DIRECTION_RECEIVE
223
+ when Transfer::Spec::DIRECTION_RECEIVE
206
224
  # init default if required in any case
207
225
  @transfer_spec_command_line['destination_root'] ||= destination_folder(transfer_spec['direction'])
208
- when Fasp::TransferSpec::DIRECTION_SEND
209
- if transfer_spec.dig('tags', Fasp::TransferSpec::TAG_RESERVED, 'node', 'access_key')
226
+ when Transfer::Spec::DIRECTION_SEND
227
+ if transfer_spec.dig('tags', Transfer::Spec::TAG_RESERVED, 'node', 'access_key')
210
228
  # gen4
211
229
  @transfer_spec_command_line.delete('destination_root') if @transfer_spec_command_line.key?('destination_root_id')
212
230
  elsif transfer_spec.key?('token')
@@ -223,12 +241,14 @@ module Aspera
223
241
  @transfer_spec_command_line['paths'] = transfer_spec['paths'] || ts_source_paths
224
242
  # updated transfer spec with command line
225
243
  updated_ts(transfer_spec)
244
+ # if TS from app has content_protection (e.g. F5), that means content is protected: ask password if not provided
245
+ if transfer_spec['content_protection'].eql?('decrypt') && !transfer_spec.key?('content_protection_password')
246
+ transfer_spec['content_protection_password'] = @opt_mgr.prompt_user_input('content protection password', true)
247
+ end
226
248
  # create transfer agent
227
- set_agent_by_options
228
- Log.log.debug{"transfer agent is a #{@agent.class}"}
229
- @agent.start_transfer(transfer_spec, token_regenerator: rest_token)
249
+ agent_instance.start_transfer(transfer_spec, token_regenerator: rest_token)
230
250
  # list of: :success or "error message string"
231
- result = @agent.wait_for_completion
251
+ result = agent_instance.wait_for_completion
232
252
  send_email_transfer_notification(transfer_spec, result)
233
253
  return result
234
254
  end
@@ -1,6 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require 'aspera/log'
4
+ require 'aspera/assert'
4
5
  require 'ruby-progressbar'
5
6
 
6
7
  module Aspera
@@ -25,9 +26,7 @@ module Aspera
25
26
 
26
27
  def event(session_id:, type:, info: nil)
27
28
  Log.log.debug{"progress: #{type} #{session_id} #{info}"}
28
- if session_id.nil? && !type.eql?(:pre_start)
29
- raise 'Internal error: session_id is nil'
30
- end
29
+ Aspera.assert(!session_id.nil? || type.eql?(:pre_start)){'session_id is nil'}
31
30
  return if @completed
32
31
  if @progress_bar.nil?
33
32
  @progress_bar = ProgressBar.create(
@@ -4,6 +4,6 @@ module Aspera
4
4
  module Cli
5
5
  # for beta add extension : .beta1
6
6
  # for dev version add extension : .pre
7
- VERSION = '4.15.0'
7
+ VERSION = '4.17.0'
8
8
  end
9
9
  end
data/lib/aspera/colors.rb CHANGED
@@ -53,4 +53,9 @@ class String
53
53
  define_method(name){self}
54
54
  end
55
55
  end
56
+ def capital_to_snake
57
+ return gsub(/([a-z\d])([A-Z])/, '\1_\2')
58
+ .gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
59
+ .downcase
60
+ end
56
61
  end
@@ -1,5 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require 'aspera/log'
4
+ require 'aspera/assert'
3
5
  module Aspera
4
6
  # helper class to build command line from a parameter list (key-value hash)
5
7
  # constructor takes hash: { 'param1':'value1', ...}
@@ -27,20 +29,20 @@ module Aspera
27
29
  # Called by provider of definition before constructor of this class so that params_definition has all mandatory fields
28
30
  def normalize_description(full_description)
29
31
  full_description.each do |name, options|
30
- raise "Expecting Hash, but have #{options.class} in #{name}" unless options.is_a?(Hash)
32
+ Aspera.assert_type(options, Hash){name}
31
33
  unsupported_keys = options.keys - OPTIONS_KEYS
32
- raise "Unsupported definition keys: #{unsupported_keys}" unless unsupported_keys.empty?
33
- raise "Missing key: cli for #{name}" unless options.key?(:cli)
34
- raise 'Key: cli must be Hash' unless options[:cli].is_a?(Hash)
35
- raise 'Missing key: cli.type' unless options[:cli].key?(:type)
36
- raise "Unsupported processing type for #{name}: #{options[:cli][:type]}" unless CLI_OPTION_TYPES.include?(options[:cli][:type])
34
+ Aspera.assert(unsupported_keys.empty?){"Unsupported definition keys: #{unsupported_keys}"}
35
+ Aspera.assert(options.key?(:cli)){"Missing key: cli for #{name}"}
36
+ Aspera.assert_type(options[:cli], Hash){'Key: cli'}
37
+ Aspera.assert(options[:cli].key?(:type)){'Missing key: cli.type'}
38
+ Aspera.assert_values(options[:cli][:type], CLI_OPTION_TYPES){"Unsupported processing type for #{name}"}
37
39
  # by default : optional
38
40
  options[:mandatory] ||= false
39
41
  options[:desc] ||= ''
40
42
  options[:desc] = "DEPRECATED: #{options[:deprecation]}\n#{options[:desc]}" if options.key?(:deprecation)
41
43
  cli = options[:cli]
42
44
  unsupported_cli_keys = cli.keys - CLI_KEYS
43
- raise "Unsupported cli keys: #{unsupported_cli_keys}" unless unsupported_cli_keys.empty?
45
+ Aspera.assert(unsupported_cli_keys.empty?){"Unsupported cli keys: #{unsupported_cli_keys}"}
44
46
  # by default : string, unless it's without arg
45
47
  options[:accepted_types] ||= options[:cli][:type].eql?(:opt_without_arg) ? :bool : :string
46
48
  # single type is placed in array
@@ -109,7 +111,7 @@ module Aspera
109
111
  end
110
112
  processing_type = read ? :get_value : options[:cli][:type]
111
113
  # check mandatory parameter (nil is valid value)
112
- raise Fasp::Error, "Missing mandatory parameter: #{name}" if options[:mandatory] && !@param_hash.key?(name)
114
+ raise Transfer::Error, "Missing mandatory parameter: #{name}" if options[:mandatory] && !@param_hash.key?(name)
113
115
  parameter_value = @param_hash[name]
114
116
  # no default setting
115
117
  # parameter_value=options[:default] if parameter_value.nil? and options.has_key?(:default)
@@ -121,11 +123,11 @@ module Aspera
121
123
  when :hash then Hash
122
124
  when :int then Integer
123
125
  when :bool then [TrueClass, FalseClass]
124
- else raise "INTERNAL: unexpected value: #{type_symbol}"
126
+ else Aspera.error_unexpected_value(type_symbol)
125
127
  end
126
128
  end.flatten
127
129
  # check that value is of expected type
128
- raise Fasp::Error, "#{name} is : #{parameter_value.class} (#{parameter_value}), shall be #{options[:accepted_types]}, " \
130
+ raise Transfer::Error, "#{name} is : #{parameter_value.class} (#{parameter_value}), shall be #{options[:accepted_types]}, " \
129
131
  unless parameter_value.nil? || expected_classes.include?(parameter_value.class)
130
132
  # special processing will be requested with type get_value
131
133
  @used_param_names.push(name) unless processing_type.eql?(:special)
@@ -147,10 +149,10 @@ module Aspera
147
149
  # :convert has name of class and encoding method
148
150
  conversion_class, conversion_method = options[:cli][:convert].split('.')
149
151
  converted_value = Kernel.const_get(conversion_class).send(conversion_method, parameter_value)
150
- raise Fasp::Error, "unsupported #{name}: #{parameter_value}" if converted_value.nil?
152
+ raise Transfer::Error, "unsupported #{name}: #{parameter_value}" if converted_value.nil?
151
153
  parameter_value = converted_value
152
154
  when NilClass
153
- else raise "not expected type for convert #{options[:cli][:convert].class} for #{name}"
155
+ else Aspera.error_unexpected_value(options[:cli][:convert].class)
154
156
  end
155
157
 
156
158
  case processing_type
@@ -159,14 +161,14 @@ module Aspera
159
161
  when :ignore, :special # ignore this parameter or process later
160
162
  return
161
163
  when :envvar # set in env var
162
- raise 'error' unless options[:cli].key?(:variable)
164
+ Aspera.assert(options[:cli].key?(:variable)){'missing key: cli.variable'}
163
165
  @result[:env][options[:cli][:variable]] = parameter_value
164
166
  when :opt_without_arg # if present and true : just add option without value
165
167
  add_param = false
166
168
  case parameter_value
167
169
  when false then nil # nothing to put on command line, no creation by default
168
170
  when true then add_param = true
169
- else raise Fasp::Error, "unsupported #{name}: #{parameter_value}"
171
+ else Aspera.error_unexpected_value(parameter_value){name}
170
172
  end
171
173
  add_param = !add_param if options[:add_on_false]
172
174
  add_command_line_options([options[:cli][:switch]]) if add_param
@@ -0,0 +1,21 @@
1
+ # frozen_string_literal: true
2
+
3
+ # coverage for tests
4
+ if ENV.key?('ENABLE_COVERAGE')
5
+ require 'simplecov'
6
+ require 'securerandom'
7
+ # compute development top folder based on this source location
8
+ development_root = 3.times.inject(File.realpath(__FILE__)) { |p, _| File.dirname(p) }
9
+ SimpleCov.root(development_root)
10
+ SimpleCov.enable_for_subprocesses if SimpleCov.respond_to?(:enable_for_subprocesses)
11
+ # keep cache data for 1 day (must be longer that time to run the whole test suite)
12
+ SimpleCov.merge_timeout(86400)
13
+ SimpleCov.command_name(SecureRandom.uuid)
14
+ SimpleCov.at_exit do
15
+ original_file_descriptor = $stdout
16
+ $stdout.reopen(File.join(development_root, 'simplecov.log'))
17
+ SimpleCov.result.format!
18
+ $stdout.reopen(original_file_descriptor)
19
+ end
20
+ SimpleCov.start
21
+ end
@@ -1,15 +1,46 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/log'
3
+ require 'aspera/assert'
4
4
  require 'singleton'
5
+ require 'openssl'
5
6
 
6
7
  module Aspera
7
8
  # a simple binary data repository
8
9
  class DataRepository
9
10
  include Singleton
11
+ # in same order as elements in folder
12
+ ELEMENTS = %i[dsa rsa uuid aspera.global-cli-client aspera.drive license]
13
+ START_INDEX = 1
14
+ DATA_FOLDER_NAME = 'data'
15
+
16
+ # decode data as expected as string
17
+ # @param name [Symbol] name of the data item
18
+ # @return [String] decoded data
19
+ def item(name)
20
+ index = ELEMENTS.index(name)
21
+ raise ArgumentError, "unknown data item #{name} (#{name.class})" unless index
22
+ raw_data = data(START_INDEX + index)
23
+ case name
24
+ when :dsa, :rsa
25
+ # generate PEM from DER
26
+ return OpenSSL::PKey.const_get(name.to_s.upcase).new(raw_data).to_pem
27
+ when :license
28
+ return Zlib::Inflate.inflate(raw_data)
29
+ when :uuid
30
+ return format('%08x-%04x-%04x-%04x-%04x%08x', *raw_data.unpack('NnnnnN'))
31
+ when :'aspera.global-cli-client', :'aspera.drive'
32
+ return Base64.urlsafe_encode64(raw_data)
33
+ else Aspera.error_unexpected_value(name)
34
+ end
35
+ end
36
+
37
+ private
38
+
39
+ private_constant :START_INDEX, :DATA_FOLDER_NAME
40
+
10
41
  # get binary value from data repository
11
42
  def data(id)
12
- File.read(File.join(__dir__, 'data', id.to_s), mode: 'rb')
43
+ File.read(File.join(__dir__, DATA_FOLDER_NAME, id.to_s), mode: 'rb')
13
44
  end
14
45
  end
15
46
  end
@@ -2,6 +2,7 @@
2
2
 
3
3
  # cspell:ignore USERPROFILE HOMEDRIVE HOMEPATH LC_CTYPE msys aarch
4
4
  require 'aspera/log'
5
+ require 'aspera/assert'
5
6
  require 'rbconfig'
6
7
 
7
8
  # cspell:words MEBI mswin bccwin
@@ -15,10 +16,11 @@ module Aspera
15
16
  OS_AIX = :aix
16
17
  OS_LIST = [OS_WINDOWS, OS_X, OS_LINUX, OS_AIX].freeze
17
18
  CPU_X86_64 = :x86_64
19
+ CPU_ARM64 = :arm64
18
20
  CPU_PPC64 = :ppc64
19
21
  CPU_PPC64LE = :ppc64le
20
22
  CPU_S390 = :s390
21
- CPU_LIST = [CPU_X86_64, CPU_PPC64, CPU_PPC64LE, CPU_S390].freeze
23
+ CPU_LIST = [CPU_X86_64, CPU_ARM64, CPU_PPC64, CPU_PPC64LE, CPU_S390].freeze
22
24
 
23
25
  BITS_PER_BYTE = 8
24
26
  MEBI = 1024 * 1024
@@ -54,8 +56,7 @@ module Aspera
54
56
  when /s390/
55
57
  return CPU_S390
56
58
  when /arm/, /aarch64/
57
- # arm on mac has rosetta 2
58
- return CPU_X86_64 if os.eql?(OS_X)
59
+ return CPU_ARM64
59
60
  end
60
61
  raise "Unknown CPU: #{RbConfig::CONFIG['host_cpu']}"
61
62
  end
@@ -88,7 +89,7 @@ module Aspera
88
89
 
89
90
  # value is provided in block
90
91
  def write_file_restricted(path, force: false, mode: nil)
91
- raise 'coding error, missing content block' unless block_given?
92
+ Aspera.assert(block_given?, exception_class: Aspera::InternalError)
92
93
  if force || !File.exist?(path)
93
94
  # Windows may give error
94
95
  File.unlink(path) rescue nil
@@ -1,17 +1,18 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/web_server_simple'
4
3
  require 'aspera/log'
4
+ require 'aspera/assert'
5
+ require 'webrick'
5
6
  require 'json'
6
7
 
7
8
  module Aspera
8
- # this class answers the Faspex /send API and creates a package on Aspera on Cloud
9
+ # Simulate the Faspex 4 /send API and creates a package on Aspera on Cloud or Faspex 5
9
10
  class Faspex4GWServlet < WEBrick::HTTPServlet::AbstractServlet
10
- # @param app_api [Aspera::AoC]
11
+ # @param app_api
11
12
  # @param app_context [String]
12
13
  def initialize(server, app_api, app_context)
14
+ Aspera.assert_values(app_api.class.name, ['Aspera::Api::AoC', 'Aspera::Rest'])
13
15
  super(server)
14
- # typed: Aspera::AoC
15
16
  @app_api = app_api
16
17
  @app_context = app_context
17
18
  end
@@ -49,7 +50,7 @@ module Aspera
49
50
  operation: 'POST',
50
51
  subpath: "packages/#{package['id']}/transfer_spec/upload",
51
52
  headers: {'Accept' => 'application/json'},
52
- url_params: {transfer_type: Aspera::Cli::Plugins::Faspex5::TRANSFER_CONNECT},
53
+ url_params: {transfer_type: Cli::Plugins::Faspex5::TRANSFER_CONNECT},
53
54
  json_params: {paths: [{'destination'=>'/'}]}
54
55
  )[:data]
55
56
  transfer_spec.delete('authentication')
@@ -67,13 +68,14 @@ module Aspera
67
68
  raise 'no payload' if request.body.nil?
68
69
  faspex_pkg_parameters = JSON.parse(request.body)
69
70
  Log.log.debug{"faspex pkg create parameters=#{faspex_pkg_parameters}"}
71
+ # compare string, as class is not yet known here
70
72
  faspex_package_create_result =
71
- if @app_api.class.name.eql?('Aspera::AoC')
73
+ case @app_api.class.name
74
+ when 'Aspera::Api::AoC'
72
75
  faspex4_send_to_aoc(faspex_pkg_parameters)
73
- elsif @app_api.class.name.eql?('Aspera::Rest')
76
+ when 'Aspera::Rest'
74
77
  faspex4_send_to_faspex5(faspex_pkg_parameters)
75
- else
76
- raise "No such adapter: #{@app_api.class}"
78
+ else Aspera.error_unexpected_value(@app_api.class.name)
77
79
  end
78
80
  Log.log.info{"faspex_package_create_result=#{faspex_package_create_result}"}
79
81
  response.status = 200
@@ -89,8 +91,8 @@ module Aspera
89
91
  else
90
92
  response.status = 400
91
93
  response['Content-Type'] = 'application/json'
92
- response.body = {error: 'Bad request'}.to_json
94
+ response.body = {error: 'Unsupported endpoint'}.to_json
93
95
  end
94
96
  end
95
97
  end # Faspex4GWServlet
96
- end # AsperaLm
98
+ end # Aspera
@@ -1,17 +1,18 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'English'
4
- require 'aspera/web_server_simple'
5
- require 'aspera/log'
6
3
  require 'json'
7
4
  require 'timeout'
5
+ require 'English'
6
+ require 'webrick'
7
+ require 'aspera/log'
8
+ require 'aspera/assert'
8
9
 
9
10
  module Aspera
10
11
  # this class answers the Faspex /send API and creates a package on Aspera on Cloud
11
12
  class Faspex4PostProcServlet < WEBrick::HTTPServlet::AbstractServlet
12
13
  ALLOWED_PARAMETERS = %i[root script_folder fail_on_error timeout_seconds].freeze
13
14
  def initialize(server, parameters)
14
- raise 'parameters must be Hash' unless parameters.is_a?(Hash)
15
+ Aspera.assert_type(parameters, Hash)
15
16
  @parameters = parameters.symbolize_keys
16
17
  Log.log.debug{Log.dump(:post_proc_parameters, @parameters)}
17
18
  raise "unexpected key in parameters config: only: #{ALLOWED_PARAMETERS.join(', ')}" if @parameters.keys.any?{|k|!ALLOWED_PARAMETERS.include?(k)}
@@ -74,4 +75,4 @@ module Aspera
74
75
  end
75
76
  end
76
77
  end # Faspex4PostProcServlet
77
- end # AsperaLm
78
+ end # Aspera
@@ -1,5 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require 'aspera/assert'
3
4
  require 'uri'
4
5
 
5
6
  module Aspera
@@ -11,14 +12,15 @@ module Aspera
11
12
  class << self
12
13
  def from_list(object_id)
13
14
  if object_id.is_a?(Array)
15
+ # compact: remove nils
14
16
  object_id = object_id.compact.map do |i|
15
17
  i.is_a?(String) && i.start_with?('https://') ? URI.parse(i).host : i.to_s
16
18
  end.join(ID_SEPARATOR)
17
19
  end
18
- raise 'id must be a String' unless object_id.is_a?(String)
20
+ Aspera.assert_type(object_id, String)
19
21
  return object_id
20
22
  .gsub(WINDOWS_PROTECTED_CHAR, PROTECTED_CHAR_REPLACE) # remove windows forbidden chars
21
- .gsub('.', PROTECTED_CHAR_REPLACE) # keep dot for extension only (nicer)
23
+ .gsub('.', PROTECTED_CHAR_REPLACE) # keep dot for extension only (nicer)
22
24
  .downcase
23
25
  end
24
26
  end
@@ -3,6 +3,7 @@
3
3
  # cspell:ignore blankslate
4
4
 
5
5
  require 'aspera/rest_error_analyzer'
6
+ require 'aspera/assert'
6
7
  require 'blankslate'
7
8
 
8
9
  Aspera::RestErrorAnalyzer.instance.add_simple_handler(name: 'JSON RPC', path: %w[error message], always: true)
@@ -34,15 +35,16 @@ module Aspera
34
35
  params: args,
35
36
  id: @request_id += 1
36
37
  })[:data]
37
- raise 'response shall be Hash' unless data.is_a?(Hash)
38
- raise 'bad version in response' unless data['jsonrpc'] == JSON_RPC_VERSION
39
- raise 'missing id in response' unless data.key?('id')
40
- raise 'both error and response' if data.key?('error') && data.key?('result')
41
- raise 'bad error response' unless
38
+ Aspera.assert_type(data, Hash){'response'}
39
+ Aspera.assert(data['jsonrpc'] == JSON_RPC_VERSION){'bad version in response'}
40
+ Aspera.assert(data.key?('id')){'missing id in response'}
41
+ Aspera.assert(!(data.key?('error') && data.key?('result'))){'both error and response'}
42
+ Aspera.assert(
42
43
  !data.key?('error') ||
43
- data['error'].is_a?(Hash) &&
44
- data['error']['code'].is_a?(Integer) &&
45
- data['error']['message'].is_a?(String)
44
+ data['error'].is_a?(Hash) &&
45
+ data['error']['code'].is_a?(Integer) &&
46
+ data['error']['message'].is_a?(String)
47
+ ){'bad error response'}
46
48
  return data['result']
47
49
  end
48
50
  end
@@ -2,6 +2,8 @@
2
2
 
3
3
  require 'aspera/hash_ext'
4
4
  require 'aspera/environment'
5
+ require 'aspera/log'
6
+ require 'aspera/assert'
5
7
  require 'symmetric_encryption/core'
6
8
  require 'yaml'
7
9
 
@@ -9,33 +11,66 @@ module Aspera
9
11
  module Keychain
10
12
  # Manage secrets in a simple Hash
11
13
  class EncryptedHash
12
- CIPHER_NAME = 'aes-256-cbc'
14
+ LEGACY_CIPHER_NAME = 'aes-256-cbc'
15
+ DEFAULT_CIPHER_NAME = 'aes-256-cbc'
16
+ FILE_TYPE = 'encrypted_hash_vault'
13
17
  CONTENT_KEYS = %i[label username password url description].freeze
18
+ FILE_KEYS = %w[version type cipher data].sort.freeze
14
19
  def initialize(path, current_password)
20
+ Aspera.assert_type(path, String){'path to vault file'}
15
21
  @path = path
22
+ @all_secrets = {}
23
+ vault_encrypted_data = nil
24
+ if File.exist?(@path)
25
+ vault_file = File.read(@path)
26
+ if vault_file.start_with?('---')
27
+ vault_info = YAML.parse(vault_file).to_ruby
28
+ Aspera.assert(vault_info.keys.sort == FILE_KEYS){'Invalid vault file'}
29
+ @cipher_name = vault_info['cipher']
30
+ vault_encrypted_data = vault_info['data']
31
+ else
32
+ # legacy vault file
33
+ @cipher_name = LEGACY_CIPHER_NAME
34
+ vault_encrypted_data = File.read(@path, mode: 'rb')
35
+ end
36
+ end
37
+ # setting password also creates the cipher
16
38
  self.password = current_password
17
- raise 'path to vault file shall be String' unless @path.is_a?(String)
18
- @all_secrets = File.exist?(@path) ? YAML.load_stream(@cipher.decrypt(File.read(@path))).first : {}
39
+ if !vault_encrypted_data.nil?
40
+ @all_secrets = YAML.load_stream(@cipher.decrypt(vault_encrypted_data)).first
41
+ end
19
42
  end
20
43
 
44
+ # set the password and cipher
21
45
  def password=(new_password)
22
46
  # number of bits in second position
23
- key_bytes = CIPHER_NAME.split('-')[1].to_i / Environment::BITS_PER_BYTE
47
+ key_bytes = DEFAULT_CIPHER_NAME.split('-')[1].to_i / Environment::BITS_PER_BYTE
24
48
  # derive key from passphrase, add trailing zeros
25
49
  key = "#{new_password}#{"\x0" * key_bytes}"[0..(key_bytes - 1)]
26
- Log.log.debug{"key=[#{key}],#{key.length}"}
27
- SymmetricEncryption.cipher = @cipher = SymmetricEncryption::Cipher.new(cipher_name: CIPHER_NAME, key: key, encoding: :none)
50
+ Log.log.trace1{"secret=[#{key}],#{key.length}"}
51
+ @cipher = SymmetricEncryption.cipher = SymmetricEncryption::Cipher.new(cipher_name: DEFAULT_CIPHER_NAME, key: key, encoding: :none)
28
52
  end
29
53
 
54
+ # save current data to file with format
30
55
  def save
31
- File.write(@path, @cipher.encrypt(YAML.dump(@all_secrets)), encoding: 'BINARY')
56
+ vault_info = {
57
+ 'version' => '1.0.0',
58
+ 'type' => FILE_TYPE,
59
+ 'cipher' => @cipher_name,
60
+ 'data' => @cipher.encrypt(YAML.dump(@all_secrets))
61
+ }
62
+ File.write(@path, YAML.dump(vault_info))
32
63
  end
33
64
 
65
+ # set a secret
66
+ # @param options [Hash] with keys :label, :username, :password, :url, :description
34
67
  def set(options)
35
- raise 'options shall be Hash' unless options.is_a?(Hash)
68
+ Aspera.assert_type(options, Hash){'options'}
36
69
  unsupported = options.keys - CONTENT_KEYS
37
- options.each_value {|v| raise 'value must be String' unless v.is_a?(String)}
38
- raise "unsupported options: #{unsupported}" unless unsupported.empty?
70
+ Aspera.assert(unsupported.empty?){"unsupported options: #{unsupported}"}
71
+ options.each_pair do |k, v|
72
+ Aspera.assert_type(v, String){k.to_s}
73
+ end
39
74
  label = options.delete(:label)
40
75
  raise "secret #{label} already exist, delete first" if @all_secrets.key?(label)
41
76
  @all_secrets[label] = options.symbolize_keys
@@ -59,7 +94,7 @@ module Aspera
59
94
  end
60
95
 
61
96
  def get(label:, exception: true)
62
- raise "Label not found: #{label}" unless @all_secrets.key?(label) || !exception
97
+ Aspera.assert(@all_secrets.key?(label)){"Label not found: #{label}"} if exception
63
98
  result = @all_secrets[label].clone
64
99
  result[:label] = label if result.is_a?(Hash)
65
100
  return result