aspera-cli 4.15.0 → 4.16.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (80) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +29 -3
  4. data/CHANGELOG.md +292 -228
  5. data/CONTRIBUTING.md +69 -18
  6. data/README.md +1102 -952
  7. data/bin/ascli +13 -31
  8. data/bin/asession +3 -1
  9. data/examples/dascli +2 -2
  10. data/lib/aspera/aoc.rb +28 -33
  11. data/lib/aspera/ascmd.rb +3 -6
  12. data/lib/aspera/assert.rb +45 -0
  13. data/lib/aspera/cli/extended_value.rb +5 -5
  14. data/lib/aspera/cli/formatter.rb +26 -13
  15. data/lib/aspera/cli/hints.rb +4 -3
  16. data/lib/aspera/cli/main.rb +16 -3
  17. data/lib/aspera/cli/manager.rb +45 -36
  18. data/lib/aspera/cli/plugin.rb +20 -13
  19. data/lib/aspera/cli/plugins/aoc.rb +103 -73
  20. data/lib/aspera/cli/plugins/ats.rb +4 -3
  21. data/lib/aspera/cli/plugins/config.rb +114 -119
  22. data/lib/aspera/cli/plugins/cos.rb +2 -2
  23. data/lib/aspera/cli/plugins/faspex.rb +23 -19
  24. data/lib/aspera/cli/plugins/faspex5.rb +75 -43
  25. data/lib/aspera/cli/plugins/node.rb +28 -15
  26. data/lib/aspera/cli/plugins/orchestrator.rb +4 -2
  27. data/lib/aspera/cli/plugins/preview.rb +9 -7
  28. data/lib/aspera/cli/plugins/server.rb +6 -3
  29. data/lib/aspera/cli/plugins/shares.rb +30 -26
  30. data/lib/aspera/cli/sync_actions.rb +9 -9
  31. data/lib/aspera/cli/transfer_agent.rb +21 -14
  32. data/lib/aspera/cli/transfer_progress.rb +2 -3
  33. data/lib/aspera/cli/version.rb +1 -1
  34. data/lib/aspera/command_line_builder.rb +13 -11
  35. data/lib/aspera/cos_node.rb +3 -2
  36. data/lib/aspera/coverage.rb +22 -0
  37. data/lib/aspera/data_repository.rb +33 -2
  38. data/lib/aspera/environment.rb +4 -2
  39. data/lib/aspera/fasp/{agent_aspera.rb → agent_alpha.rb} +29 -39
  40. data/lib/aspera/fasp/agent_base.rb +17 -7
  41. data/lib/aspera/fasp/agent_direct.rb +88 -84
  42. data/lib/aspera/fasp/agent_httpgw.rb +4 -3
  43. data/lib/aspera/fasp/agent_node.rb +3 -2
  44. data/lib/aspera/fasp/agent_trsdk.rb +79 -37
  45. data/lib/aspera/fasp/installation.rb +51 -12
  46. data/lib/aspera/fasp/management.rb +11 -6
  47. data/lib/aspera/fasp/parameters.rb +53 -47
  48. data/lib/aspera/fasp/resume_policy.rb +7 -5
  49. data/lib/aspera/fasp/sync.rb +273 -0
  50. data/lib/aspera/fasp/transfer_spec.rb +10 -8
  51. data/lib/aspera/fasp/uri.rb +2 -2
  52. data/lib/aspera/faspex_gw.rb +11 -8
  53. data/lib/aspera/faspex_postproc.rb +6 -5
  54. data/lib/aspera/id_generator.rb +3 -1
  55. data/lib/aspera/json_rpc.rb +10 -8
  56. data/lib/aspera/keychain/encrypted_hash.rb +46 -11
  57. data/lib/aspera/keychain/macos_security.rb +15 -13
  58. data/lib/aspera/log.rb +4 -3
  59. data/lib/aspera/nagios.rb +7 -2
  60. data/lib/aspera/node.rb +17 -16
  61. data/lib/aspera/node_simulator.rb +214 -0
  62. data/lib/aspera/oauth.rb +22 -19
  63. data/lib/aspera/persistency_action_once.rb +13 -14
  64. data/lib/aspera/persistency_folder.rb +3 -2
  65. data/lib/aspera/preview/file_types.rb +53 -267
  66. data/lib/aspera/preview/generator.rb +7 -5
  67. data/lib/aspera/preview/terminal.rb +14 -5
  68. data/lib/aspera/preview/utils.rb +8 -7
  69. data/lib/aspera/proxy_auto_config.rb +6 -3
  70. data/lib/aspera/rest.rb +29 -13
  71. data/lib/aspera/rest_error_analyzer.rb +1 -0
  72. data/lib/aspera/rest_errors_aspera.rb +2 -0
  73. data/lib/aspera/secret_hider.rb +5 -2
  74. data/lib/aspera/ssh.rb +10 -8
  75. data/lib/aspera/temp_file_manager.rb +1 -1
  76. data/lib/aspera/web_server_simple.rb +2 -1
  77. data.tar.gz.sig +0 -0
  78. metadata +96 -45
  79. metadata.gz.sig +0 -0
  80. data/lib/aspera/sync.rb +0 -219
@@ -1,7 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require 'aspera/cli/plugins/node'
4
-
4
+ require 'aspera/assert'
5
5
  module Aspera
6
6
  module Cli
7
7
  module Plugins
@@ -49,8 +49,6 @@ module Aspera
49
49
 
50
50
  def initialize(env)
51
51
  super(env)
52
- options.declare(:type, 'Type of user/group for operations', values: %i[any local ldap saml], default: :any)
53
- options.parse_options!
54
52
  end
55
53
 
56
54
  SAML_IMPORT_MANDATORY = %w[id name_id].freeze
@@ -84,63 +82,69 @@ module Aspera
84
82
  return Node.new(@agents, api: api_shares_node).execute_action(repo_command)
85
83
  when :admin
86
84
  api_shares_admin = basic_auth_api('api/v1')
87
- admin_command = options.get_next_command(%i[user group share node].freeze)
85
+ admin_command = options.get_next_command(%i[node share transfer_settings user group].freeze)
88
86
  case admin_command
89
87
  when :node
90
88
  return entity_action(api_shares_admin, 'data/nodes')
89
+ when :share
90
+ share_command = options.get_next_command(%i[user_permissions group_permissions].concat(Plugin::ALL_OPS))
91
+ case share_command
92
+ when *Plugin::ALL_OPS
93
+ return entity_command(share_command, api_shares_admin, 'data/shares')
94
+ # return {type: :object_list, data: all_shares, fields: %w[id name status status_message]}
95
+ when :user_permissions, :group_permissions
96
+ share_id = instance_identifier
97
+ return entity_action(api_shares_admin, "data/shares/#{share_id}/#{share_command}")
98
+ end
99
+ when :transfer_settings
100
+ xfer_settings_command = options.get_next_command(%i[show modify])
101
+ return entity_command(xfer_settings_command, api_shares_admin, 'data/transfer_settings', is_singleton: true)
91
102
  when :user, :group
92
103
  entity_type = admin_command
93
- entities_location = options.get_option(:type, mandatory: true)
104
+ entities_location = options.get_next_command(%i[all local ldap saml])
94
105
  entities_path = "data/#{entities_location}_#{entity_type}s"
95
106
  entity_action = nil
96
107
  case entities_location
97
- when :any
108
+ when :all
98
109
  entities_path = "data/#{entity_type}s"
99
110
  entity_action = %i[list show delete]
100
111
  entity_action.concat(USR_GRP_SETTINGS)
101
112
  entity_action.push(:users) if entity_type.eql?(:group)
102
113
  entity_action.freeze
103
114
  when :local
104
- entity_action = %i[list show create modify delete].freeze
115
+ entity_action = %i[list show delete create modify].freeze
105
116
  when :ldap
106
117
  entity_action = %i[add].freeze
107
118
  when :saml
108
119
  entity_action = %i[import].freeze
109
120
  end
110
121
  entity_verb = options.get_next_command(entity_action)
111
- # entity_path = "#{entities_path}/#{instance_identifier}" if %i[app_authorizations share_permissions].include?(entity_verb)
112
122
  case entity_verb
113
- when *Plugin::ALL_OPS
123
+ when *Plugin::ALL_OPS # list, show, delete, create, modify
114
124
  display_fields = entity_type.eql?(:user) ? %w[id username first_name last_name email] : nil
115
- display_fields.push(:directory_user) if entity_type.eql?(:user) && entities_location.eql?(:any)
125
+ display_fields.push(:directory_user) if entity_type.eql?(:user) && entities_location.eql?(:all)
116
126
  return entity_command(entity_verb, api_shares_admin, entities_path, display_fields: display_fields)
117
- when :import
127
+ when *USR_GRP_SETTINGS # transfer_settings, app_authorizations, share_permissions
128
+ group_id = instance_identifier
129
+ entities_path = "#{entities_path}/#{group_id}/#{entity_verb}"
130
+ return entity_action(api_shares_admin, entities_path, is_singleton: !entity_verb.eql?(:share_permissions))
131
+ when :import # saml
118
132
  return do_bulk_operation(command: entity_verb, descr: 'user information') do |entity_parameters|
119
133
  entity_parameters = entity_parameters.transform_keys{|k|k.gsub(/\s+/, '_').downcase}
120
- raise 'expecting Hash' unless entity_parameters.is_a?(Hash)
134
+ assert_type(entity_parameters, Hash)
121
135
  SAML_IMPORT_MANDATORY.each{|p|raise "missing mandatory field: #{p}" if entity_parameters[p].nil?}
122
136
  entity_parameters.each_key do |p|
123
137
  raise "unsupported field: #{p}, use: #{SAML_IMPORT_ALLOWED.join(',')}" unless SAML_IMPORT_ALLOWED.include?(p)
124
138
  end
125
139
  api_shares_admin.create("#{entities_path}/import", entity_parameters)[:data]
126
140
  end
127
- when :add
141
+ when :add # ldap
128
142
  return do_bulk_operation(command: entity_verb, descr: "#{entity_type} name", values: String) do |entity_name|
129
143
  api_shares_admin.create(entities_path, {entity_type=>entity_name})[:data]
130
144
  end
131
- when *USR_GRP_SETTINGS
132
- group_id = instance_identifier
133
- entities_path = "#{entities_path}/#{group_id}/#{entity_verb}"
134
- return entity_action(api_shares_admin, entities_path, is_singleton: !entity_verb.eql?(:share_permissions))
135
- end
136
- when :share
137
- share_command = options.get_next_command(%i[user_permissions group_permissions].concat(Plugin::ALL_OPS))
138
- case share_command
139
- when *Plugin::ALL_OPS
140
- return entity_command(share_command, api_shares_admin, 'data/shares')
141
- # return {type: :object_list, data: all_shares, fields: %w[id name status status_message]}
142
- when :user_permissions, :group_permissions
143
- return entity_action(api_shares_admin, "data/shares/#{instance_identifier}/#{share_command}")
145
+ when :users # group
146
+ raise "TODO, not implemented"
147
+ else error_unexpected_value(entity_verb)
144
148
  end
145
149
  end
146
150
  end
@@ -1,13 +1,14 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/sync'
3
+ require 'aspera/fasp/sync'
4
+ require 'aspera/assert'
4
5
 
5
6
  module Aspera
6
7
  module Cli
7
8
  # Module for sync actions
8
9
  module SyncActions
9
10
  SIMPLE_ARGUMENTS_SYNC = {
10
- direction: Aspera::Sync::DIRECTIONS,
11
+ direction: Aspera::Fasp::Sync::DIRECTIONS,
11
12
  local_dir: String,
12
13
  remote_dir: String
13
14
  }.stringify_keys.freeze
@@ -19,8 +20,7 @@ module Aspera
19
20
  end
20
21
 
21
22
  def execute_sync_action(&block)
22
- # options = Aspera::Cli::Manager.new
23
- raise 'Internal Error: No block given' unless block
23
+ assert(block){'No block given'}
24
24
  command = options.get_next_command(%i[start admin])
25
25
  # try to get 3 arguments as simple arguments
26
26
  case command
@@ -45,13 +45,13 @@ module Aspera
45
45
  :sync_info,
46
46
  mandatory: false,
47
47
  default: {'sessions' => [{'name' => File.basename(simple_session_args['local_dir'])}]})
48
- raise "sync_info shall be a Hash with key 'sessions' with Array of Hash: #{async_params}" unless async_params.is_a?(Hash) &&
49
- async_params['sessions']&.is_a?(Array) &&
50
- async_params['sessions'].first.is_a?(Hash)
48
+ assert_type(async_params, Hash){'sync_info'}
49
+ assert_type(async_params['sessions'], Array){'sync_info[sessions]'}
50
+ assert_type(async_params['sessions'].first, Hash){'sync_info[sessions][0]'}
51
51
  async_params['sessions'].first.merge!(simple_session_args)
52
52
  end
53
53
  Log.log.debug{Log.dump('async_params', async_params)}
54
- Aspera::Sync.start(async_params, &block)
54
+ Aspera::Fasp::Sync.start(async_params, &block)
55
55
  return Main.result_success
56
56
  when :admin
57
57
  command2 = options.get_next_command([:status])
@@ -59,7 +59,7 @@ module Aspera
59
59
  when :status
60
60
  sync_session_name = options.get_next_argument('name of sync session', mandatory: false, type: String)
61
61
  async_params = options.get_option(:sync_info, mandatory: true)
62
- return {type: :single_object, data: Aspera::Sync.admin_status(async_params, sync_session_name)}
62
+ return {type: :single_object, data: Aspera::Fasp::Sync.admin_status(async_params, sync_session_name)}
63
63
  end # command2
64
64
  end # command
65
65
  end # execute_action
@@ -1,7 +1,10 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require 'aspera/fasp/agent_base'
3
4
  require 'aspera/fasp/transfer_spec'
4
5
  require 'aspera/cli/info'
6
+ require 'aspera/log'
7
+ require 'aspera/assert'
5
8
 
6
9
  module Aspera
7
10
  module Cli
@@ -28,7 +31,7 @@ module Aspera
28
31
  :FILE_LIST_FROM_TRANSFER_SPEC,
29
32
  :FILE_LIST_OPTIONS,
30
33
  :DEFAULT_TRANSFER_NOTIFY_TEMPLATE
31
- TRANSFER_AGENTS = %i[direct node connect httpgw trsdk].freeze
34
+ TRANSFER_AGENTS = Fasp::AgentBase.agent_list.freeze
32
35
 
33
36
  class << self
34
37
  # @return :success if all sessions statuses returned by "start" are success
@@ -65,7 +68,7 @@ module Aspera
65
68
 
66
69
  # multiple option are merged
67
70
  def option_transfer_spec=(value)
68
- raise 'option ts shall be a Hash' unless value.is_a?(Hash)
71
+ assert_type(value, Hash){'ts'}
69
72
  @transfer_spec_command_line.deep_merge!(value)
70
73
  end
71
74
 
@@ -92,8 +95,8 @@ module Aspera
92
95
  end
93
96
 
94
97
  # analyze options and create new agent if not already created or set
95
- def set_agent_by_options
96
- return nil unless @agent.nil?
98
+ def agent_instance
99
+ return @agent unless @agent.nil?
97
100
  agent_type = @opt_mgr.get_option(:transfer, mandatory: true)
98
101
  # agent plugin is loaded on demand to avoid loading unnecessary dependencies
99
102
  require "aspera/fasp/agent_#{agent_type}"
@@ -117,7 +120,8 @@ module Aspera
117
120
  # get agent instance
118
121
  new_agent = Kernel.const_get("Aspera::Fasp::Agent#{agent_type.capitalize}").new(agent_options)
119
122
  self.agent_instance = new_agent
120
- return nil
123
+ Log.log.debug{"transfer agent is a #{@agent.class}"}
124
+ return @agent
121
125
  end
122
126
 
123
127
  # return destination folder for transfers
@@ -133,7 +137,7 @@ module Aspera
133
137
  case direction.to_s
134
138
  when Fasp::TransferSpec::DIRECTION_SEND then dest_folder = '/'
135
139
  when Fasp::TransferSpec::DIRECTION_RECEIVE then dest_folder = '.'
136
- else raise "wrong direction: #{direction}"
140
+ else error_unexpected_value(direction)
137
141
  end
138
142
  return dest_folder
139
143
  end
@@ -181,14 +185,15 @@ module Aspera
181
185
  if !@transfer_paths.nil?
182
186
  Log.log.warn('--sources overrides paths from --ts')
183
187
  end
184
- case @opt_mgr.get_option(:src_type, mandatory: true)
188
+ source_type=@opt_mgr.get_option(:src_type, mandatory: true)
189
+ case source_type
185
190
  when :list
186
191
  # when providing a list, just specify source
187
192
  @transfer_paths = file_list.map{|i|{'source' => i}}
188
193
  when :pair
189
- raise Cli::BadArgument, "When using pair, provide an even number of paths: #{file_list.length}" unless file_list.length.even?
194
+ assert(file_list.length.even?, exception_class: Cli::BadArgument){"When using pair, provide an even number of paths: #{file_list.length}"}
190
195
  @transfer_paths = file_list.each_slice(2).to_a.map{|s, d|{'source' => s, 'destination' => d}}
191
- else raise 'Unsupported src_type'
196
+ else error_unexpected_value(source_type)
192
197
  end
193
198
  Log.log.debug{"paths=#{@transfer_paths}"}
194
199
  return @transfer_paths
@@ -199,7 +204,7 @@ module Aspera
199
204
  # @param rest_token [Rest] if oauth token regeneration supported
200
205
  def start(transfer_spec, rest_token: nil)
201
206
  # check parameters
202
- raise 'transfer_spec must be hash' unless transfer_spec.is_a?(Hash)
207
+ assert_type(transfer_spec, Hash){'transfer_spec'}
203
208
  # process :src option
204
209
  case transfer_spec['direction']
205
210
  when Fasp::TransferSpec::DIRECTION_RECEIVE
@@ -223,12 +228,14 @@ module Aspera
223
228
  @transfer_spec_command_line['paths'] = transfer_spec['paths'] || ts_source_paths
224
229
  # updated transfer spec with command line
225
230
  updated_ts(transfer_spec)
231
+ # if TS from app has content_protection (e.g. F5), that means content is protected: ask password if not provided
232
+ if transfer_spec['content_protection'].eql?('decrypt') && !transfer_spec.key?('content_protection_password')
233
+ transfer_spec['content_protection_password'] = @opt_mgr.prompt_user_input('content protection password', true)
234
+ end
226
235
  # create transfer agent
227
- set_agent_by_options
228
- Log.log.debug{"transfer agent is a #{@agent.class}"}
229
- @agent.start_transfer(transfer_spec, token_regenerator: rest_token)
236
+ agent_instance.start_transfer(transfer_spec, token_regenerator: rest_token)
230
237
  # list of: :success or "error message string"
231
- result = @agent.wait_for_completion
238
+ result = agent_instance.wait_for_completion
232
239
  send_email_transfer_notification(transfer_spec, result)
233
240
  return result
234
241
  end
@@ -1,6 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require 'aspera/log'
4
+ require 'aspera/assert'
4
5
  require 'ruby-progressbar'
5
6
 
6
7
  module Aspera
@@ -25,9 +26,7 @@ module Aspera
25
26
 
26
27
  def event(session_id:, type:, info: nil)
27
28
  Log.log.debug{"progress: #{type} #{session_id} #{info}"}
28
- if session_id.nil? && !type.eql?(:pre_start)
29
- raise 'Internal error: session_id is nil'
30
- end
29
+ assert(!session_id.nil? || type.eql?(:pre_start)){'session_id is nil'}
31
30
  return if @completed
32
31
  if @progress_bar.nil?
33
32
  @progress_bar = ProgressBar.create(
@@ -4,6 +4,6 @@ module Aspera
4
4
  module Cli
5
5
  # for beta add extension : .beta1
6
6
  # for dev version add extension : .pre
7
- VERSION = '4.15.0'
7
+ VERSION = '4.16.0'
8
8
  end
9
9
  end
@@ -1,5 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require 'aspera/log'
4
+ require 'aspera/assert'
3
5
  module Aspera
4
6
  # helper class to build command line from a parameter list (key-value hash)
5
7
  # constructor takes hash: { 'param1':'value1', ...}
@@ -27,20 +29,20 @@ module Aspera
27
29
  # Called by provider of definition before constructor of this class so that params_definition has all mandatory fields
28
30
  def normalize_description(full_description)
29
31
  full_description.each do |name, options|
30
- raise "Expecting Hash, but have #{options.class} in #{name}" unless options.is_a?(Hash)
32
+ assert_type(options, Hash){name}
31
33
  unsupported_keys = options.keys - OPTIONS_KEYS
32
- raise "Unsupported definition keys: #{unsupported_keys}" unless unsupported_keys.empty?
33
- raise "Missing key: cli for #{name}" unless options.key?(:cli)
34
- raise 'Key: cli must be Hash' unless options[:cli].is_a?(Hash)
35
- raise 'Missing key: cli.type' unless options[:cli].key?(:type)
36
- raise "Unsupported processing type for #{name}: #{options[:cli][:type]}" unless CLI_OPTION_TYPES.include?(options[:cli][:type])
34
+ assert(unsupported_keys.empty?){"Unsupported definition keys: #{unsupported_keys}"}
35
+ assert(options.key?(:cli)){"Missing key: cli for #{name}"}
36
+ assert_type(options[:cli], Hash){'Key: cli'}
37
+ assert(options[:cli].key?(:type)){'Missing key: cli.type'}
38
+ assert_values(options[:cli][:type], CLI_OPTION_TYPES){"Unsupported processing type for #{name}"}
37
39
  # by default : optional
38
40
  options[:mandatory] ||= false
39
41
  options[:desc] ||= ''
40
42
  options[:desc] = "DEPRECATED: #{options[:deprecation]}\n#{options[:desc]}" if options.key?(:deprecation)
41
43
  cli = options[:cli]
42
44
  unsupported_cli_keys = cli.keys - CLI_KEYS
43
- raise "Unsupported cli keys: #{unsupported_cli_keys}" unless unsupported_cli_keys.empty?
45
+ assert(unsupported_cli_keys.empty?){"Unsupported cli keys: #{unsupported_cli_keys}"}
44
46
  # by default : string, unless it's without arg
45
47
  options[:accepted_types] ||= options[:cli][:type].eql?(:opt_without_arg) ? :bool : :string
46
48
  # single type is placed in array
@@ -121,7 +123,7 @@ module Aspera
121
123
  when :hash then Hash
122
124
  when :int then Integer
123
125
  when :bool then [TrueClass, FalseClass]
124
- else raise "INTERNAL: unexpected value: #{type_symbol}"
126
+ else error_unexpected_value(type_symbol)
125
127
  end
126
128
  end.flatten
127
129
  # check that value is of expected type
@@ -150,7 +152,7 @@ module Aspera
150
152
  raise Fasp::Error, "unsupported #{name}: #{parameter_value}" if converted_value.nil?
151
153
  parameter_value = converted_value
152
154
  when NilClass
153
- else raise "not expected type for convert #{options[:cli][:convert].class} for #{name}"
155
+ else error_unexpected_value(options[:cli][:convert].class)
154
156
  end
155
157
 
156
158
  case processing_type
@@ -159,14 +161,14 @@ module Aspera
159
161
  when :ignore, :special # ignore this parameter or process later
160
162
  return
161
163
  when :envvar # set in env var
162
- raise 'error' unless options[:cli].key?(:variable)
164
+ assert(options[:cli].key?(:variable)){'missing key: cli.variable'}
163
165
  @result[:env][options[:cli][:variable]] = parameter_value
164
166
  when :opt_without_arg # if present and true : just add option without value
165
167
  add_param = false
166
168
  case parameter_value
167
169
  when false then nil # nothing to put on command line, no creation by default
168
170
  when true then add_param = true
169
- else raise Fasp::Error, "unsupported #{name}: #{parameter_value}"
171
+ else error_unexpected_value(parameter_value){name}
170
172
  end
171
173
  add_param = !add_param if options[:add_on_false]
172
174
  add_command_line_options([options[:cli][:switch]]) if add_param
@@ -1,6 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require 'aspera/log'
4
+ require 'aspera/assert'
4
5
  require 'aspera/rest'
5
6
  require 'aspera/oauth'
6
7
  require 'xmlsimple'
@@ -10,9 +11,9 @@ module Aspera
10
11
  class << self
11
12
  def parameters_from_svc_credentials(service_credentials, bucket_region)
12
13
  # check necessary contents
13
- raise 'service_credentials must be a Hash' unless service_credentials.is_a?(Hash)
14
+ assert_type(service_credentials, Hash){'service_credentials'}
14
15
  %w[apikey resource_instance_id endpoints].each do |field|
15
- raise "service_credentials must have a field: #{field}" unless service_credentials.key?(field)
16
+ assert(service_credentials.key?(field)){"service_credentials must have a field: #{field}"}
16
17
  end
17
18
  Aspera::Log.dump('service_credentials', service_credentials)
18
19
  # read endpoints from service provided in service credentials
@@ -0,0 +1,22 @@
1
+ # frozen_string_literal: true
2
+
3
+ # coverage for tests
4
+ if ENV.key?('ENABLE_COVERAGE')
5
+ require 'simplecov'
6
+ require 'securerandom'
7
+ # compute gem source root based on this script location, assuming it is in bin/
8
+ # use dirname instead of gsub, in case folder separator is not /
9
+ development_root = 3.times.inject(File.realpath(__FILE__)) { |p, _| File.dirname(p) }
10
+ SimpleCov.root(development_root)
11
+ SimpleCov.enable_for_subprocesses if SimpleCov.respond_to?(:enable_for_subprocesses)
12
+ # keep cache data for 1 day (must be longer that time to run the whole test suite)
13
+ SimpleCov.merge_timeout(86400)
14
+ SimpleCov.command_name(SecureRandom.uuid)
15
+ SimpleCov.at_exit do
16
+ original_file_descriptor = $stdout
17
+ $stdout.reopen(File.join(development_root, 'simplecov.log'))
18
+ SimpleCov.result.format!
19
+ $stdout.reopen(original_file_descriptor)
20
+ end
21
+ SimpleCov.start
22
+ end
@@ -1,15 +1,46 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/log'
3
+ require 'aspera/assert'
4
4
  require 'singleton'
5
+ require 'openssl'
5
6
 
6
7
  module Aspera
7
8
  # a simple binary data repository
8
9
  class DataRepository
9
10
  include Singleton
11
+ # in same order as elements in folder
12
+ ELEMENTS = %i[dsa rsa uuid aspera.global-cli-client aspera.drive license]
13
+ START_INDEX = 1
14
+ DATA_FOLDER_NAME = 'data'
15
+
16
+ # decode data as expected as string
17
+ # @param name [Symbol] name of the data item
18
+ # @return [String] decoded data
19
+ def item(name)
20
+ index = ELEMENTS.index(name)
21
+ raise ArgumentError, "unknown data item #{name} (#{name.class})" unless index
22
+ raw_data = data(START_INDEX + index)
23
+ case name
24
+ when :dsa, :rsa
25
+ # generate PEM from DER
26
+ return OpenSSL::PKey.const_get(name.to_s.upcase).new(raw_data).to_pem
27
+ when :license
28
+ return Zlib::Inflate.inflate(raw_data)
29
+ when :uuid
30
+ return format('%08x-%04x-%04x-%04x-%04x%08x', *raw_data.unpack('NnnnnN'))
31
+ when :'aspera.global-cli-client', :'aspera.drive'
32
+ return Base64.urlsafe_encode64(raw_data)
33
+ else error_unexpected_value(name)
34
+ end
35
+ end
36
+
37
+ private
38
+
39
+ private_constant :START_INDEX, :DATA_FOLDER_NAME
40
+
10
41
  # get binary value from data repository
11
42
  def data(id)
12
- File.read(File.join(__dir__, 'data', id.to_s), mode: 'rb')
43
+ File.read(File.join(__dir__, DATA_FOLDER_NAME, id.to_s), mode: 'rb')
13
44
  end
14
45
  end
15
46
  end
@@ -2,6 +2,7 @@
2
2
 
3
3
  # cspell:ignore USERPROFILE HOMEDRIVE HOMEPATH LC_CTYPE msys aarch
4
4
  require 'aspera/log'
5
+ require 'aspera/assert'
5
6
  require 'rbconfig'
6
7
 
7
8
  # cspell:words MEBI mswin bccwin
@@ -15,10 +16,11 @@ module Aspera
15
16
  OS_AIX = :aix
16
17
  OS_LIST = [OS_WINDOWS, OS_X, OS_LINUX, OS_AIX].freeze
17
18
  CPU_X86_64 = :x86_64
19
+ CPU_ARM64 = :arm64
18
20
  CPU_PPC64 = :ppc64
19
21
  CPU_PPC64LE = :ppc64le
20
22
  CPU_S390 = :s390
21
- CPU_LIST = [CPU_X86_64, CPU_PPC64, CPU_PPC64LE, CPU_S390].freeze
23
+ CPU_LIST = [CPU_X86_64, CPU_ARM64, CPU_PPC64, CPU_PPC64LE, CPU_S390].freeze
22
24
 
23
25
  BITS_PER_BYTE = 8
24
26
  MEBI = 1024 * 1024
@@ -88,7 +90,7 @@ module Aspera
88
90
 
89
91
  # value is provided in block
90
92
  def write_file_restricted(path, force: false, mode: nil)
91
- raise 'coding error, missing content block' unless block_given?
93
+ assert(block_given?, exception_class: Aspera::InternalError)
92
94
  if force || !File.exist?(path)
93
95
  # Windows may give error
94
96
  File.unlink(path) rescue nil
@@ -2,13 +2,14 @@
2
2
 
3
3
  require 'aspera/fasp/agent_base'
4
4
  require 'aspera/rest'
5
+ require 'aspera/log'
5
6
  require 'aspera/json_rpc'
6
7
  require 'aspera/open_application'
7
8
  require 'securerandom'
8
9
 
9
10
  module Aspera
10
11
  module Fasp
11
- class AgentAspera < Aspera::Fasp::AgentBase
12
+ class AgentAlpha < Aspera::Fasp::AgentBase
12
13
  # try twice the main init url in sequence
13
14
  START_URIS = ['aspera://']
14
15
  # delay between each try to start connect
@@ -66,53 +67,42 @@ module Aspera
66
67
  # if there is a token, we ask connect client to use well known ssh private keys
67
68
  # instead of asking password
68
69
  transfer_spec['authentication'] = 'token' if transfer_spec.key?('token')
69
- @client_app_api.start_transfer(app_id: @application_id,transfer_spec: transfer_spec)
70
- # @xfer_id = res['transfer_specs'].first['transfer_spec']['tags'][Fasp::TransferSpec::TAG_RESERVED]['xfer_id']
70
+ result = @client_app_api.start_transfer(app_id: @application_id, desktop_spec: {}, transfer_spec: transfer_spec)
71
+ @xfer_id = result['uuid']
71
72
  end
72
73
 
73
74
  def wait_for_transfers_completion
74
- client_activity_args = {'aspera_client_settings' => @client_settings}
75
75
  started = false
76
76
  pre_calc = false
77
- session_id = @xfer_id
78
77
  begin
79
78
  loop do
80
- tr_info = @client_api.create("transfers/info/#{@xfer_id}", client_activity_args)[:data]
81
- Log.log.trace1{Log.dump(:tr_info, tr_info)}
82
- if tr_info['transfer_info'].is_a?(Hash)
83
- transfer = tr_info['transfer_info']
84
- if transfer.nil?
85
- Log.log.warn('no session in Connect')
86
- break
79
+ transfer = @client_app_api.get_transfer(app_id: @application_id, transfer_id: @xfer_id)
80
+ case transfer['status']
81
+ when 'initiating', 'queued'
82
+ notify_progress(session_id: nil, type: :pre_start, info: transfer['status'])
83
+ when 'running'
84
+ if !started
85
+ notify_progress(session_id: @xfer_id, type: :session_start)
86
+ started = true
87
87
  end
88
- # TODO: get session id
89
- case transfer['status']
90
- when 'initiating', 'queued'
91
- notify_progress(session_id: nil, type: :pre_start, info: transfer['status'])
92
- when 'running'
93
- if !started
94
- notify_progress(session_id: session_id, type: :session_start)
95
- started = true
96
- end
97
- if !pre_calc && (transfer['bytes_expected'] != 0)
98
- notify_progress(type: :session_size, session_id: session_id, info: transfer['bytes_expected'])
99
- pre_calc = true
100
- else
101
- notify_progress(type: :transfer, session_id: session_id, info: transfer['bytes_written'])
102
- end
103
- when 'completed'
104
- notify_progress(type: :end, session_id: session_id)
105
- break
106
- when 'failed'
107
- notify_progress(type: :end, session_id: session_id)
108
- raise Fasp::Error, transfer['error_desc']
109
- when 'cancelled'
110
- notify_progress(type: :end, session_id: session_id)
111
- raise Fasp::Error, 'Transfer cancelled by user'
88
+ if !pre_calc && (transfer['bytes_expected'] != 0)
89
+ notify_progress(type: :session_size, session_id: @xfer_id, info: transfer['bytes_expected'])
90
+ pre_calc = true
112
91
  else
113
- notify_progress(type: :end, session_id: session_id)
114
- raise Fasp::Error, "unknown status: #{transfer['status']}: #{transfer['error_desc']}"
92
+ notify_progress(type: :transfer, session_id: @xfer_id, info: transfer['bytes_written'])
115
93
  end
94
+ when 'completed'
95
+ notify_progress(type: :end, session_id: @xfer_id)
96
+ break
97
+ when 'failed'
98
+ notify_progress(type: :end, session_id: @xfer_id)
99
+ raise Fasp::Error, transfer['error_desc']
100
+ when 'cancelled'
101
+ notify_progress(type: :end, session_id: @xfer_id)
102
+ raise Fasp::Error, 'Transfer cancelled by user'
103
+ else
104
+ notify_progress(type: :end, session_id: @xfer_id)
105
+ raise Fasp::Error, "unknown status: #{transfer['status']}: #{transfer['error_desc']}"
116
106
  end
117
107
  sleep(1)
118
108
  end
@@ -121,6 +111,6 @@ module Aspera
121
111
  end
122
112
  return [:success]
123
113
  end # wait
124
- end # AgentAspera
114
+ end # AgentAlpha
125
115
  end
126
116
  end
@@ -1,5 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require 'aspera/log'
4
+ require 'aspera/assert'
3
5
  module Aspera
4
6
  module Fasp
5
7
  # Base class for transfer agents
@@ -10,7 +12,9 @@ module Aspera
10
12
  result = options.symbolize_keys
11
13
  available = default.map{|k, v|"#{k}(#{v})"}.join(', ')
12
14
  result.each do |k, _v|
13
- raise "Unknown transfer agent parameter: #{k}, expect one of #{available}" unless default.key?(k)
15
+ assert_values(k, default.keys){"transfer agent parameter: #{k}"}
16
+ # check it is the expected type: too limiting, as we can have an Integer or Float, or symbol and string
17
+ # raise "Invalid value for transfer agent parameter: #{k}, expect #{default[k].class.name}" unless default[k].nil? || v.is_a?(default[k].class)
14
18
  end
15
19
  default.each do |k, v|
16
20
  raise "Missing required agent parameter: #{k}. Parameters: #{available}" if v.eql?(:required) && !result.key?(k)
@@ -18,24 +22,30 @@ module Aspera
18
22
  end
19
23
  return result
20
24
  end
21
- end
25
+
26
+ def agent_list
27
+ Dir.entries(File.dirname(File.expand_path(__FILE__))).select do |file|
28
+ file.start_with?('agent_') && !file.eql?('agent_base.rb')
29
+ end.map{|file|file.sub(/^agent_/, '').sub(/\.rb$/, '').to_sym}
30
+ end
31
+ end
22
32
  def wait_for_completion
23
33
  # list of: :success or "error message string"
24
34
  statuses = wait_for_transfers_completion
25
35
  @progress&.reset
26
- raise "internal error: bad statuses type: #{statuses.class}" unless statuses.is_a?(Array)
27
- raise "internal error: bad statuses content: #{statuses}" unless statuses.select{|i|!i.eql?(:success) && !i.is_a?(StandardError)}.empty?
36
+ assert_type(statuses, Array)
37
+ assert(statuses.select{|i|!i.eql?(:success) && !i.is_a?(StandardError)}.empty?){"bad statuses content: #{statuses}"}
28
38
  return statuses
29
39
  end
30
40
 
31
41
  private
32
42
 
33
43
  def initialize(options)
34
- raise 'internal error' unless respond_to?(:start_transfer)
35
- raise 'internal error' unless respond_to?(:wait_for_transfers_completion)
36
44
  # method `shutdown` is optional
45
+ assert(respond_to?(:start_transfer))
46
+ assert(respond_to?(:wait_for_transfers_completion))
47
+ assert_type(options, Hash){'transfer agent options'}
37
48
  Log.log.debug{Log.dump(:agent_options, options)}
38
- raise "transfer agent options expecting Hash, but have #{options.class}" unless options.is_a?(Hash)
39
49
  @progress = options[:progress]
40
50
  options.delete(:progress)
41
51
  end