aspera-cli 4.15.0 → 4.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +29 -3
  4. data/CHANGELOG.md +292 -228
  5. data/CONTRIBUTING.md +69 -18
  6. data/README.md +1102 -952
  7. data/bin/ascli +13 -31
  8. data/bin/asession +3 -1
  9. data/examples/dascli +2 -2
  10. data/lib/aspera/aoc.rb +28 -33
  11. data/lib/aspera/ascmd.rb +3 -6
  12. data/lib/aspera/assert.rb +45 -0
  13. data/lib/aspera/cli/extended_value.rb +5 -5
  14. data/lib/aspera/cli/formatter.rb +26 -13
  15. data/lib/aspera/cli/hints.rb +4 -3
  16. data/lib/aspera/cli/main.rb +16 -3
  17. data/lib/aspera/cli/manager.rb +45 -36
  18. data/lib/aspera/cli/plugin.rb +20 -13
  19. data/lib/aspera/cli/plugins/aoc.rb +103 -73
  20. data/lib/aspera/cli/plugins/ats.rb +4 -3
  21. data/lib/aspera/cli/plugins/config.rb +114 -119
  22. data/lib/aspera/cli/plugins/cos.rb +2 -2
  23. data/lib/aspera/cli/plugins/faspex.rb +23 -19
  24. data/lib/aspera/cli/plugins/faspex5.rb +75 -43
  25. data/lib/aspera/cli/plugins/node.rb +28 -15
  26. data/lib/aspera/cli/plugins/orchestrator.rb +4 -2
  27. data/lib/aspera/cli/plugins/preview.rb +9 -7
  28. data/lib/aspera/cli/plugins/server.rb +6 -3
  29. data/lib/aspera/cli/plugins/shares.rb +30 -26
  30. data/lib/aspera/cli/sync_actions.rb +9 -9
  31. data/lib/aspera/cli/transfer_agent.rb +21 -14
  32. data/lib/aspera/cli/transfer_progress.rb +2 -3
  33. data/lib/aspera/cli/version.rb +1 -1
  34. data/lib/aspera/command_line_builder.rb +13 -11
  35. data/lib/aspera/cos_node.rb +3 -2
  36. data/lib/aspera/coverage.rb +22 -0
  37. data/lib/aspera/data_repository.rb +33 -2
  38. data/lib/aspera/environment.rb +4 -2
  39. data/lib/aspera/fasp/{agent_aspera.rb → agent_alpha.rb} +29 -39
  40. data/lib/aspera/fasp/agent_base.rb +17 -7
  41. data/lib/aspera/fasp/agent_direct.rb +88 -84
  42. data/lib/aspera/fasp/agent_httpgw.rb +4 -3
  43. data/lib/aspera/fasp/agent_node.rb +3 -2
  44. data/lib/aspera/fasp/agent_trsdk.rb +79 -37
  45. data/lib/aspera/fasp/installation.rb +51 -12
  46. data/lib/aspera/fasp/management.rb +11 -6
  47. data/lib/aspera/fasp/parameters.rb +53 -47
  48. data/lib/aspera/fasp/resume_policy.rb +7 -5
  49. data/lib/aspera/fasp/sync.rb +273 -0
  50. data/lib/aspera/fasp/transfer_spec.rb +10 -8
  51. data/lib/aspera/fasp/uri.rb +2 -2
  52. data/lib/aspera/faspex_gw.rb +11 -8
  53. data/lib/aspera/faspex_postproc.rb +6 -5
  54. data/lib/aspera/id_generator.rb +3 -1
  55. data/lib/aspera/json_rpc.rb +10 -8
  56. data/lib/aspera/keychain/encrypted_hash.rb +46 -11
  57. data/lib/aspera/keychain/macos_security.rb +15 -13
  58. data/lib/aspera/log.rb +4 -3
  59. data/lib/aspera/nagios.rb +7 -2
  60. data/lib/aspera/node.rb +17 -16
  61. data/lib/aspera/node_simulator.rb +214 -0
  62. data/lib/aspera/oauth.rb +22 -19
  63. data/lib/aspera/persistency_action_once.rb +13 -14
  64. data/lib/aspera/persistency_folder.rb +3 -2
  65. data/lib/aspera/preview/file_types.rb +53 -267
  66. data/lib/aspera/preview/generator.rb +7 -5
  67. data/lib/aspera/preview/terminal.rb +14 -5
  68. data/lib/aspera/preview/utils.rb +8 -7
  69. data/lib/aspera/proxy_auto_config.rb +6 -3
  70. data/lib/aspera/rest.rb +29 -13
  71. data/lib/aspera/rest_error_analyzer.rb +1 -0
  72. data/lib/aspera/rest_errors_aspera.rb +2 -0
  73. data/lib/aspera/secret_hider.rb +5 -2
  74. data/lib/aspera/ssh.rb +10 -8
  75. data/lib/aspera/temp_file_manager.rb +1 -1
  76. data/lib/aspera/web_server_simple.rb +2 -1
  77. data.tar.gz.sig +0 -0
  78. metadata +96 -45
  79. metadata.gz.sig +0 -0
  80. data/lib/aspera/sync.rb +0 -219
@@ -1,7 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require 'aspera/cli/plugins/node'
4
-
4
+ require 'aspera/assert'
5
5
  module Aspera
6
6
  module Cli
7
7
  module Plugins
@@ -49,8 +49,6 @@ module Aspera
49
49
 
50
50
  def initialize(env)
51
51
  super(env)
52
- options.declare(:type, 'Type of user/group for operations', values: %i[any local ldap saml], default: :any)
53
- options.parse_options!
54
52
  end
55
53
 
56
54
  SAML_IMPORT_MANDATORY = %w[id name_id].freeze
@@ -84,63 +82,69 @@ module Aspera
84
82
  return Node.new(@agents, api: api_shares_node).execute_action(repo_command)
85
83
  when :admin
86
84
  api_shares_admin = basic_auth_api('api/v1')
87
- admin_command = options.get_next_command(%i[user group share node].freeze)
85
+ admin_command = options.get_next_command(%i[node share transfer_settings user group].freeze)
88
86
  case admin_command
89
87
  when :node
90
88
  return entity_action(api_shares_admin, 'data/nodes')
89
+ when :share
90
+ share_command = options.get_next_command(%i[user_permissions group_permissions].concat(Plugin::ALL_OPS))
91
+ case share_command
92
+ when *Plugin::ALL_OPS
93
+ return entity_command(share_command, api_shares_admin, 'data/shares')
94
+ # return {type: :object_list, data: all_shares, fields: %w[id name status status_message]}
95
+ when :user_permissions, :group_permissions
96
+ share_id = instance_identifier
97
+ return entity_action(api_shares_admin, "data/shares/#{share_id}/#{share_command}")
98
+ end
99
+ when :transfer_settings
100
+ xfer_settings_command = options.get_next_command(%i[show modify])
101
+ return entity_command(xfer_settings_command, api_shares_admin, 'data/transfer_settings', is_singleton: true)
91
102
  when :user, :group
92
103
  entity_type = admin_command
93
- entities_location = options.get_option(:type, mandatory: true)
104
+ entities_location = options.get_next_command(%i[all local ldap saml])
94
105
  entities_path = "data/#{entities_location}_#{entity_type}s"
95
106
  entity_action = nil
96
107
  case entities_location
97
- when :any
108
+ when :all
98
109
  entities_path = "data/#{entity_type}s"
99
110
  entity_action = %i[list show delete]
100
111
  entity_action.concat(USR_GRP_SETTINGS)
101
112
  entity_action.push(:users) if entity_type.eql?(:group)
102
113
  entity_action.freeze
103
114
  when :local
104
- entity_action = %i[list show create modify delete].freeze
115
+ entity_action = %i[list show delete create modify].freeze
105
116
  when :ldap
106
117
  entity_action = %i[add].freeze
107
118
  when :saml
108
119
  entity_action = %i[import].freeze
109
120
  end
110
121
  entity_verb = options.get_next_command(entity_action)
111
- # entity_path = "#{entities_path}/#{instance_identifier}" if %i[app_authorizations share_permissions].include?(entity_verb)
112
122
  case entity_verb
113
- when *Plugin::ALL_OPS
123
+ when *Plugin::ALL_OPS # list, show, delete, create, modify
114
124
  display_fields = entity_type.eql?(:user) ? %w[id username first_name last_name email] : nil
115
- display_fields.push(:directory_user) if entity_type.eql?(:user) && entities_location.eql?(:any)
125
+ display_fields.push(:directory_user) if entity_type.eql?(:user) && entities_location.eql?(:all)
116
126
  return entity_command(entity_verb, api_shares_admin, entities_path, display_fields: display_fields)
117
- when :import
127
+ when *USR_GRP_SETTINGS # transfer_settings, app_authorizations, share_permissions
128
+ group_id = instance_identifier
129
+ entities_path = "#{entities_path}/#{group_id}/#{entity_verb}"
130
+ return entity_action(api_shares_admin, entities_path, is_singleton: !entity_verb.eql?(:share_permissions))
131
+ when :import # saml
118
132
  return do_bulk_operation(command: entity_verb, descr: 'user information') do |entity_parameters|
119
133
  entity_parameters = entity_parameters.transform_keys{|k|k.gsub(/\s+/, '_').downcase}
120
- raise 'expecting Hash' unless entity_parameters.is_a?(Hash)
134
+ assert_type(entity_parameters, Hash)
121
135
  SAML_IMPORT_MANDATORY.each{|p|raise "missing mandatory field: #{p}" if entity_parameters[p].nil?}
122
136
  entity_parameters.each_key do |p|
123
137
  raise "unsupported field: #{p}, use: #{SAML_IMPORT_ALLOWED.join(',')}" unless SAML_IMPORT_ALLOWED.include?(p)
124
138
  end
125
139
  api_shares_admin.create("#{entities_path}/import", entity_parameters)[:data]
126
140
  end
127
- when :add
141
+ when :add # ldap
128
142
  return do_bulk_operation(command: entity_verb, descr: "#{entity_type} name", values: String) do |entity_name|
129
143
  api_shares_admin.create(entities_path, {entity_type=>entity_name})[:data]
130
144
  end
131
- when *USR_GRP_SETTINGS
132
- group_id = instance_identifier
133
- entities_path = "#{entities_path}/#{group_id}/#{entity_verb}"
134
- return entity_action(api_shares_admin, entities_path, is_singleton: !entity_verb.eql?(:share_permissions))
135
- end
136
- when :share
137
- share_command = options.get_next_command(%i[user_permissions group_permissions].concat(Plugin::ALL_OPS))
138
- case share_command
139
- when *Plugin::ALL_OPS
140
- return entity_command(share_command, api_shares_admin, 'data/shares')
141
- # return {type: :object_list, data: all_shares, fields: %w[id name status status_message]}
142
- when :user_permissions, :group_permissions
143
- return entity_action(api_shares_admin, "data/shares/#{instance_identifier}/#{share_command}")
145
+ when :users # group
146
+ raise "TODO, not implemented"
147
+ else error_unexpected_value(entity_verb)
144
148
  end
145
149
  end
146
150
  end
@@ -1,13 +1,14 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/sync'
3
+ require 'aspera/fasp/sync'
4
+ require 'aspera/assert'
4
5
 
5
6
  module Aspera
6
7
  module Cli
7
8
  # Module for sync actions
8
9
  module SyncActions
9
10
  SIMPLE_ARGUMENTS_SYNC = {
10
- direction: Aspera::Sync::DIRECTIONS,
11
+ direction: Aspera::Fasp::Sync::DIRECTIONS,
11
12
  local_dir: String,
12
13
  remote_dir: String
13
14
  }.stringify_keys.freeze
@@ -19,8 +20,7 @@ module Aspera
19
20
  end
20
21
 
21
22
  def execute_sync_action(&block)
22
- # options = Aspera::Cli::Manager.new
23
- raise 'Internal Error: No block given' unless block
23
+ assert(block){'No block given'}
24
24
  command = options.get_next_command(%i[start admin])
25
25
  # try to get 3 arguments as simple arguments
26
26
  case command
@@ -45,13 +45,13 @@ module Aspera
45
45
  :sync_info,
46
46
  mandatory: false,
47
47
  default: {'sessions' => [{'name' => File.basename(simple_session_args['local_dir'])}]})
48
- raise "sync_info shall be a Hash with key 'sessions' with Array of Hash: #{async_params}" unless async_params.is_a?(Hash) &&
49
- async_params['sessions']&.is_a?(Array) &&
50
- async_params['sessions'].first.is_a?(Hash)
48
+ assert_type(async_params, Hash){'sync_info'}
49
+ assert_type(async_params['sessions'], Array){'sync_info[sessions]'}
50
+ assert_type(async_params['sessions'].first, Hash){'sync_info[sessions][0]'}
51
51
  async_params['sessions'].first.merge!(simple_session_args)
52
52
  end
53
53
  Log.log.debug{Log.dump('async_params', async_params)}
54
- Aspera::Sync.start(async_params, &block)
54
+ Aspera::Fasp::Sync.start(async_params, &block)
55
55
  return Main.result_success
56
56
  when :admin
57
57
  command2 = options.get_next_command([:status])
@@ -59,7 +59,7 @@ module Aspera
59
59
  when :status
60
60
  sync_session_name = options.get_next_argument('name of sync session', mandatory: false, type: String)
61
61
  async_params = options.get_option(:sync_info, mandatory: true)
62
- return {type: :single_object, data: Aspera::Sync.admin_status(async_params, sync_session_name)}
62
+ return {type: :single_object, data: Aspera::Fasp::Sync.admin_status(async_params, sync_session_name)}
63
63
  end # command2
64
64
  end # command
65
65
  end # execute_action
@@ -1,7 +1,10 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require 'aspera/fasp/agent_base'
3
4
  require 'aspera/fasp/transfer_spec'
4
5
  require 'aspera/cli/info'
6
+ require 'aspera/log'
7
+ require 'aspera/assert'
5
8
 
6
9
  module Aspera
7
10
  module Cli
@@ -28,7 +31,7 @@ module Aspera
28
31
  :FILE_LIST_FROM_TRANSFER_SPEC,
29
32
  :FILE_LIST_OPTIONS,
30
33
  :DEFAULT_TRANSFER_NOTIFY_TEMPLATE
31
- TRANSFER_AGENTS = %i[direct node connect httpgw trsdk].freeze
34
+ TRANSFER_AGENTS = Fasp::AgentBase.agent_list.freeze
32
35
 
33
36
  class << self
34
37
  # @return :success if all sessions statuses returned by "start" are success
@@ -65,7 +68,7 @@ module Aspera
65
68
 
66
69
  # multiple option are merged
67
70
  def option_transfer_spec=(value)
68
- raise 'option ts shall be a Hash' unless value.is_a?(Hash)
71
+ assert_type(value, Hash){'ts'}
69
72
  @transfer_spec_command_line.deep_merge!(value)
70
73
  end
71
74
 
@@ -92,8 +95,8 @@ module Aspera
92
95
  end
93
96
 
94
97
  # analyze options and create new agent if not already created or set
95
- def set_agent_by_options
96
- return nil unless @agent.nil?
98
+ def agent_instance
99
+ return @agent unless @agent.nil?
97
100
  agent_type = @opt_mgr.get_option(:transfer, mandatory: true)
98
101
  # agent plugin is loaded on demand to avoid loading unnecessary dependencies
99
102
  require "aspera/fasp/agent_#{agent_type}"
@@ -117,7 +120,8 @@ module Aspera
117
120
  # get agent instance
118
121
  new_agent = Kernel.const_get("Aspera::Fasp::Agent#{agent_type.capitalize}").new(agent_options)
119
122
  self.agent_instance = new_agent
120
- return nil
123
+ Log.log.debug{"transfer agent is a #{@agent.class}"}
124
+ return @agent
121
125
  end
122
126
 
123
127
  # return destination folder for transfers
@@ -133,7 +137,7 @@ module Aspera
133
137
  case direction.to_s
134
138
  when Fasp::TransferSpec::DIRECTION_SEND then dest_folder = '/'
135
139
  when Fasp::TransferSpec::DIRECTION_RECEIVE then dest_folder = '.'
136
- else raise "wrong direction: #{direction}"
140
+ else error_unexpected_value(direction)
137
141
  end
138
142
  return dest_folder
139
143
  end
@@ -181,14 +185,15 @@ module Aspera
181
185
  if !@transfer_paths.nil?
182
186
  Log.log.warn('--sources overrides paths from --ts')
183
187
  end
184
- case @opt_mgr.get_option(:src_type, mandatory: true)
188
+ source_type=@opt_mgr.get_option(:src_type, mandatory: true)
189
+ case source_type
185
190
  when :list
186
191
  # when providing a list, just specify source
187
192
  @transfer_paths = file_list.map{|i|{'source' => i}}
188
193
  when :pair
189
- raise Cli::BadArgument, "When using pair, provide an even number of paths: #{file_list.length}" unless file_list.length.even?
194
+ assert(file_list.length.even?, exception_class: Cli::BadArgument){"When using pair, provide an even number of paths: #{file_list.length}"}
190
195
  @transfer_paths = file_list.each_slice(2).to_a.map{|s, d|{'source' => s, 'destination' => d}}
191
- else raise 'Unsupported src_type'
196
+ else error_unexpected_value(source_type)
192
197
  end
193
198
  Log.log.debug{"paths=#{@transfer_paths}"}
194
199
  return @transfer_paths
@@ -199,7 +204,7 @@ module Aspera
199
204
  # @param rest_token [Rest] if oauth token regeneration supported
200
205
  def start(transfer_spec, rest_token: nil)
201
206
  # check parameters
202
- raise 'transfer_spec must be hash' unless transfer_spec.is_a?(Hash)
207
+ assert_type(transfer_spec, Hash){'transfer_spec'}
203
208
  # process :src option
204
209
  case transfer_spec['direction']
205
210
  when Fasp::TransferSpec::DIRECTION_RECEIVE
@@ -223,12 +228,14 @@ module Aspera
223
228
  @transfer_spec_command_line['paths'] = transfer_spec['paths'] || ts_source_paths
224
229
  # updated transfer spec with command line
225
230
  updated_ts(transfer_spec)
231
+ # if TS from app has content_protection (e.g. F5), that means content is protected: ask password if not provided
232
+ if transfer_spec['content_protection'].eql?('decrypt') && !transfer_spec.key?('content_protection_password')
233
+ transfer_spec['content_protection_password'] = @opt_mgr.prompt_user_input('content protection password', true)
234
+ end
226
235
  # create transfer agent
227
- set_agent_by_options
228
- Log.log.debug{"transfer agent is a #{@agent.class}"}
229
- @agent.start_transfer(transfer_spec, token_regenerator: rest_token)
236
+ agent_instance.start_transfer(transfer_spec, token_regenerator: rest_token)
230
237
  # list of: :success or "error message string"
231
- result = @agent.wait_for_completion
238
+ result = agent_instance.wait_for_completion
232
239
  send_email_transfer_notification(transfer_spec, result)
233
240
  return result
234
241
  end
@@ -1,6 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require 'aspera/log'
4
+ require 'aspera/assert'
4
5
  require 'ruby-progressbar'
5
6
 
6
7
  module Aspera
@@ -25,9 +26,7 @@ module Aspera
25
26
 
26
27
  def event(session_id:, type:, info: nil)
27
28
  Log.log.debug{"progress: #{type} #{session_id} #{info}"}
28
- if session_id.nil? && !type.eql?(:pre_start)
29
- raise 'Internal error: session_id is nil'
30
- end
29
+ assert(!session_id.nil? || type.eql?(:pre_start)){'session_id is nil'}
31
30
  return if @completed
32
31
  if @progress_bar.nil?
33
32
  @progress_bar = ProgressBar.create(
@@ -4,6 +4,6 @@ module Aspera
4
4
  module Cli
5
5
  # for beta add extension : .beta1
6
6
  # for dev version add extension : .pre
7
- VERSION = '4.15.0'
7
+ VERSION = '4.16.0'
8
8
  end
9
9
  end
@@ -1,5 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require 'aspera/log'
4
+ require 'aspera/assert'
3
5
  module Aspera
4
6
  # helper class to build command line from a parameter list (key-value hash)
5
7
  # constructor takes hash: { 'param1':'value1', ...}
@@ -27,20 +29,20 @@ module Aspera
27
29
  # Called by provider of definition before constructor of this class so that params_definition has all mandatory fields
28
30
  def normalize_description(full_description)
29
31
  full_description.each do |name, options|
30
- raise "Expecting Hash, but have #{options.class} in #{name}" unless options.is_a?(Hash)
32
+ assert_type(options, Hash){name}
31
33
  unsupported_keys = options.keys - OPTIONS_KEYS
32
- raise "Unsupported definition keys: #{unsupported_keys}" unless unsupported_keys.empty?
33
- raise "Missing key: cli for #{name}" unless options.key?(:cli)
34
- raise 'Key: cli must be Hash' unless options[:cli].is_a?(Hash)
35
- raise 'Missing key: cli.type' unless options[:cli].key?(:type)
36
- raise "Unsupported processing type for #{name}: #{options[:cli][:type]}" unless CLI_OPTION_TYPES.include?(options[:cli][:type])
34
+ assert(unsupported_keys.empty?){"Unsupported definition keys: #{unsupported_keys}"}
35
+ assert(options.key?(:cli)){"Missing key: cli for #{name}"}
36
+ assert_type(options[:cli], Hash){'Key: cli'}
37
+ assert(options[:cli].key?(:type)){'Missing key: cli.type'}
38
+ assert_values(options[:cli][:type], CLI_OPTION_TYPES){"Unsupported processing type for #{name}"}
37
39
  # by default : optional
38
40
  options[:mandatory] ||= false
39
41
  options[:desc] ||= ''
40
42
  options[:desc] = "DEPRECATED: #{options[:deprecation]}\n#{options[:desc]}" if options.key?(:deprecation)
41
43
  cli = options[:cli]
42
44
  unsupported_cli_keys = cli.keys - CLI_KEYS
43
- raise "Unsupported cli keys: #{unsupported_cli_keys}" unless unsupported_cli_keys.empty?
45
+ assert(unsupported_cli_keys.empty?){"Unsupported cli keys: #{unsupported_cli_keys}"}
44
46
  # by default : string, unless it's without arg
45
47
  options[:accepted_types] ||= options[:cli][:type].eql?(:opt_without_arg) ? :bool : :string
46
48
  # single type is placed in array
@@ -121,7 +123,7 @@ module Aspera
121
123
  when :hash then Hash
122
124
  when :int then Integer
123
125
  when :bool then [TrueClass, FalseClass]
124
- else raise "INTERNAL: unexpected value: #{type_symbol}"
126
+ else error_unexpected_value(type_symbol)
125
127
  end
126
128
  end.flatten
127
129
  # check that value is of expected type
@@ -150,7 +152,7 @@ module Aspera
150
152
  raise Fasp::Error, "unsupported #{name}: #{parameter_value}" if converted_value.nil?
151
153
  parameter_value = converted_value
152
154
  when NilClass
153
- else raise "not expected type for convert #{options[:cli][:convert].class} for #{name}"
155
+ else error_unexpected_value(options[:cli][:convert].class)
154
156
  end
155
157
 
156
158
  case processing_type
@@ -159,14 +161,14 @@ module Aspera
159
161
  when :ignore, :special # ignore this parameter or process later
160
162
  return
161
163
  when :envvar # set in env var
162
- raise 'error' unless options[:cli].key?(:variable)
164
+ assert(options[:cli].key?(:variable)){'missing key: cli.variable'}
163
165
  @result[:env][options[:cli][:variable]] = parameter_value
164
166
  when :opt_without_arg # if present and true : just add option without value
165
167
  add_param = false
166
168
  case parameter_value
167
169
  when false then nil # nothing to put on command line, no creation by default
168
170
  when true then add_param = true
169
- else raise Fasp::Error, "unsupported #{name}: #{parameter_value}"
171
+ else error_unexpected_value(parameter_value){name}
170
172
  end
171
173
  add_param = !add_param if options[:add_on_false]
172
174
  add_command_line_options([options[:cli][:switch]]) if add_param
@@ -1,6 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require 'aspera/log'
4
+ require 'aspera/assert'
4
5
  require 'aspera/rest'
5
6
  require 'aspera/oauth'
6
7
  require 'xmlsimple'
@@ -10,9 +11,9 @@ module Aspera
10
11
  class << self
11
12
  def parameters_from_svc_credentials(service_credentials, bucket_region)
12
13
  # check necessary contents
13
- raise 'service_credentials must be a Hash' unless service_credentials.is_a?(Hash)
14
+ assert_type(service_credentials, Hash){'service_credentials'}
14
15
  %w[apikey resource_instance_id endpoints].each do |field|
15
- raise "service_credentials must have a field: #{field}" unless service_credentials.key?(field)
16
+ assert(service_credentials.key?(field)){"service_credentials must have a field: #{field}"}
16
17
  end
17
18
  Aspera::Log.dump('service_credentials', service_credentials)
18
19
  # read endpoints from service provided in service credentials
@@ -0,0 +1,22 @@
1
+ # frozen_string_literal: true
2
+
3
+ # coverage for tests
4
+ if ENV.key?('ENABLE_COVERAGE')
5
+ require 'simplecov'
6
+ require 'securerandom'
7
+ # compute gem source root based on this script location, assuming it is in bin/
8
+ # use dirname instead of gsub, in case folder separator is not /
9
+ development_root = 3.times.inject(File.realpath(__FILE__)) { |p, _| File.dirname(p) }
10
+ SimpleCov.root(development_root)
11
+ SimpleCov.enable_for_subprocesses if SimpleCov.respond_to?(:enable_for_subprocesses)
12
+ # keep cache data for 1 day (must be longer that time to run the whole test suite)
13
+ SimpleCov.merge_timeout(86400)
14
+ SimpleCov.command_name(SecureRandom.uuid)
15
+ SimpleCov.at_exit do
16
+ original_file_descriptor = $stdout
17
+ $stdout.reopen(File.join(development_root, 'simplecov.log'))
18
+ SimpleCov.result.format!
19
+ $stdout.reopen(original_file_descriptor)
20
+ end
21
+ SimpleCov.start
22
+ end
@@ -1,15 +1,46 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/log'
3
+ require 'aspera/assert'
4
4
  require 'singleton'
5
+ require 'openssl'
5
6
 
6
7
  module Aspera
7
8
  # a simple binary data repository
8
9
  class DataRepository
9
10
  include Singleton
11
+ # in same order as elements in folder
12
+ ELEMENTS = %i[dsa rsa uuid aspera.global-cli-client aspera.drive license]
13
+ START_INDEX = 1
14
+ DATA_FOLDER_NAME = 'data'
15
+
16
+ # decode data as expected as string
17
+ # @param name [Symbol] name of the data item
18
+ # @return [String] decoded data
19
+ def item(name)
20
+ index = ELEMENTS.index(name)
21
+ raise ArgumentError, "unknown data item #{name} (#{name.class})" unless index
22
+ raw_data = data(START_INDEX + index)
23
+ case name
24
+ when :dsa, :rsa
25
+ # generate PEM from DER
26
+ return OpenSSL::PKey.const_get(name.to_s.upcase).new(raw_data).to_pem
27
+ when :license
28
+ return Zlib::Inflate.inflate(raw_data)
29
+ when :uuid
30
+ return format('%08x-%04x-%04x-%04x-%04x%08x', *raw_data.unpack('NnnnnN'))
31
+ when :'aspera.global-cli-client', :'aspera.drive'
32
+ return Base64.urlsafe_encode64(raw_data)
33
+ else error_unexpected_value(name)
34
+ end
35
+ end
36
+
37
+ private
38
+
39
+ private_constant :START_INDEX, :DATA_FOLDER_NAME
40
+
10
41
  # get binary value from data repository
11
42
  def data(id)
12
- File.read(File.join(__dir__, 'data', id.to_s), mode: 'rb')
43
+ File.read(File.join(__dir__, DATA_FOLDER_NAME, id.to_s), mode: 'rb')
13
44
  end
14
45
  end
15
46
  end
@@ -2,6 +2,7 @@
2
2
 
3
3
  # cspell:ignore USERPROFILE HOMEDRIVE HOMEPATH LC_CTYPE msys aarch
4
4
  require 'aspera/log'
5
+ require 'aspera/assert'
5
6
  require 'rbconfig'
6
7
 
7
8
  # cspell:words MEBI mswin bccwin
@@ -15,10 +16,11 @@ module Aspera
15
16
  OS_AIX = :aix
16
17
  OS_LIST = [OS_WINDOWS, OS_X, OS_LINUX, OS_AIX].freeze
17
18
  CPU_X86_64 = :x86_64
19
+ CPU_ARM64 = :arm64
18
20
  CPU_PPC64 = :ppc64
19
21
  CPU_PPC64LE = :ppc64le
20
22
  CPU_S390 = :s390
21
- CPU_LIST = [CPU_X86_64, CPU_PPC64, CPU_PPC64LE, CPU_S390].freeze
23
+ CPU_LIST = [CPU_X86_64, CPU_ARM64, CPU_PPC64, CPU_PPC64LE, CPU_S390].freeze
22
24
 
23
25
  BITS_PER_BYTE = 8
24
26
  MEBI = 1024 * 1024
@@ -88,7 +90,7 @@ module Aspera
88
90
 
89
91
  # value is provided in block
90
92
  def write_file_restricted(path, force: false, mode: nil)
91
- raise 'coding error, missing content block' unless block_given?
93
+ assert(block_given?, exception_class: Aspera::InternalError)
92
94
  if force || !File.exist?(path)
93
95
  # Windows may give error
94
96
  File.unlink(path) rescue nil
@@ -2,13 +2,14 @@
2
2
 
3
3
  require 'aspera/fasp/agent_base'
4
4
  require 'aspera/rest'
5
+ require 'aspera/log'
5
6
  require 'aspera/json_rpc'
6
7
  require 'aspera/open_application'
7
8
  require 'securerandom'
8
9
 
9
10
  module Aspera
10
11
  module Fasp
11
- class AgentAspera < Aspera::Fasp::AgentBase
12
+ class AgentAlpha < Aspera::Fasp::AgentBase
12
13
  # try twice the main init url in sequence
13
14
  START_URIS = ['aspera://']
14
15
  # delay between each try to start connect
@@ -66,53 +67,42 @@ module Aspera
66
67
  # if there is a token, we ask connect client to use well known ssh private keys
67
68
  # instead of asking password
68
69
  transfer_spec['authentication'] = 'token' if transfer_spec.key?('token')
69
- @client_app_api.start_transfer(app_id: @application_id,transfer_spec: transfer_spec)
70
- # @xfer_id = res['transfer_specs'].first['transfer_spec']['tags'][Fasp::TransferSpec::TAG_RESERVED]['xfer_id']
70
+ result = @client_app_api.start_transfer(app_id: @application_id, desktop_spec: {}, transfer_spec: transfer_spec)
71
+ @xfer_id = result['uuid']
71
72
  end
72
73
 
73
74
  def wait_for_transfers_completion
74
- client_activity_args = {'aspera_client_settings' => @client_settings}
75
75
  started = false
76
76
  pre_calc = false
77
- session_id = @xfer_id
78
77
  begin
79
78
  loop do
80
- tr_info = @client_api.create("transfers/info/#{@xfer_id}", client_activity_args)[:data]
81
- Log.log.trace1{Log.dump(:tr_info, tr_info)}
82
- if tr_info['transfer_info'].is_a?(Hash)
83
- transfer = tr_info['transfer_info']
84
- if transfer.nil?
85
- Log.log.warn('no session in Connect')
86
- break
79
+ transfer = @client_app_api.get_transfer(app_id: @application_id, transfer_id: @xfer_id)
80
+ case transfer['status']
81
+ when 'initiating', 'queued'
82
+ notify_progress(session_id: nil, type: :pre_start, info: transfer['status'])
83
+ when 'running'
84
+ if !started
85
+ notify_progress(session_id: @xfer_id, type: :session_start)
86
+ started = true
87
87
  end
88
- # TODO: get session id
89
- case transfer['status']
90
- when 'initiating', 'queued'
91
- notify_progress(session_id: nil, type: :pre_start, info: transfer['status'])
92
- when 'running'
93
- if !started
94
- notify_progress(session_id: session_id, type: :session_start)
95
- started = true
96
- end
97
- if !pre_calc && (transfer['bytes_expected'] != 0)
98
- notify_progress(type: :session_size, session_id: session_id, info: transfer['bytes_expected'])
99
- pre_calc = true
100
- else
101
- notify_progress(type: :transfer, session_id: session_id, info: transfer['bytes_written'])
102
- end
103
- when 'completed'
104
- notify_progress(type: :end, session_id: session_id)
105
- break
106
- when 'failed'
107
- notify_progress(type: :end, session_id: session_id)
108
- raise Fasp::Error, transfer['error_desc']
109
- when 'cancelled'
110
- notify_progress(type: :end, session_id: session_id)
111
- raise Fasp::Error, 'Transfer cancelled by user'
88
+ if !pre_calc && (transfer['bytes_expected'] != 0)
89
+ notify_progress(type: :session_size, session_id: @xfer_id, info: transfer['bytes_expected'])
90
+ pre_calc = true
112
91
  else
113
- notify_progress(type: :end, session_id: session_id)
114
- raise Fasp::Error, "unknown status: #{transfer['status']}: #{transfer['error_desc']}"
92
+ notify_progress(type: :transfer, session_id: @xfer_id, info: transfer['bytes_written'])
115
93
  end
94
+ when 'completed'
95
+ notify_progress(type: :end, session_id: @xfer_id)
96
+ break
97
+ when 'failed'
98
+ notify_progress(type: :end, session_id: @xfer_id)
99
+ raise Fasp::Error, transfer['error_desc']
100
+ when 'cancelled'
101
+ notify_progress(type: :end, session_id: @xfer_id)
102
+ raise Fasp::Error, 'Transfer cancelled by user'
103
+ else
104
+ notify_progress(type: :end, session_id: @xfer_id)
105
+ raise Fasp::Error, "unknown status: #{transfer['status']}: #{transfer['error_desc']}"
116
106
  end
117
107
  sleep(1)
118
108
  end
@@ -121,6 +111,6 @@ module Aspera
121
111
  end
122
112
  return [:success]
123
113
  end # wait
124
- end # AgentAspera
114
+ end # AgentAlpha
125
115
  end
126
116
  end
@@ -1,5 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require 'aspera/log'
4
+ require 'aspera/assert'
3
5
  module Aspera
4
6
  module Fasp
5
7
  # Base class for transfer agents
@@ -10,7 +12,9 @@ module Aspera
10
12
  result = options.symbolize_keys
11
13
  available = default.map{|k, v|"#{k}(#{v})"}.join(', ')
12
14
  result.each do |k, _v|
13
- raise "Unknown transfer agent parameter: #{k}, expect one of #{available}" unless default.key?(k)
15
+ assert_values(k, default.keys){"transfer agent parameter: #{k}"}
16
+ # check it is the expected type: too limiting, as we can have an Integer or Float, or symbol and string
17
+ # raise "Invalid value for transfer agent parameter: #{k}, expect #{default[k].class.name}" unless default[k].nil? || v.is_a?(default[k].class)
14
18
  end
15
19
  default.each do |k, v|
16
20
  raise "Missing required agent parameter: #{k}. Parameters: #{available}" if v.eql?(:required) && !result.key?(k)
@@ -18,24 +22,30 @@ module Aspera
18
22
  end
19
23
  return result
20
24
  end
21
- end
25
+
26
+ def agent_list
27
+ Dir.entries(File.dirname(File.expand_path(__FILE__))).select do |file|
28
+ file.start_with?('agent_') && !file.eql?('agent_base.rb')
29
+ end.map{|file|file.sub(/^agent_/, '').sub(/\.rb$/, '').to_sym}
30
+ end
31
+ end
22
32
  def wait_for_completion
23
33
  # list of: :success or "error message string"
24
34
  statuses = wait_for_transfers_completion
25
35
  @progress&.reset
26
- raise "internal error: bad statuses type: #{statuses.class}" unless statuses.is_a?(Array)
27
- raise "internal error: bad statuses content: #{statuses}" unless statuses.select{|i|!i.eql?(:success) && !i.is_a?(StandardError)}.empty?
36
+ assert_type(statuses, Array)
37
+ assert(statuses.select{|i|!i.eql?(:success) && !i.is_a?(StandardError)}.empty?){"bad statuses content: #{statuses}"}
28
38
  return statuses
29
39
  end
30
40
 
31
41
  private
32
42
 
33
43
  def initialize(options)
34
- raise 'internal error' unless respond_to?(:start_transfer)
35
- raise 'internal error' unless respond_to?(:wait_for_transfers_completion)
36
44
  # method `shutdown` is optional
45
+ assert(respond_to?(:start_transfer))
46
+ assert(respond_to?(:wait_for_transfers_completion))
47
+ assert_type(options, Hash){'transfer agent options'}
37
48
  Log.log.debug{Log.dump(:agent_options, options)}
38
- raise "transfer agent options expecting Hash, but have #{options.class}" unless options.is_a?(Hash)
39
49
  @progress = options[:progress]
40
50
  options.delete(:progress)
41
51
  end