aspera-cli 4.15.0 → 4.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/BUGS.md +29 -3
- data/CHANGELOG.md +292 -228
- data/CONTRIBUTING.md +69 -18
- data/README.md +1102 -952
- data/bin/ascli +13 -31
- data/bin/asession +3 -1
- data/examples/dascli +2 -2
- data/lib/aspera/aoc.rb +28 -33
- data/lib/aspera/ascmd.rb +3 -6
- data/lib/aspera/assert.rb +45 -0
- data/lib/aspera/cli/extended_value.rb +5 -5
- data/lib/aspera/cli/formatter.rb +26 -13
- data/lib/aspera/cli/hints.rb +4 -3
- data/lib/aspera/cli/main.rb +16 -3
- data/lib/aspera/cli/manager.rb +45 -36
- data/lib/aspera/cli/plugin.rb +20 -13
- data/lib/aspera/cli/plugins/aoc.rb +103 -73
- data/lib/aspera/cli/plugins/ats.rb +4 -3
- data/lib/aspera/cli/plugins/config.rb +114 -119
- data/lib/aspera/cli/plugins/cos.rb +2 -2
- data/lib/aspera/cli/plugins/faspex.rb +23 -19
- data/lib/aspera/cli/plugins/faspex5.rb +75 -43
- data/lib/aspera/cli/plugins/node.rb +28 -15
- data/lib/aspera/cli/plugins/orchestrator.rb +4 -2
- data/lib/aspera/cli/plugins/preview.rb +9 -7
- data/lib/aspera/cli/plugins/server.rb +6 -3
- data/lib/aspera/cli/plugins/shares.rb +30 -26
- data/lib/aspera/cli/sync_actions.rb +9 -9
- data/lib/aspera/cli/transfer_agent.rb +21 -14
- data/lib/aspera/cli/transfer_progress.rb +2 -3
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/command_line_builder.rb +13 -11
- data/lib/aspera/cos_node.rb +3 -2
- data/lib/aspera/coverage.rb +22 -0
- data/lib/aspera/data_repository.rb +33 -2
- data/lib/aspera/environment.rb +4 -2
- data/lib/aspera/fasp/{agent_aspera.rb → agent_alpha.rb} +29 -39
- data/lib/aspera/fasp/agent_base.rb +17 -7
- data/lib/aspera/fasp/agent_direct.rb +88 -84
- data/lib/aspera/fasp/agent_httpgw.rb +4 -3
- data/lib/aspera/fasp/agent_node.rb +3 -2
- data/lib/aspera/fasp/agent_trsdk.rb +79 -37
- data/lib/aspera/fasp/installation.rb +51 -12
- data/lib/aspera/fasp/management.rb +11 -6
- data/lib/aspera/fasp/parameters.rb +53 -47
- data/lib/aspera/fasp/resume_policy.rb +7 -5
- data/lib/aspera/fasp/sync.rb +273 -0
- data/lib/aspera/fasp/transfer_spec.rb +10 -8
- data/lib/aspera/fasp/uri.rb +2 -2
- data/lib/aspera/faspex_gw.rb +11 -8
- data/lib/aspera/faspex_postproc.rb +6 -5
- data/lib/aspera/id_generator.rb +3 -1
- data/lib/aspera/json_rpc.rb +10 -8
- data/lib/aspera/keychain/encrypted_hash.rb +46 -11
- data/lib/aspera/keychain/macos_security.rb +15 -13
- data/lib/aspera/log.rb +4 -3
- data/lib/aspera/nagios.rb +7 -2
- data/lib/aspera/node.rb +17 -16
- data/lib/aspera/node_simulator.rb +214 -0
- data/lib/aspera/oauth.rb +22 -19
- data/lib/aspera/persistency_action_once.rb +13 -14
- data/lib/aspera/persistency_folder.rb +3 -2
- data/lib/aspera/preview/file_types.rb +53 -267
- data/lib/aspera/preview/generator.rb +7 -5
- data/lib/aspera/preview/terminal.rb +14 -5
- data/lib/aspera/preview/utils.rb +8 -7
- data/lib/aspera/proxy_auto_config.rb +6 -3
- data/lib/aspera/rest.rb +29 -13
- data/lib/aspera/rest_error_analyzer.rb +1 -0
- data/lib/aspera/rest_errors_aspera.rb +2 -0
- data/lib/aspera/secret_hider.rb +5 -2
- data/lib/aspera/ssh.rb +10 -8
- data/lib/aspera/temp_file_manager.rb +1 -1
- data/lib/aspera/web_server_simple.rb +2 -1
- data.tar.gz.sig +0 -0
- metadata +96 -45
- metadata.gz.sig +0 -0
- data/lib/aspera/sync.rb +0 -219
@@ -1,7 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require 'aspera/cli/plugins/node'
|
4
|
-
|
4
|
+
require 'aspera/assert'
|
5
5
|
module Aspera
|
6
6
|
module Cli
|
7
7
|
module Plugins
|
@@ -49,8 +49,6 @@ module Aspera
|
|
49
49
|
|
50
50
|
def initialize(env)
|
51
51
|
super(env)
|
52
|
-
options.declare(:type, 'Type of user/group for operations', values: %i[any local ldap saml], default: :any)
|
53
|
-
options.parse_options!
|
54
52
|
end
|
55
53
|
|
56
54
|
SAML_IMPORT_MANDATORY = %w[id name_id].freeze
|
@@ -84,63 +82,69 @@ module Aspera
|
|
84
82
|
return Node.new(@agents, api: api_shares_node).execute_action(repo_command)
|
85
83
|
when :admin
|
86
84
|
api_shares_admin = basic_auth_api('api/v1')
|
87
|
-
admin_command = options.get_next_command(%i[user group
|
85
|
+
admin_command = options.get_next_command(%i[node share transfer_settings user group].freeze)
|
88
86
|
case admin_command
|
89
87
|
when :node
|
90
88
|
return entity_action(api_shares_admin, 'data/nodes')
|
89
|
+
when :share
|
90
|
+
share_command = options.get_next_command(%i[user_permissions group_permissions].concat(Plugin::ALL_OPS))
|
91
|
+
case share_command
|
92
|
+
when *Plugin::ALL_OPS
|
93
|
+
return entity_command(share_command, api_shares_admin, 'data/shares')
|
94
|
+
# return {type: :object_list, data: all_shares, fields: %w[id name status status_message]}
|
95
|
+
when :user_permissions, :group_permissions
|
96
|
+
share_id = instance_identifier
|
97
|
+
return entity_action(api_shares_admin, "data/shares/#{share_id}/#{share_command}")
|
98
|
+
end
|
99
|
+
when :transfer_settings
|
100
|
+
xfer_settings_command = options.get_next_command(%i[show modify])
|
101
|
+
return entity_command(xfer_settings_command, api_shares_admin, 'data/transfer_settings', is_singleton: true)
|
91
102
|
when :user, :group
|
92
103
|
entity_type = admin_command
|
93
|
-
entities_location = options.
|
104
|
+
entities_location = options.get_next_command(%i[all local ldap saml])
|
94
105
|
entities_path = "data/#{entities_location}_#{entity_type}s"
|
95
106
|
entity_action = nil
|
96
107
|
case entities_location
|
97
|
-
when :
|
108
|
+
when :all
|
98
109
|
entities_path = "data/#{entity_type}s"
|
99
110
|
entity_action = %i[list show delete]
|
100
111
|
entity_action.concat(USR_GRP_SETTINGS)
|
101
112
|
entity_action.push(:users) if entity_type.eql?(:group)
|
102
113
|
entity_action.freeze
|
103
114
|
when :local
|
104
|
-
entity_action = %i[list show create modify
|
115
|
+
entity_action = %i[list show delete create modify].freeze
|
105
116
|
when :ldap
|
106
117
|
entity_action = %i[add].freeze
|
107
118
|
when :saml
|
108
119
|
entity_action = %i[import].freeze
|
109
120
|
end
|
110
121
|
entity_verb = options.get_next_command(entity_action)
|
111
|
-
# entity_path = "#{entities_path}/#{instance_identifier}" if %i[app_authorizations share_permissions].include?(entity_verb)
|
112
122
|
case entity_verb
|
113
|
-
when *Plugin::ALL_OPS
|
123
|
+
when *Plugin::ALL_OPS # list, show, delete, create, modify
|
114
124
|
display_fields = entity_type.eql?(:user) ? %w[id username first_name last_name email] : nil
|
115
|
-
display_fields.push(:directory_user) if entity_type.eql?(:user) && entities_location.eql?(:
|
125
|
+
display_fields.push(:directory_user) if entity_type.eql?(:user) && entities_location.eql?(:all)
|
116
126
|
return entity_command(entity_verb, api_shares_admin, entities_path, display_fields: display_fields)
|
117
|
-
when
|
127
|
+
when *USR_GRP_SETTINGS # transfer_settings, app_authorizations, share_permissions
|
128
|
+
group_id = instance_identifier
|
129
|
+
entities_path = "#{entities_path}/#{group_id}/#{entity_verb}"
|
130
|
+
return entity_action(api_shares_admin, entities_path, is_singleton: !entity_verb.eql?(:share_permissions))
|
131
|
+
when :import # saml
|
118
132
|
return do_bulk_operation(command: entity_verb, descr: 'user information') do |entity_parameters|
|
119
133
|
entity_parameters = entity_parameters.transform_keys{|k|k.gsub(/\s+/, '_').downcase}
|
120
|
-
|
134
|
+
assert_type(entity_parameters, Hash)
|
121
135
|
SAML_IMPORT_MANDATORY.each{|p|raise "missing mandatory field: #{p}" if entity_parameters[p].nil?}
|
122
136
|
entity_parameters.each_key do |p|
|
123
137
|
raise "unsupported field: #{p}, use: #{SAML_IMPORT_ALLOWED.join(',')}" unless SAML_IMPORT_ALLOWED.include?(p)
|
124
138
|
end
|
125
139
|
api_shares_admin.create("#{entities_path}/import", entity_parameters)[:data]
|
126
140
|
end
|
127
|
-
when :add
|
141
|
+
when :add # ldap
|
128
142
|
return do_bulk_operation(command: entity_verb, descr: "#{entity_type} name", values: String) do |entity_name|
|
129
143
|
api_shares_admin.create(entities_path, {entity_type=>entity_name})[:data]
|
130
144
|
end
|
131
|
-
when
|
132
|
-
|
133
|
-
|
134
|
-
return entity_action(api_shares_admin, entities_path, is_singleton: !entity_verb.eql?(:share_permissions))
|
135
|
-
end
|
136
|
-
when :share
|
137
|
-
share_command = options.get_next_command(%i[user_permissions group_permissions].concat(Plugin::ALL_OPS))
|
138
|
-
case share_command
|
139
|
-
when *Plugin::ALL_OPS
|
140
|
-
return entity_command(share_command, api_shares_admin, 'data/shares')
|
141
|
-
# return {type: :object_list, data: all_shares, fields: %w[id name status status_message]}
|
142
|
-
when :user_permissions, :group_permissions
|
143
|
-
return entity_action(api_shares_admin, "data/shares/#{instance_identifier}/#{share_command}")
|
145
|
+
when :users # group
|
146
|
+
raise "TODO, not implemented"
|
147
|
+
else error_unexpected_value(entity_verb)
|
144
148
|
end
|
145
149
|
end
|
146
150
|
end
|
@@ -1,13 +1,14 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require 'aspera/sync'
|
3
|
+
require 'aspera/fasp/sync'
|
4
|
+
require 'aspera/assert'
|
4
5
|
|
5
6
|
module Aspera
|
6
7
|
module Cli
|
7
8
|
# Module for sync actions
|
8
9
|
module SyncActions
|
9
10
|
SIMPLE_ARGUMENTS_SYNC = {
|
10
|
-
direction: Aspera::Sync::DIRECTIONS,
|
11
|
+
direction: Aspera::Fasp::Sync::DIRECTIONS,
|
11
12
|
local_dir: String,
|
12
13
|
remote_dir: String
|
13
14
|
}.stringify_keys.freeze
|
@@ -19,8 +20,7 @@ module Aspera
|
|
19
20
|
end
|
20
21
|
|
21
22
|
def execute_sync_action(&block)
|
22
|
-
|
23
|
-
raise 'Internal Error: No block given' unless block
|
23
|
+
assert(block){'No block given'}
|
24
24
|
command = options.get_next_command(%i[start admin])
|
25
25
|
# try to get 3 arguments as simple arguments
|
26
26
|
case command
|
@@ -45,13 +45,13 @@ module Aspera
|
|
45
45
|
:sync_info,
|
46
46
|
mandatory: false,
|
47
47
|
default: {'sessions' => [{'name' => File.basename(simple_session_args['local_dir'])}]})
|
48
|
-
|
49
|
-
|
50
|
-
|
48
|
+
assert_type(async_params, Hash){'sync_info'}
|
49
|
+
assert_type(async_params['sessions'], Array){'sync_info[sessions]'}
|
50
|
+
assert_type(async_params['sessions'].first, Hash){'sync_info[sessions][0]'}
|
51
51
|
async_params['sessions'].first.merge!(simple_session_args)
|
52
52
|
end
|
53
53
|
Log.log.debug{Log.dump('async_params', async_params)}
|
54
|
-
Aspera::Sync.start(async_params, &block)
|
54
|
+
Aspera::Fasp::Sync.start(async_params, &block)
|
55
55
|
return Main.result_success
|
56
56
|
when :admin
|
57
57
|
command2 = options.get_next_command([:status])
|
@@ -59,7 +59,7 @@ module Aspera
|
|
59
59
|
when :status
|
60
60
|
sync_session_name = options.get_next_argument('name of sync session', mandatory: false, type: String)
|
61
61
|
async_params = options.get_option(:sync_info, mandatory: true)
|
62
|
-
return {type: :single_object, data: Aspera::Sync.admin_status(async_params, sync_session_name)}
|
62
|
+
return {type: :single_object, data: Aspera::Fasp::Sync.admin_status(async_params, sync_session_name)}
|
63
63
|
end # command2
|
64
64
|
end # command
|
65
65
|
end # execute_action
|
@@ -1,7 +1,10 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require 'aspera/fasp/agent_base'
|
3
4
|
require 'aspera/fasp/transfer_spec'
|
4
5
|
require 'aspera/cli/info'
|
6
|
+
require 'aspera/log'
|
7
|
+
require 'aspera/assert'
|
5
8
|
|
6
9
|
module Aspera
|
7
10
|
module Cli
|
@@ -28,7 +31,7 @@ module Aspera
|
|
28
31
|
:FILE_LIST_FROM_TRANSFER_SPEC,
|
29
32
|
:FILE_LIST_OPTIONS,
|
30
33
|
:DEFAULT_TRANSFER_NOTIFY_TEMPLATE
|
31
|
-
TRANSFER_AGENTS =
|
34
|
+
TRANSFER_AGENTS = Fasp::AgentBase.agent_list.freeze
|
32
35
|
|
33
36
|
class << self
|
34
37
|
# @return :success if all sessions statuses returned by "start" are success
|
@@ -65,7 +68,7 @@ module Aspera
|
|
65
68
|
|
66
69
|
# multiple option are merged
|
67
70
|
def option_transfer_spec=(value)
|
68
|
-
|
71
|
+
assert_type(value, Hash){'ts'}
|
69
72
|
@transfer_spec_command_line.deep_merge!(value)
|
70
73
|
end
|
71
74
|
|
@@ -92,8 +95,8 @@ module Aspera
|
|
92
95
|
end
|
93
96
|
|
94
97
|
# analyze options and create new agent if not already created or set
|
95
|
-
def
|
96
|
-
return
|
98
|
+
def agent_instance
|
99
|
+
return @agent unless @agent.nil?
|
97
100
|
agent_type = @opt_mgr.get_option(:transfer, mandatory: true)
|
98
101
|
# agent plugin is loaded on demand to avoid loading unnecessary dependencies
|
99
102
|
require "aspera/fasp/agent_#{agent_type}"
|
@@ -117,7 +120,8 @@ module Aspera
|
|
117
120
|
# get agent instance
|
118
121
|
new_agent = Kernel.const_get("Aspera::Fasp::Agent#{agent_type.capitalize}").new(agent_options)
|
119
122
|
self.agent_instance = new_agent
|
120
|
-
|
123
|
+
Log.log.debug{"transfer agent is a #{@agent.class}"}
|
124
|
+
return @agent
|
121
125
|
end
|
122
126
|
|
123
127
|
# return destination folder for transfers
|
@@ -133,7 +137,7 @@ module Aspera
|
|
133
137
|
case direction.to_s
|
134
138
|
when Fasp::TransferSpec::DIRECTION_SEND then dest_folder = '/'
|
135
139
|
when Fasp::TransferSpec::DIRECTION_RECEIVE then dest_folder = '.'
|
136
|
-
else
|
140
|
+
else error_unexpected_value(direction)
|
137
141
|
end
|
138
142
|
return dest_folder
|
139
143
|
end
|
@@ -181,14 +185,15 @@ module Aspera
|
|
181
185
|
if !@transfer_paths.nil?
|
182
186
|
Log.log.warn('--sources overrides paths from --ts')
|
183
187
|
end
|
184
|
-
|
188
|
+
source_type=@opt_mgr.get_option(:src_type, mandatory: true)
|
189
|
+
case source_type
|
185
190
|
when :list
|
186
191
|
# when providing a list, just specify source
|
187
192
|
@transfer_paths = file_list.map{|i|{'source' => i}}
|
188
193
|
when :pair
|
189
|
-
|
194
|
+
assert(file_list.length.even?, exception_class: Cli::BadArgument){"When using pair, provide an even number of paths: #{file_list.length}"}
|
190
195
|
@transfer_paths = file_list.each_slice(2).to_a.map{|s, d|{'source' => s, 'destination' => d}}
|
191
|
-
else
|
196
|
+
else error_unexpected_value(source_type)
|
192
197
|
end
|
193
198
|
Log.log.debug{"paths=#{@transfer_paths}"}
|
194
199
|
return @transfer_paths
|
@@ -199,7 +204,7 @@ module Aspera
|
|
199
204
|
# @param rest_token [Rest] if oauth token regeneration supported
|
200
205
|
def start(transfer_spec, rest_token: nil)
|
201
206
|
# check parameters
|
202
|
-
|
207
|
+
assert_type(transfer_spec, Hash){'transfer_spec'}
|
203
208
|
# process :src option
|
204
209
|
case transfer_spec['direction']
|
205
210
|
when Fasp::TransferSpec::DIRECTION_RECEIVE
|
@@ -223,12 +228,14 @@ module Aspera
|
|
223
228
|
@transfer_spec_command_line['paths'] = transfer_spec['paths'] || ts_source_paths
|
224
229
|
# updated transfer spec with command line
|
225
230
|
updated_ts(transfer_spec)
|
231
|
+
# if TS from app has content_protection (e.g. F5), that means content is protected: ask password if not provided
|
232
|
+
if transfer_spec['content_protection'].eql?('decrypt') && !transfer_spec.key?('content_protection_password')
|
233
|
+
transfer_spec['content_protection_password'] = @opt_mgr.prompt_user_input('content protection password', true)
|
234
|
+
end
|
226
235
|
# create transfer agent
|
227
|
-
|
228
|
-
Log.log.debug{"transfer agent is a #{@agent.class}"}
|
229
|
-
@agent.start_transfer(transfer_spec, token_regenerator: rest_token)
|
236
|
+
agent_instance.start_transfer(transfer_spec, token_regenerator: rest_token)
|
230
237
|
# list of: :success or "error message string"
|
231
|
-
result =
|
238
|
+
result = agent_instance.wait_for_completion
|
232
239
|
send_email_transfer_notification(transfer_spec, result)
|
233
240
|
return result
|
234
241
|
end
|
@@ -1,6 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require 'aspera/log'
|
4
|
+
require 'aspera/assert'
|
4
5
|
require 'ruby-progressbar'
|
5
6
|
|
6
7
|
module Aspera
|
@@ -25,9 +26,7 @@ module Aspera
|
|
25
26
|
|
26
27
|
def event(session_id:, type:, info: nil)
|
27
28
|
Log.log.debug{"progress: #{type} #{session_id} #{info}"}
|
28
|
-
|
29
|
-
raise 'Internal error: session_id is nil'
|
30
|
-
end
|
29
|
+
assert(!session_id.nil? || type.eql?(:pre_start)){'session_id is nil'}
|
31
30
|
return if @completed
|
32
31
|
if @progress_bar.nil?
|
33
32
|
@progress_bar = ProgressBar.create(
|
data/lib/aspera/cli/version.rb
CHANGED
@@ -1,5 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require 'aspera/log'
|
4
|
+
require 'aspera/assert'
|
3
5
|
module Aspera
|
4
6
|
# helper class to build command line from a parameter list (key-value hash)
|
5
7
|
# constructor takes hash: { 'param1':'value1', ...}
|
@@ -27,20 +29,20 @@ module Aspera
|
|
27
29
|
# Called by provider of definition before constructor of this class so that params_definition has all mandatory fields
|
28
30
|
def normalize_description(full_description)
|
29
31
|
full_description.each do |name, options|
|
30
|
-
|
32
|
+
assert_type(options, Hash){name}
|
31
33
|
unsupported_keys = options.keys - OPTIONS_KEYS
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
34
|
+
assert(unsupported_keys.empty?){"Unsupported definition keys: #{unsupported_keys}"}
|
35
|
+
assert(options.key?(:cli)){"Missing key: cli for #{name}"}
|
36
|
+
assert_type(options[:cli], Hash){'Key: cli'}
|
37
|
+
assert(options[:cli].key?(:type)){'Missing key: cli.type'}
|
38
|
+
assert_values(options[:cli][:type], CLI_OPTION_TYPES){"Unsupported processing type for #{name}"}
|
37
39
|
# by default : optional
|
38
40
|
options[:mandatory] ||= false
|
39
41
|
options[:desc] ||= ''
|
40
42
|
options[:desc] = "DEPRECATED: #{options[:deprecation]}\n#{options[:desc]}" if options.key?(:deprecation)
|
41
43
|
cli = options[:cli]
|
42
44
|
unsupported_cli_keys = cli.keys - CLI_KEYS
|
43
|
-
|
45
|
+
assert(unsupported_cli_keys.empty?){"Unsupported cli keys: #{unsupported_cli_keys}"}
|
44
46
|
# by default : string, unless it's without arg
|
45
47
|
options[:accepted_types] ||= options[:cli][:type].eql?(:opt_without_arg) ? :bool : :string
|
46
48
|
# single type is placed in array
|
@@ -121,7 +123,7 @@ module Aspera
|
|
121
123
|
when :hash then Hash
|
122
124
|
when :int then Integer
|
123
125
|
when :bool then [TrueClass, FalseClass]
|
124
|
-
else
|
126
|
+
else error_unexpected_value(type_symbol)
|
125
127
|
end
|
126
128
|
end.flatten
|
127
129
|
# check that value is of expected type
|
@@ -150,7 +152,7 @@ module Aspera
|
|
150
152
|
raise Fasp::Error, "unsupported #{name}: #{parameter_value}" if converted_value.nil?
|
151
153
|
parameter_value = converted_value
|
152
154
|
when NilClass
|
153
|
-
else
|
155
|
+
else error_unexpected_value(options[:cli][:convert].class)
|
154
156
|
end
|
155
157
|
|
156
158
|
case processing_type
|
@@ -159,14 +161,14 @@ module Aspera
|
|
159
161
|
when :ignore, :special # ignore this parameter or process later
|
160
162
|
return
|
161
163
|
when :envvar # set in env var
|
162
|
-
|
164
|
+
assert(options[:cli].key?(:variable)){'missing key: cli.variable'}
|
163
165
|
@result[:env][options[:cli][:variable]] = parameter_value
|
164
166
|
when :opt_without_arg # if present and true : just add option without value
|
165
167
|
add_param = false
|
166
168
|
case parameter_value
|
167
169
|
when false then nil # nothing to put on command line, no creation by default
|
168
170
|
when true then add_param = true
|
169
|
-
else
|
171
|
+
else error_unexpected_value(parameter_value){name}
|
170
172
|
end
|
171
173
|
add_param = !add_param if options[:add_on_false]
|
172
174
|
add_command_line_options([options[:cli][:switch]]) if add_param
|
data/lib/aspera/cos_node.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require 'aspera/log'
|
4
|
+
require 'aspera/assert'
|
4
5
|
require 'aspera/rest'
|
5
6
|
require 'aspera/oauth'
|
6
7
|
require 'xmlsimple'
|
@@ -10,9 +11,9 @@ module Aspera
|
|
10
11
|
class << self
|
11
12
|
def parameters_from_svc_credentials(service_credentials, bucket_region)
|
12
13
|
# check necessary contents
|
13
|
-
|
14
|
+
assert_type(service_credentials, Hash){'service_credentials'}
|
14
15
|
%w[apikey resource_instance_id endpoints].each do |field|
|
15
|
-
|
16
|
+
assert(service_credentials.key?(field)){"service_credentials must have a field: #{field}"}
|
16
17
|
end
|
17
18
|
Aspera::Log.dump('service_credentials', service_credentials)
|
18
19
|
# read endpoints from service provided in service credentials
|
@@ -0,0 +1,22 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# coverage for tests
|
4
|
+
if ENV.key?('ENABLE_COVERAGE')
|
5
|
+
require 'simplecov'
|
6
|
+
require 'securerandom'
|
7
|
+
# compute gem source root based on this script location, assuming it is in bin/
|
8
|
+
# use dirname instead of gsub, in case folder separator is not /
|
9
|
+
development_root = 3.times.inject(File.realpath(__FILE__)) { |p, _| File.dirname(p) }
|
10
|
+
SimpleCov.root(development_root)
|
11
|
+
SimpleCov.enable_for_subprocesses if SimpleCov.respond_to?(:enable_for_subprocesses)
|
12
|
+
# keep cache data for 1 day (must be longer that time to run the whole test suite)
|
13
|
+
SimpleCov.merge_timeout(86400)
|
14
|
+
SimpleCov.command_name(SecureRandom.uuid)
|
15
|
+
SimpleCov.at_exit do
|
16
|
+
original_file_descriptor = $stdout
|
17
|
+
$stdout.reopen(File.join(development_root, 'simplecov.log'))
|
18
|
+
SimpleCov.result.format!
|
19
|
+
$stdout.reopen(original_file_descriptor)
|
20
|
+
end
|
21
|
+
SimpleCov.start
|
22
|
+
end
|
@@ -1,15 +1,46 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require 'aspera/
|
3
|
+
require 'aspera/assert'
|
4
4
|
require 'singleton'
|
5
|
+
require 'openssl'
|
5
6
|
|
6
7
|
module Aspera
|
7
8
|
# a simple binary data repository
|
8
9
|
class DataRepository
|
9
10
|
include Singleton
|
11
|
+
# in same order as elements in folder
|
12
|
+
ELEMENTS = %i[dsa rsa uuid aspera.global-cli-client aspera.drive license]
|
13
|
+
START_INDEX = 1
|
14
|
+
DATA_FOLDER_NAME = 'data'
|
15
|
+
|
16
|
+
# decode data as expected as string
|
17
|
+
# @param name [Symbol] name of the data item
|
18
|
+
# @return [String] decoded data
|
19
|
+
def item(name)
|
20
|
+
index = ELEMENTS.index(name)
|
21
|
+
raise ArgumentError, "unknown data item #{name} (#{name.class})" unless index
|
22
|
+
raw_data = data(START_INDEX + index)
|
23
|
+
case name
|
24
|
+
when :dsa, :rsa
|
25
|
+
# generate PEM from DER
|
26
|
+
return OpenSSL::PKey.const_get(name.to_s.upcase).new(raw_data).to_pem
|
27
|
+
when :license
|
28
|
+
return Zlib::Inflate.inflate(raw_data)
|
29
|
+
when :uuid
|
30
|
+
return format('%08x-%04x-%04x-%04x-%04x%08x', *raw_data.unpack('NnnnnN'))
|
31
|
+
when :'aspera.global-cli-client', :'aspera.drive'
|
32
|
+
return Base64.urlsafe_encode64(raw_data)
|
33
|
+
else error_unexpected_value(name)
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
private
|
38
|
+
|
39
|
+
private_constant :START_INDEX, :DATA_FOLDER_NAME
|
40
|
+
|
10
41
|
# get binary value from data repository
|
11
42
|
def data(id)
|
12
|
-
File.read(File.join(__dir__,
|
43
|
+
File.read(File.join(__dir__, DATA_FOLDER_NAME, id.to_s), mode: 'rb')
|
13
44
|
end
|
14
45
|
end
|
15
46
|
end
|
data/lib/aspera/environment.rb
CHANGED
@@ -2,6 +2,7 @@
|
|
2
2
|
|
3
3
|
# cspell:ignore USERPROFILE HOMEDRIVE HOMEPATH LC_CTYPE msys aarch
|
4
4
|
require 'aspera/log'
|
5
|
+
require 'aspera/assert'
|
5
6
|
require 'rbconfig'
|
6
7
|
|
7
8
|
# cspell:words MEBI mswin bccwin
|
@@ -15,10 +16,11 @@ module Aspera
|
|
15
16
|
OS_AIX = :aix
|
16
17
|
OS_LIST = [OS_WINDOWS, OS_X, OS_LINUX, OS_AIX].freeze
|
17
18
|
CPU_X86_64 = :x86_64
|
19
|
+
CPU_ARM64 = :arm64
|
18
20
|
CPU_PPC64 = :ppc64
|
19
21
|
CPU_PPC64LE = :ppc64le
|
20
22
|
CPU_S390 = :s390
|
21
|
-
CPU_LIST = [CPU_X86_64, CPU_PPC64, CPU_PPC64LE, CPU_S390].freeze
|
23
|
+
CPU_LIST = [CPU_X86_64, CPU_ARM64, CPU_PPC64, CPU_PPC64LE, CPU_S390].freeze
|
22
24
|
|
23
25
|
BITS_PER_BYTE = 8
|
24
26
|
MEBI = 1024 * 1024
|
@@ -88,7 +90,7 @@ module Aspera
|
|
88
90
|
|
89
91
|
# value is provided in block
|
90
92
|
def write_file_restricted(path, force: false, mode: nil)
|
91
|
-
|
93
|
+
assert(block_given?, exception_class: Aspera::InternalError)
|
92
94
|
if force || !File.exist?(path)
|
93
95
|
# Windows may give error
|
94
96
|
File.unlink(path) rescue nil
|
@@ -2,13 +2,14 @@
|
|
2
2
|
|
3
3
|
require 'aspera/fasp/agent_base'
|
4
4
|
require 'aspera/rest'
|
5
|
+
require 'aspera/log'
|
5
6
|
require 'aspera/json_rpc'
|
6
7
|
require 'aspera/open_application'
|
7
8
|
require 'securerandom'
|
8
9
|
|
9
10
|
module Aspera
|
10
11
|
module Fasp
|
11
|
-
class
|
12
|
+
class AgentAlpha < Aspera::Fasp::AgentBase
|
12
13
|
# try twice the main init url in sequence
|
13
14
|
START_URIS = ['aspera://']
|
14
15
|
# delay between each try to start connect
|
@@ -66,53 +67,42 @@ module Aspera
|
|
66
67
|
# if there is a token, we ask connect client to use well known ssh private keys
|
67
68
|
# instead of asking password
|
68
69
|
transfer_spec['authentication'] = 'token' if transfer_spec.key?('token')
|
69
|
-
@client_app_api.start_transfer(app_id: @application_id,transfer_spec: transfer_spec)
|
70
|
-
|
70
|
+
result = @client_app_api.start_transfer(app_id: @application_id, desktop_spec: {}, transfer_spec: transfer_spec)
|
71
|
+
@xfer_id = result['uuid']
|
71
72
|
end
|
72
73
|
|
73
74
|
def wait_for_transfers_completion
|
74
|
-
client_activity_args = {'aspera_client_settings' => @client_settings}
|
75
75
|
started = false
|
76
76
|
pre_calc = false
|
77
|
-
session_id = @xfer_id
|
78
77
|
begin
|
79
78
|
loop do
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
79
|
+
transfer = @client_app_api.get_transfer(app_id: @application_id, transfer_id: @xfer_id)
|
80
|
+
case transfer['status']
|
81
|
+
when 'initiating', 'queued'
|
82
|
+
notify_progress(session_id: nil, type: :pre_start, info: transfer['status'])
|
83
|
+
when 'running'
|
84
|
+
if !started
|
85
|
+
notify_progress(session_id: @xfer_id, type: :session_start)
|
86
|
+
started = true
|
87
87
|
end
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
notify_progress(session_id: nil, type: :pre_start, info: transfer['status'])
|
92
|
-
when 'running'
|
93
|
-
if !started
|
94
|
-
notify_progress(session_id: session_id, type: :session_start)
|
95
|
-
started = true
|
96
|
-
end
|
97
|
-
if !pre_calc && (transfer['bytes_expected'] != 0)
|
98
|
-
notify_progress(type: :session_size, session_id: session_id, info: transfer['bytes_expected'])
|
99
|
-
pre_calc = true
|
100
|
-
else
|
101
|
-
notify_progress(type: :transfer, session_id: session_id, info: transfer['bytes_written'])
|
102
|
-
end
|
103
|
-
when 'completed'
|
104
|
-
notify_progress(type: :end, session_id: session_id)
|
105
|
-
break
|
106
|
-
when 'failed'
|
107
|
-
notify_progress(type: :end, session_id: session_id)
|
108
|
-
raise Fasp::Error, transfer['error_desc']
|
109
|
-
when 'cancelled'
|
110
|
-
notify_progress(type: :end, session_id: session_id)
|
111
|
-
raise Fasp::Error, 'Transfer cancelled by user'
|
88
|
+
if !pre_calc && (transfer['bytes_expected'] != 0)
|
89
|
+
notify_progress(type: :session_size, session_id: @xfer_id, info: transfer['bytes_expected'])
|
90
|
+
pre_calc = true
|
112
91
|
else
|
113
|
-
notify_progress(type: :
|
114
|
-
raise Fasp::Error, "unknown status: #{transfer['status']}: #{transfer['error_desc']}"
|
92
|
+
notify_progress(type: :transfer, session_id: @xfer_id, info: transfer['bytes_written'])
|
115
93
|
end
|
94
|
+
when 'completed'
|
95
|
+
notify_progress(type: :end, session_id: @xfer_id)
|
96
|
+
break
|
97
|
+
when 'failed'
|
98
|
+
notify_progress(type: :end, session_id: @xfer_id)
|
99
|
+
raise Fasp::Error, transfer['error_desc']
|
100
|
+
when 'cancelled'
|
101
|
+
notify_progress(type: :end, session_id: @xfer_id)
|
102
|
+
raise Fasp::Error, 'Transfer cancelled by user'
|
103
|
+
else
|
104
|
+
notify_progress(type: :end, session_id: @xfer_id)
|
105
|
+
raise Fasp::Error, "unknown status: #{transfer['status']}: #{transfer['error_desc']}"
|
116
106
|
end
|
117
107
|
sleep(1)
|
118
108
|
end
|
@@ -121,6 +111,6 @@ module Aspera
|
|
121
111
|
end
|
122
112
|
return [:success]
|
123
113
|
end # wait
|
124
|
-
end #
|
114
|
+
end # AgentAlpha
|
125
115
|
end
|
126
116
|
end
|
@@ -1,5 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require 'aspera/log'
|
4
|
+
require 'aspera/assert'
|
3
5
|
module Aspera
|
4
6
|
module Fasp
|
5
7
|
# Base class for transfer agents
|
@@ -10,7 +12,9 @@ module Aspera
|
|
10
12
|
result = options.symbolize_keys
|
11
13
|
available = default.map{|k, v|"#{k}(#{v})"}.join(', ')
|
12
14
|
result.each do |k, _v|
|
13
|
-
|
15
|
+
assert_values(k, default.keys){"transfer agent parameter: #{k}"}
|
16
|
+
# check it is the expected type: too limiting, as we can have an Integer or Float, or symbol and string
|
17
|
+
# raise "Invalid value for transfer agent parameter: #{k}, expect #{default[k].class.name}" unless default[k].nil? || v.is_a?(default[k].class)
|
14
18
|
end
|
15
19
|
default.each do |k, v|
|
16
20
|
raise "Missing required agent parameter: #{k}. Parameters: #{available}" if v.eql?(:required) && !result.key?(k)
|
@@ -18,24 +22,30 @@ module Aspera
|
|
18
22
|
end
|
19
23
|
return result
|
20
24
|
end
|
21
|
-
|
25
|
+
|
26
|
+
def agent_list
|
27
|
+
Dir.entries(File.dirname(File.expand_path(__FILE__))).select do |file|
|
28
|
+
file.start_with?('agent_') && !file.eql?('agent_base.rb')
|
29
|
+
end.map{|file|file.sub(/^agent_/, '').sub(/\.rb$/, '').to_sym}
|
30
|
+
end
|
31
|
+
end
|
22
32
|
def wait_for_completion
|
23
33
|
# list of: :success or "error message string"
|
24
34
|
statuses = wait_for_transfers_completion
|
25
35
|
@progress&.reset
|
26
|
-
|
27
|
-
|
36
|
+
assert_type(statuses, Array)
|
37
|
+
assert(statuses.select{|i|!i.eql?(:success) && !i.is_a?(StandardError)}.empty?){"bad statuses content: #{statuses}"}
|
28
38
|
return statuses
|
29
39
|
end
|
30
40
|
|
31
41
|
private
|
32
42
|
|
33
43
|
def initialize(options)
|
34
|
-
raise 'internal error' unless respond_to?(:start_transfer)
|
35
|
-
raise 'internal error' unless respond_to?(:wait_for_transfers_completion)
|
36
44
|
# method `shutdown` is optional
|
45
|
+
assert(respond_to?(:start_transfer))
|
46
|
+
assert(respond_to?(:wait_for_transfers_completion))
|
47
|
+
assert_type(options, Hash){'transfer agent options'}
|
37
48
|
Log.log.debug{Log.dump(:agent_options, options)}
|
38
|
-
raise "transfer agent options expecting Hash, but have #{options.class}" unless options.is_a?(Hash)
|
39
49
|
@progress = options[:progress]
|
40
50
|
options.delete(:progress)
|
41
51
|
end
|