aspera-cli 4.10.0 → 4.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/BUGS.md +19 -0
- data/CHANGELOG.md +528 -0
- data/CONTRIBUTING.md +143 -0
- data/README.md +977 -589
- data/bin/ascli +4 -4
- data/bin/asession +12 -12
- data/docs/test_env.conf +29 -19
- data/examples/aoc.rb +6 -6
- data/examples/dascli +18 -16
- data/examples/faspex4.rb +15 -15
- data/examples/node.rb +12 -12
- data/examples/proxy.pac +2 -2
- data/examples/server.rb +12 -12
- data/lib/aspera/aoc.rb +344 -272
- data/lib/aspera/ascmd.rb +56 -54
- data/lib/aspera/ats_api.rb +4 -4
- data/lib/aspera/cli/basic_auth_plugin.rb +15 -12
- data/lib/aspera/cli/extended_value.rb +9 -9
- data/lib/aspera/cli/{formater.rb → formatter.rb} +69 -69
- data/lib/aspera/cli/listener/line_dump.rb +1 -1
- data/lib/aspera/cli/listener/logger.rb +1 -1
- data/lib/aspera/cli/listener/progress.rb +5 -6
- data/lib/aspera/cli/listener/progress_multi.rb +16 -21
- data/lib/aspera/cli/main.rb +72 -73
- data/lib/aspera/cli/manager.rb +112 -112
- data/lib/aspera/cli/plugin.rb +68 -48
- data/lib/aspera/cli/plugins/alee.rb +4 -4
- data/lib/aspera/cli/plugins/aoc.rb +322 -720
- data/lib/aspera/cli/plugins/ats.rb +50 -52
- data/lib/aspera/cli/plugins/bss.rb +10 -10
- data/lib/aspera/cli/plugins/config.rb +514 -410
- data/lib/aspera/cli/plugins/console.rb +12 -12
- data/lib/aspera/cli/plugins/cos.rb +18 -20
- data/lib/aspera/cli/plugins/faspex.rb +134 -136
- data/lib/aspera/cli/plugins/faspex5.rb +235 -70
- data/lib/aspera/cli/plugins/node.rb +378 -309
- data/lib/aspera/cli/plugins/orchestrator.rb +52 -49
- data/lib/aspera/cli/plugins/preview.rb +129 -120
- data/lib/aspera/cli/plugins/server.rb +137 -83
- data/lib/aspera/cli/plugins/shares.rb +77 -52
- data/lib/aspera/cli/plugins/sync.rb +13 -33
- data/lib/aspera/cli/transfer_agent.rb +61 -61
- data/lib/aspera/cli/version.rb +2 -1
- data/lib/aspera/colors.rb +3 -3
- data/lib/aspera/command_line_builder.rb +78 -74
- data/lib/aspera/cos_node.rb +31 -29
- data/lib/aspera/data_repository.rb +1 -1
- data/lib/aspera/environment.rb +30 -28
- data/lib/aspera/fasp/agent_base.rb +17 -15
- data/lib/aspera/fasp/agent_connect.rb +34 -32
- data/lib/aspera/fasp/agent_direct.rb +70 -73
- data/lib/aspera/fasp/agent_httpgw.rb +79 -74
- data/lib/aspera/fasp/agent_node.rb +26 -26
- data/lib/aspera/fasp/agent_trsdk.rb +20 -20
- data/lib/aspera/fasp/error.rb +3 -2
- data/lib/aspera/fasp/error_info.rb +11 -8
- data/lib/aspera/fasp/installation.rb +80 -80
- data/lib/aspera/fasp/listener.rb +2 -2
- data/lib/aspera/fasp/parameters.rb +103 -92
- data/lib/aspera/fasp/parameters.yaml +313 -214
- data/lib/aspera/fasp/resume_policy.rb +10 -10
- data/lib/aspera/fasp/transfer_spec.rb +22 -2
- data/lib/aspera/fasp/uri.rb +7 -7
- data/lib/aspera/faspex_gw.rb +80 -159
- data/lib/aspera/faspex_postproc.rb +77 -0
- data/lib/aspera/hash_ext.rb +3 -3
- data/lib/aspera/id_generator.rb +5 -5
- data/lib/aspera/keychain/encrypted_hash.rb +23 -28
- data/lib/aspera/keychain/macos_security.rb +21 -20
- data/lib/aspera/log.rb +13 -13
- data/lib/aspera/nagios.rb +24 -23
- data/lib/aspera/node.rb +217 -38
- data/lib/aspera/oauth.rb +78 -74
- data/lib/aspera/open_application.rb +19 -11
- data/lib/aspera/persistency_action_once.rb +4 -4
- data/lib/aspera/persistency_folder.rb +13 -13
- data/lib/aspera/preview/file_types.rb +8 -8
- data/lib/aspera/preview/generator.rb +67 -67
- data/lib/aspera/preview/utils.rb +27 -27
- data/lib/aspera/proxy_auto_config.js +63 -63
- data/lib/aspera/proxy_auto_config.rb +19 -19
- data/lib/aspera/rest.rb +65 -67
- data/lib/aspera/rest_call_error.rb +2 -1
- data/lib/aspera/rest_error_analyzer.rb +22 -21
- data/lib/aspera/rest_errors_aspera.rb +16 -16
- data/lib/aspera/secret_hider.rb +17 -14
- data/lib/aspera/ssh.rb +15 -14
- data/lib/aspera/sync.rb +177 -62
- data/lib/aspera/temp_file_manager.rb +2 -2
- data/lib/aspera/uri_reader.rb +4 -4
- data/lib/aspera/web_auth.rb +13 -64
- data/lib/aspera/web_server_simple.rb +76 -0
- data.tar.gz.sig +0 -0
- metadata +11 -6
- metadata.gz.sig +0 -0
@@ -20,36 +20,36 @@ module Aspera
|
|
20
20
|
@parameters = DEFAULTS.dup
|
21
21
|
if !params.nil?
|
22
22
|
raise "expecting Hash (or nil), but have #{params.class}" unless params.is_a?(Hash)
|
23
|
-
params.each do |k,v|
|
24
|
-
raise "unknown resume parameter: #{k}, expect one of #{DEFAULTS.keys.map(&:to_s).join(',')}" unless DEFAULTS.
|
23
|
+
params.each do |k, v|
|
24
|
+
raise "unknown resume parameter: #{k}, expect one of #{DEFAULTS.keys.map(&:to_s).join(',')}" unless DEFAULTS.key?(k)
|
25
25
|
raise "#{k} must be Integer" unless v.is_a?(Integer)
|
26
26
|
@parameters[k] = v
|
27
27
|
end
|
28
28
|
end
|
29
|
-
Log.log.debug
|
29
|
+
Log.log.debug{"resume params=#{@parameters}"}
|
30
30
|
end
|
31
31
|
|
32
32
|
# calls block a number of times (resumes) until success or limit reached
|
33
33
|
# this is re-entrant, one resumer can handle multiple transfers in //
|
34
34
|
def execute_with_resume
|
35
|
-
raise 'block
|
35
|
+
raise 'block mandatory' unless block_given?
|
36
36
|
# maximum of retry
|
37
37
|
remaining_resumes = @parameters[:iter_max]
|
38
38
|
sleep_seconds = @parameters[:sleep_initial]
|
39
|
-
Log.log.debug
|
40
|
-
# try to send the file until ascp is
|
39
|
+
Log.log.debug{"retries=#{remaining_resumes}"}
|
40
|
+
# try to send the file until ascp is successful
|
41
41
|
loop do
|
42
|
-
Log.log.debug('transfer starting')
|
42
|
+
Log.log.debug('transfer starting')
|
43
43
|
begin
|
44
44
|
# call provided block
|
45
45
|
yield
|
46
46
|
break
|
47
47
|
rescue Fasp::Error => e
|
48
|
-
Log.log.warn
|
48
|
+
Log.log.warn{"An error occurred: #{e.message}"}
|
49
49
|
# failure in ascp
|
50
50
|
if e.retryable?
|
51
51
|
# exit if we exceed the max number of retry
|
52
|
-
raise Fasp::Error,'Maximum number of retry reached' if remaining_resumes <= 0
|
52
|
+
raise Fasp::Error, 'Maximum number of retry reached' if remaining_resumes <= 0
|
53
53
|
else
|
54
54
|
# give one chance only to non retryable errors
|
55
55
|
unless remaining_resumes.eql?(@parameters[:iter_max])
|
@@ -61,7 +61,7 @@ module Aspera
|
|
61
61
|
|
62
62
|
# take this retry in account
|
63
63
|
remaining_resumes -= 1
|
64
|
-
Log.log.warn
|
64
|
+
Log.log.warn{"resuming in #{sleep_seconds} seconds (retry left:#{remaining_resumes})"}
|
65
65
|
|
66
66
|
# wait a bit before retrying, maybe network condition will be better
|
67
67
|
sleep(sleep_seconds)
|
@@ -16,10 +16,30 @@ module Aspera
|
|
16
16
|
'fasp_port' => UDP_PORT
|
17
17
|
}.freeze
|
18
18
|
# define constants for enums of parameters: <paramater>_<enum>, e.g. CIPHER_AES_128
|
19
|
-
Aspera::Fasp::Parameters.description.each do |k,v|
|
19
|
+
Aspera::Fasp::Parameters.description.each do |k, v|
|
20
20
|
next unless v[:enum].is_a?(Array)
|
21
21
|
v[:enum].each do |enum|
|
22
|
-
TransferSpec.const_set("#{k.to_s.upcase}_#{enum.upcase.gsub(/[^A-Z0-9]/,'_')}", enum.freeze)
|
22
|
+
TransferSpec.const_set("#{k.to_s.upcase}_#{enum.upcase.gsub(/[^A-Z0-9]/, '_')}", enum.freeze)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
class << self
|
26
|
+
def action_to_direction(tspec, command)
|
27
|
+
raise 'transfer spec must be a Hash' unless tspec.is_a?(Hash)
|
28
|
+
tspec['direction'] = case command.to_sym
|
29
|
+
when :upload then DIRECTION_SEND
|
30
|
+
when :download then DIRECTION_RECEIVE
|
31
|
+
else raise 'Error: upload or download only'
|
32
|
+
end
|
33
|
+
return tspec
|
34
|
+
end
|
35
|
+
|
36
|
+
def action(tspec)
|
37
|
+
raise 'transfer spec must be a Hash' unless tspec.is_a?(Hash)
|
38
|
+
return case tspec['direction']
|
39
|
+
when DIRECTION_SEND then :upload
|
40
|
+
when DIRECTION_RECEIVE then :download
|
41
|
+
else raise 'Error: upload or download only'
|
42
|
+
end
|
23
43
|
end
|
24
44
|
end
|
25
45
|
end
|
data/lib/aspera/fasp/uri.rb
CHANGED
@@ -5,10 +5,10 @@ require 'aspera/command_line_builder'
|
|
5
5
|
|
6
6
|
module Aspera
|
7
7
|
module Fasp
|
8
|
-
# translates a "faspe:" URI (used in Faspex) into transfer spec hash
|
8
|
+
# translates a "faspe:" URI (used in Faspex 4) into transfer spec hash
|
9
9
|
class Uri
|
10
10
|
def initialize(fasplink)
|
11
|
-
@fasp_uri = URI.parse(fasplink.gsub(' ','%20'))
|
11
|
+
@fasp_uri = URI.parse(fasplink.gsub(' ', '%20'))
|
12
12
|
# TODO: check scheme is faspe
|
13
13
|
end
|
14
14
|
|
@@ -34,16 +34,16 @@ module Aspera
|
|
34
34
|
when 'minrate' then result_ts['min_rate_kbps'] = value.to_i
|
35
35
|
when 'port' then result_ts['fasp_port'] = value.to_i
|
36
36
|
when 'bwcap' then result_ts['target_rate_cap_kbps'] = value.to_i
|
37
|
-
when 'enc' then result_ts['cipher'] = value.gsub(/^aes/,'aes-').gsub(/cfb$/,'-cfb').gsub(/gcm$/,'-gcm').gsub(/--/,'-')
|
37
|
+
when 'enc' then result_ts['cipher'] = value.gsub(/^aes/, 'aes-').gsub(/cfb$/, '-cfb').gsub(/gcm$/, '-gcm').gsub(/--/, '-')
|
38
38
|
when 'tags64' then result_ts['tags'] = JSON.parse(Base64.strict_decode64(value))
|
39
39
|
when 'createpath' then result_ts['create_dir'] = CommandLineBuilder.yes_to_true(value)
|
40
40
|
when 'fallback' then result_ts['http_fallback'] = CommandLineBuilder.yes_to_true(value)
|
41
41
|
when 'lockpolicy' then result_ts['lock_rate_policy'] = CommandLineBuilder.yes_to_true(value)
|
42
42
|
when 'lockminrate' then result_ts['lock_min_rate'] = CommandLineBuilder.yes_to_true(value)
|
43
|
-
when 'auth' then Log.log.debug
|
44
|
-
when 'v' then Log.log.debug
|
45
|
-
when 'protect' then Log.log.debug
|
46
|
-
else Log.log.warn
|
43
|
+
when 'auth' then Log.log.debug{"ignoring auth #{name}=#{value}"} # TODO: translate into transfer spec ? yes/no
|
44
|
+
when 'v' then Log.log.debug{"ignoring v #{name}=#{value}"} # TODO: translate into transfer spec ? 2
|
45
|
+
when 'protect' then Log.log.debug{"ignoring protect #{name}=#{value}"} # TODO: translate into transfer spec ?
|
46
|
+
else Log.log.warn{"URI parameter ignored: #{name} = #{value}"}
|
47
47
|
end
|
48
48
|
end
|
49
49
|
return result_ts
|
data/lib/aspera/faspex_gw.rb
CHANGED
@@ -1,174 +1,95 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require 'aspera/web_server_simple'
|
3
4
|
require 'aspera/log'
|
4
|
-
require 'aspera/aoc'
|
5
|
-
require 'aspera/fasp/transfer_spec'
|
6
|
-
require 'aspera/cli/main'
|
7
|
-
require 'webrick'
|
8
|
-
require 'webrick/https'
|
9
|
-
require 'securerandom'
|
10
|
-
require 'openssl'
|
11
5
|
require 'json'
|
12
6
|
|
13
7
|
module Aspera
|
14
8
|
# this class answers the Faspex /send API and creates a package on Aspera on Cloud
|
15
|
-
class
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
# {"delivery":{"use_encryption_at_rest":false,"note":"note","sources":[{"paths":["file1"]}],"title":"my title","recipients":["email1"],"send_upload_result":true}}
|
25
|
-
# {
|
26
|
-
# "delivery"=>{
|
27
|
-
# "use_encryption_at_rest"=>false,
|
28
|
-
# "note"=>"note",
|
29
|
-
# "sources"=>[{"paths"=>["file1"]}],
|
30
|
-
# "title"=>"my title",
|
31
|
-
# "recipients"=>["email1"],
|
32
|
-
# "send_upload_result"=>true
|
33
|
-
# }
|
34
|
-
# }
|
35
|
-
def process_faspex_send(request, response)
|
36
|
-
raise 'no payload' if request.body.nil?
|
37
|
-
|
38
|
-
faspex_pkg_parameters = JSON.parse(request.body)
|
39
|
-
faspex_pkg_delivery = faspex_pkg_parameters['delivery']
|
40
|
-
Log.log.debug("faspex pkg create parameters=#{faspex_pkg_parameters}")
|
41
|
-
|
42
|
-
# get recipient ids
|
43
|
-
files_pkg_recipients = []
|
44
|
-
faspex_pkg_delivery['recipients'].each do |recipient_email|
|
45
|
-
user_lookup = @aoc_api_user.read('contacts',
|
46
|
-
{ 'current_workspace_id' => @aoc_workspace_id, 'q' => recipient_email })[:data]
|
47
|
-
raise StandardError,
|
48
|
-
"no such unique user: #{recipient_email} / #{user_lookup}" unless !user_lookup.nil? && user_lookup.length.eql?(1)
|
49
|
-
|
50
|
-
recipient_user_info = user_lookup.first
|
51
|
-
files_pkg_recipients.push({
|
52
|
-
'id' => recipient_user_info['source_id'],
|
53
|
-
'type' => recipient_user_info['source_type']
|
54
|
-
})
|
55
|
-
end
|
56
|
-
|
57
|
-
# create a new package with one file
|
58
|
-
the_package = @aoc_api_user.create('packages', {
|
59
|
-
'file_names' => faspex_pkg_delivery['sources'][0]['paths'],
|
60
|
-
'name' => faspex_pkg_delivery['title'],
|
61
|
-
'note' => faspex_pkg_delivery['note'],
|
62
|
-
'recipients' => files_pkg_recipients,
|
63
|
-
'workspace_id' => @aoc_workspace_id
|
64
|
-
})[:data]
|
65
|
-
|
66
|
-
# get node information for the node on which package must be created
|
67
|
-
node_info = @aoc_api_user.read("nodes/#{the_package['node_id']}")[:data]
|
68
|
-
|
69
|
-
# get transfer token (for node)
|
70
|
-
node_auth_bearer_token = @aoc_api_user.oauth_token(scope: AoC.node_scope(node_info['access_key'],
|
71
|
-
AoC::SCOPE_NODE_USER))
|
72
|
-
|
73
|
-
# tell Files what to expect in package: 1 transfer (can also be done after transfer)
|
74
|
-
@aoc_api_user.update("packages/#{the_package['id']}", { 'sent' => true, 'transfers_expected' => 1 })
|
75
|
-
|
76
|
-
# to return an error:
|
77
|
-
# response.status=400
|
78
|
-
# return 'ERROR HERE'
|
79
|
-
|
80
|
-
# TODO: check about xfer_*
|
81
|
-
ts_tags = {
|
82
|
-
'aspera' => {
|
83
|
-
'files' => { 'package_id' => the_package['id'], 'package_operation' => 'upload' },
|
84
|
-
'node' => { 'access_key' => node_info['access_key'], 'file_id' => the_package['contents_file_id'] },
|
85
|
-
'xfer_id' => SecureRandom.uuid,
|
86
|
-
'xfer_retry' => 3600
|
87
|
-
}
|
88
|
-
}
|
89
|
-
# this transfer spec is for transfer to AoC
|
90
|
-
faspex_transfer_spec = {
|
91
|
-
'direction' => 'send',
|
92
|
-
'remote_host' => node_info['host'],
|
93
|
-
'remote_user' => Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER,
|
94
|
-
'ssh_port' => Fasp::TransferSpec::SSH_PORT,
|
95
|
-
'fasp_port' => Fasp::TransferSpec::UDP_PORT,
|
96
|
-
'tags' => ts_tags,
|
97
|
-
'token' => node_auth_bearer_token,
|
98
|
-
'paths' => [{ 'destination' => '/' }],
|
99
|
-
'cookie' => 'unused',
|
100
|
-
'create_dir' => true,
|
101
|
-
'rate_policy' => 'fair',
|
102
|
-
'rate_policy_allowed' => 'fixed',
|
103
|
-
'min_rate_cap_kbps' => nil,
|
104
|
-
'min_rate_kbps' => 0,
|
105
|
-
'target_rate_percentage' => nil,
|
106
|
-
'lock_target_rate' => nil,
|
107
|
-
'fasp_url' => 'unused',
|
108
|
-
'lock_min_rate' => true,
|
109
|
-
'lock_rate_policy' => true,
|
110
|
-
'source_root' => '',
|
111
|
-
'content_protection' => nil,
|
112
|
-
'target_rate_cap_kbps' => 20_000, # TODO: is this value useful ?
|
113
|
-
'target_rate_kbps' => 10_000, # TODO: get from where?
|
114
|
-
'cipher' => 'aes-128',
|
115
|
-
'cipher_allowed' => nil,
|
116
|
-
'http_fallback' => false,
|
117
|
-
'http_fallback_port' => nil,
|
118
|
-
'https_fallback_port' => nil,
|
119
|
-
'destination_root' => '/'
|
120
|
-
}
|
121
|
-
# but we place it in a Faspex package creation response
|
122
|
-
faspex_package_create_result = {
|
123
|
-
'links' => { 'status' => 'unused' },
|
124
|
-
'xfer_sessions' => [faspex_transfer_spec]
|
125
|
-
}
|
126
|
-
Log.log.info("faspex_package_create_result=#{faspex_package_create_result}")
|
127
|
-
response.status = 200
|
128
|
-
response.content_type = 'application/json'
|
129
|
-
response.body = JSON.generate(faspex_package_create_result)
|
130
|
-
end
|
131
|
-
|
132
|
-
def do_GET(request, response)
|
133
|
-
case request.path
|
134
|
-
when '/aspera/faspex/send'
|
135
|
-
process_faspex_send(request, response)
|
136
|
-
else
|
137
|
-
response.status = 400
|
138
|
-
'ERROR HERE'
|
139
|
-
end
|
140
|
-
end
|
141
|
-
end # FxGwServlet
|
9
|
+
class Faspex4GWServlet < WEBrick::HTTPServlet::AbstractServlet
|
10
|
+
# @param app_api [Aspera::AoC]
|
11
|
+
# @param app_context [String]
|
12
|
+
def initialize(server, app_api, app_context)
|
13
|
+
super(server)
|
14
|
+
# typed: Aspera::AoC
|
15
|
+
@app_api = app_api
|
16
|
+
@app_context = app_context
|
17
|
+
end
|
142
18
|
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
19
|
+
# Map Faspex 4 /send API to AoC package create
|
20
|
+
# parameters from user to Faspex API call
|
21
|
+
# https://developer.ibm.com/apis/catalog/aspera--aspera-faspex-client-sdk/Sending%20Packages%20(API%20v.3)
|
22
|
+
def faspex4_send_to_aoc(faspex_pkg_parameters)
|
23
|
+
faspex_pkg_delivery = faspex_pkg_parameters['delivery']
|
24
|
+
package_data = {
|
25
|
+
# 'file_names' => faspex_pkg_delivery['sources'][0]['paths'],
|
26
|
+
'name' => faspex_pkg_delivery['title'],
|
27
|
+
'note' => faspex_pkg_delivery['note'],
|
28
|
+
'recipients' => faspex_pkg_delivery['recipients'],
|
29
|
+
'workspace_id' => @app_context
|
30
|
+
}
|
31
|
+
created_package = @app_api.create_package_simple(package_data, true, @new_user_option)
|
32
|
+
# but we place it in a Faspex package creation response
|
33
|
+
return {
|
34
|
+
'links' => { 'status' => 'unused' },
|
35
|
+
'xfer_sessions' => [created_package[:spec]]
|
36
|
+
}
|
154
37
|
end
|
155
38
|
|
156
|
-
def
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
39
|
+
def faspex4_send_to_faspex5(faspex_pkg_parameters)
|
40
|
+
faspex_pkg_delivery = faspex_pkg_parameters['delivery']
|
41
|
+
package_data = {
|
42
|
+
'title' => faspex_pkg_delivery['title'],
|
43
|
+
'note' => faspex_pkg_delivery['note'],
|
44
|
+
'recipients' => faspex_pkg_delivery['recipients'].map{|name|{'name'=>name}}
|
45
|
+
}
|
46
|
+
package = @app_api.create('packages', package_data)[:data]
|
47
|
+
# TODO: option to send from remote source or httpgw
|
48
|
+
transfer_spec = @app_api.call(
|
49
|
+
operation: 'POST',
|
50
|
+
subpath: "packages/#{package['id']}/transfer_spec/upload",
|
51
|
+
headers: {'Accept' => 'application/json'},
|
52
|
+
url_params: {transfer_type: Aspera::Cli::Plugins::Faspex5::TRANSFER_CONNECT},
|
53
|
+
json_params: {paths: [{'destination'=>'/'}]}
|
54
|
+
)[:data]
|
55
|
+
transfer_spec.delete('authentication')
|
56
|
+
# but we place it in a Faspex package creation response
|
57
|
+
return {
|
58
|
+
'links' => { 'status' => 'unused' },
|
59
|
+
'xfer_sessions' => [transfer_spec]
|
162
60
|
}
|
163
|
-
Log.log.info("Server started on port #{webrick_options[:Port]}")
|
164
|
-
@server = WEBrick::HTTPServer.new(webrick_options)
|
165
|
-
@server.mount('/aspera/faspex', FxGwServlet, a_aoc_api_user, a_workspace_id)
|
166
|
-
@server.mount('/newuser', NewUserServlet)
|
167
|
-
trap('INT') { @server.shutdown }
|
168
61
|
end
|
169
62
|
|
170
|
-
def
|
171
|
-
|
63
|
+
def do_POST(request, response)
|
64
|
+
case request.path
|
65
|
+
when '/aspera/faspex/send'
|
66
|
+
begin
|
67
|
+
raise 'no payload' if request.body.nil?
|
68
|
+
faspex_pkg_parameters = JSON.parse(request.body)
|
69
|
+
Log.log.debug{"faspex pkg create parameters=#{faspex_pkg_parameters}"}
|
70
|
+
faspex_package_create_result =
|
71
|
+
if @app_api.is_a?(Aspera::AoC)
|
72
|
+
faspex4_send_to_aoc(faspex_pkg_parameters)
|
73
|
+
elsif @app_api.is_a?(Aspera::Rest)
|
74
|
+
faspex4_send_to_faspex5(faspex_pkg_parameters)
|
75
|
+
else
|
76
|
+
raise "No such adapter: #{@app_api.class}"
|
77
|
+
end
|
78
|
+
Log.log.info{"faspex_package_create_result=#{faspex_package_create_result}"}
|
79
|
+
response.status = 200
|
80
|
+
response.content_type = 'application/json'
|
81
|
+
response.body = JSON.generate(faspex_package_create_result)
|
82
|
+
rescue => e
|
83
|
+
response.status = 500
|
84
|
+
response['Content-Type'] = 'application/json'
|
85
|
+
response.body = {error: e.message}.to_json
|
86
|
+
Log.log.error(e.message)
|
87
|
+
end
|
88
|
+
else
|
89
|
+
response.status = 400
|
90
|
+
response['Content-Type'] = 'application/json'
|
91
|
+
response.body = {error: 'Bad request'}.to_json
|
92
|
+
end
|
172
93
|
end
|
173
|
-
end #
|
94
|
+
end # Faspex4GWServlet
|
174
95
|
end # AsperaLm
|
@@ -0,0 +1,77 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'English'
|
4
|
+
require 'aspera/web_server_simple'
|
5
|
+
require 'aspera/log'
|
6
|
+
require 'json'
|
7
|
+
require 'timeout'
|
8
|
+
|
9
|
+
module Aspera
|
10
|
+
# this class answers the Faspex /send API and creates a package on Aspera on Cloud
|
11
|
+
class Faspex4PostProcServlet < WEBrick::HTTPServlet::AbstractServlet
|
12
|
+
ALLOWED_PARAMETERS = %i[root script_folder fail_on_error timeout_seconds].freeze
|
13
|
+
def initialize(server, parameters)
|
14
|
+
raise 'parameters must be Hash' unless parameters.is_a?(Hash)
|
15
|
+
@parameters = parameters.symbolize_keys
|
16
|
+
Log.dump(:postproc_parameters, @parameters)
|
17
|
+
raise "unexpected key in parameters config: only: #{ALLOWED_PARAMETERS.join(', ')}" if @parameters.keys.any?{|k|!ALLOWED_PARAMETERS.include?(k)}
|
18
|
+
@parameters[:script_folder] ||= '.'
|
19
|
+
@parameters[:fail_on_error] ||= false
|
20
|
+
@parameters[:timeout_seconds] ||= 60
|
21
|
+
super(server)
|
22
|
+
Log.log.debug{"Faspex4PostProcServlet initialized"}
|
23
|
+
end
|
24
|
+
|
25
|
+
def do_POST(request, response)
|
26
|
+
Log.log.debug{"request=#{request.path}"}
|
27
|
+
begin
|
28
|
+
# only accept requests on the root
|
29
|
+
if !request.path.start_with?(@parameters[:root])
|
30
|
+
response.status = 400
|
31
|
+
response['Content-Type'] = 'application/json'
|
32
|
+
response.body = {status: 'error', message: 'Request outside domain'}.to_json
|
33
|
+
return
|
34
|
+
end
|
35
|
+
if request.body.nil?
|
36
|
+
response.status = 400
|
37
|
+
response['Content-Type'] = 'application/json'
|
38
|
+
response.body = {status: 'error', message: 'Empty request'}.to_json
|
39
|
+
return
|
40
|
+
end
|
41
|
+
# build script path by removing domain, and adding script folder
|
42
|
+
script_file = request.path[@parameters[:root].size .. ]
|
43
|
+
Log.log.debug{"script file=#{script_file}"}
|
44
|
+
script_path = File.join(@parameters[:script_folder], script_file)
|
45
|
+
Log.log.debug{"script=#{script_path}"}
|
46
|
+
webhook_parameters = JSON.parse(request.body)
|
47
|
+
Log.dump(:webhook_parameters, webhook_parameters)
|
48
|
+
# env expects only strings
|
49
|
+
environment = webhook_parameters.each_with_object({}) { |(k, v), h| h[k] = v.to_s }
|
50
|
+
post_proc_pid = Process.spawn(environment, [script_path, script_path])
|
51
|
+
Log.log.debug{"pid=#{post_proc_pid}"}
|
52
|
+
raise 'no pid' if post_proc_pid.nil?
|
53
|
+
# "wait" for process to avoid zombie
|
54
|
+
Timeout.timeout(@parameters[:timeout_seconds]) do
|
55
|
+
Process.wait(post_proc_pid)
|
56
|
+
post_proc_pid = nil
|
57
|
+
end
|
58
|
+
process_status = $CHILD_STATUS
|
59
|
+
raise "script #{script_path} failed with code #{process_status.exitstatus}" if !process_status.success? && @parameters[:fail_on_error]
|
60
|
+
response.status = 200
|
61
|
+
response.content_type = 'application/json'
|
62
|
+
response.body = JSON.generate({status: 'success', script: script_path, exit_code: process_status.exitstatus})
|
63
|
+
Log.log.debug{'Script executed successfully'}
|
64
|
+
rescue => e
|
65
|
+
Log.log.error("Script failed: #{e.class}:#{e.message}")
|
66
|
+
if !post_proc_pid.nil?
|
67
|
+
Process.kill('SIGKILL', post_proc_pid)
|
68
|
+
Process.wait(post_proc_pid)
|
69
|
+
Log.log.error("Killed process: #{post_proc_pid}")
|
70
|
+
end
|
71
|
+
response.status = 500
|
72
|
+
response['Content-Type'] = 'application/json'
|
73
|
+
response.body = {status: 'error', script: script_path, message: e.message}.to_json
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end # Faspex4PostProcServlet
|
77
|
+
end # AsperaLm
|
data/lib/aspera/hash_ext.rb
CHANGED
@@ -2,11 +2,11 @@
|
|
2
2
|
|
3
3
|
class ::Hash
|
4
4
|
def deep_merge(second)
|
5
|
-
merge(second){|_key,v1,v2|Hash === v1 && Hash === v2 ? v1.deep_merge(v2) : v2}
|
5
|
+
merge(second){|_key, v1, v2|Hash === v1 && Hash === v2 ? v1.deep_merge(v2) : v2}
|
6
6
|
end
|
7
7
|
|
8
8
|
def deep_merge!(second)
|
9
|
-
merge!(second){|_key,v1,v2|Hash === v1 && Hash === v2 ? v1.deep_merge!(v2) : v2}
|
9
|
+
merge!(second){|_key, v1, v2|Hash === v1 && Hash === v2 ? v1.deep_merge!(v2) : v2}
|
10
10
|
end
|
11
11
|
end
|
12
12
|
|
@@ -14,7 +14,7 @@ end
|
|
14
14
|
unless Hash.method_defined?(:transform_keys)
|
15
15
|
class Hash
|
16
16
|
def transform_keys
|
17
|
-
return each_with_object({}){|(k,v),memo|memo[yield(k)]=v} if block_given?
|
17
|
+
return each_with_object({}){|(k, v), memo|memo[yield(k)] = v} if block_given?
|
18
18
|
raise 'missing block'
|
19
19
|
end
|
20
20
|
end
|
data/lib/aspera/id_generator.rb
CHANGED
@@ -7,7 +7,7 @@ module Aspera
|
|
7
7
|
ID_SEPARATOR = '_'
|
8
8
|
WINDOWS_PROTECTED_CHAR = %r{[/:"<>\\*?]}.freeze
|
9
9
|
PROTECTED_CHAR_REPLACE = '_'
|
10
|
-
private_constant :ID_SEPARATOR
|
10
|
+
private_constant :ID_SEPARATOR, :PROTECTED_CHAR_REPLACE, :WINDOWS_PROTECTED_CHAR
|
11
11
|
class << self
|
12
12
|
def from_list(object_id)
|
13
13
|
if object_id.is_a?(Array)
|
@@ -16,10 +16,10 @@ module Aspera
|
|
16
16
|
end.join(ID_SEPARATOR)
|
17
17
|
end
|
18
18
|
raise 'id must be a String' unless object_id.is_a?(String)
|
19
|
-
return object_id
|
20
|
-
gsub(WINDOWS_PROTECTED_CHAR,PROTECTED_CHAR_REPLACE)
|
21
|
-
gsub('.',PROTECTED_CHAR_REPLACE)
|
22
|
-
downcase
|
19
|
+
return object_id
|
20
|
+
.gsub(WINDOWS_PROTECTED_CHAR, PROTECTED_CHAR_REPLACE) # remove windows forbidden chars
|
21
|
+
.gsub('.', PROTECTED_CHAR_REPLACE) # keep dot for extension only (nicer)
|
22
|
+
.downcase
|
23
23
|
end
|
24
24
|
end
|
25
25
|
end
|
@@ -1,6 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require 'aspera/hash_ext'
|
4
|
+
require 'aspera/environment'
|
4
5
|
require 'symmetric_encryption/core'
|
5
6
|
require 'yaml'
|
6
7
|
|
@@ -8,65 +9,59 @@ module Aspera
|
|
8
9
|
module Keychain
|
9
10
|
# Manage secrets in a simple Hash
|
10
11
|
class EncryptedHash
|
11
|
-
CIPHER_NAME='aes-256-cbc'
|
12
|
-
|
13
|
-
def initialize(path,current_password)
|
14
|
-
@path=path
|
15
|
-
self.password=current_password
|
12
|
+
CIPHER_NAME = 'aes-256-cbc'
|
13
|
+
CONTENT_KEYS = %i[label username password url description].freeze
|
14
|
+
def initialize(path, current_password)
|
15
|
+
@path = path
|
16
|
+
self.password = current_password
|
16
17
|
raise 'path to vault file shall be String' unless @path.is_a?(String)
|
17
|
-
@all_secrets=File.exist?(@path) ? YAML.load_stream(@cipher.decrypt(File.read(@path))).first : {}
|
18
|
+
@all_secrets = File.exist?(@path) ? YAML.load_stream(@cipher.decrypt(File.read(@path))).first : {}
|
18
19
|
end
|
19
20
|
|
20
21
|
def password=(new_password)
|
21
|
-
|
22
|
-
|
23
|
-
key
|
24
|
-
|
25
|
-
|
22
|
+
# number of bits in second position
|
23
|
+
key_bytes = CIPHER_NAME.split('-')[1].to_i / Environment::BITS_PER_BYTE
|
24
|
+
# derive key from passphrase, add trailing zeros
|
25
|
+
key = "#{new_password}#{"\x0" * key_bytes}"[0..(key_bytes - 1)]
|
26
|
+
Log.log.debug{"key=[#{key}],#{key.length}"}
|
27
|
+
SymmetricEncryption.cipher = @cipher = SymmetricEncryption::Cipher.new(cipher_name: CIPHER_NAME, key: key, encoding: :none)
|
26
28
|
end
|
27
29
|
|
28
30
|
def save
|
29
|
-
File.write(@path, @cipher.encrypt(YAML.dump(@all_secrets)),encoding: 'BINARY')
|
31
|
+
File.write(@path, @cipher.encrypt(YAML.dump(@all_secrets)), encoding: 'BINARY')
|
30
32
|
end
|
31
33
|
|
32
34
|
def set(options)
|
33
35
|
raise 'options shall be Hash' unless options.is_a?(Hash)
|
34
|
-
unsupported = options.keys -
|
36
|
+
unsupported = options.keys - CONTENT_KEYS
|
37
|
+
options.each_value {|v| raise 'value must be String' unless v.is_a?(String)}
|
35
38
|
raise "unsupported options: #{unsupported}" unless unsupported.empty?
|
36
39
|
label = options.delete(:label)
|
37
|
-
raise "secret #{label} already exist, delete first" if @all_secrets.
|
40
|
+
raise "secret #{label} already exist, delete first" if @all_secrets.key?(label)
|
38
41
|
@all_secrets[label] = options.symbolize_keys
|
39
42
|
save
|
40
43
|
end
|
41
44
|
|
42
45
|
def list
|
43
46
|
result = []
|
44
|
-
@all_secrets.each do |label,values|
|
47
|
+
@all_secrets.each do |label, values|
|
45
48
|
normal = values.symbolize_keys
|
46
49
|
normal[:label] = label
|
47
|
-
|
50
|
+
CONTENT_KEYS.each{|k|normal[k] = '' unless normal.key?(k)}
|
48
51
|
result.push(normal)
|
49
52
|
end
|
50
53
|
return result
|
51
54
|
end
|
52
55
|
|
53
|
-
def delete(
|
54
|
-
raise 'options shall be Hash' unless options.is_a?(Hash)
|
55
|
-
unsupported = options.keys - %i[label]
|
56
|
-
raise "unsupported options: #{unsupported}" unless unsupported.empty?
|
57
|
-
label=options[:label]
|
56
|
+
def delete(label:)
|
58
57
|
@all_secrets.delete(label)
|
59
58
|
save
|
60
59
|
end
|
61
60
|
|
62
|
-
def get(
|
63
|
-
raise
|
64
|
-
unsupported = options.keys - %i[label]
|
65
|
-
raise "unsupported options: #{unsupported}" unless unsupported.empty?
|
66
|
-
label=options[:label]
|
61
|
+
def get(label:, exception: true)
|
62
|
+
raise "Label not found: #{label}" unless @all_secrets.key?(label) || !exception
|
67
63
|
result = @all_secrets[label].clone
|
68
|
-
|
69
|
-
result[:label]=label
|
64
|
+
result[:label] = label if result.is_a?(Hash)
|
70
65
|
return result
|
71
66
|
end
|
72
67
|
end
|