aspera-cli 4.10.0 → 4.12.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/BUGS.md +19 -0
- data/CHANGELOG.md +528 -0
- data/CONTRIBUTING.md +143 -0
- data/README.md +977 -589
- data/bin/ascli +4 -4
- data/bin/asession +12 -12
- data/docs/test_env.conf +29 -19
- data/examples/aoc.rb +6 -6
- data/examples/dascli +18 -16
- data/examples/faspex4.rb +15 -15
- data/examples/node.rb +12 -12
- data/examples/proxy.pac +2 -2
- data/examples/server.rb +12 -12
- data/lib/aspera/aoc.rb +344 -272
- data/lib/aspera/ascmd.rb +56 -54
- data/lib/aspera/ats_api.rb +4 -4
- data/lib/aspera/cli/basic_auth_plugin.rb +15 -12
- data/lib/aspera/cli/extended_value.rb +9 -9
- data/lib/aspera/cli/{formater.rb → formatter.rb} +69 -69
- data/lib/aspera/cli/listener/line_dump.rb +1 -1
- data/lib/aspera/cli/listener/logger.rb +1 -1
- data/lib/aspera/cli/listener/progress.rb +5 -6
- data/lib/aspera/cli/listener/progress_multi.rb +16 -21
- data/lib/aspera/cli/main.rb +72 -73
- data/lib/aspera/cli/manager.rb +112 -112
- data/lib/aspera/cli/plugin.rb +68 -48
- data/lib/aspera/cli/plugins/alee.rb +4 -4
- data/lib/aspera/cli/plugins/aoc.rb +322 -720
- data/lib/aspera/cli/plugins/ats.rb +50 -52
- data/lib/aspera/cli/plugins/bss.rb +10 -10
- data/lib/aspera/cli/plugins/config.rb +514 -410
- data/lib/aspera/cli/plugins/console.rb +12 -12
- data/lib/aspera/cli/plugins/cos.rb +18 -20
- data/lib/aspera/cli/plugins/faspex.rb +134 -136
- data/lib/aspera/cli/plugins/faspex5.rb +235 -70
- data/lib/aspera/cli/plugins/node.rb +378 -309
- data/lib/aspera/cli/plugins/orchestrator.rb +52 -49
- data/lib/aspera/cli/plugins/preview.rb +129 -120
- data/lib/aspera/cli/plugins/server.rb +137 -83
- data/lib/aspera/cli/plugins/shares.rb +77 -52
- data/lib/aspera/cli/plugins/sync.rb +13 -33
- data/lib/aspera/cli/transfer_agent.rb +61 -61
- data/lib/aspera/cli/version.rb +2 -1
- data/lib/aspera/colors.rb +3 -3
- data/lib/aspera/command_line_builder.rb +78 -74
- data/lib/aspera/cos_node.rb +31 -29
- data/lib/aspera/data_repository.rb +1 -1
- data/lib/aspera/environment.rb +30 -28
- data/lib/aspera/fasp/agent_base.rb +17 -15
- data/lib/aspera/fasp/agent_connect.rb +34 -32
- data/lib/aspera/fasp/agent_direct.rb +70 -73
- data/lib/aspera/fasp/agent_httpgw.rb +79 -74
- data/lib/aspera/fasp/agent_node.rb +26 -26
- data/lib/aspera/fasp/agent_trsdk.rb +20 -20
- data/lib/aspera/fasp/error.rb +3 -2
- data/lib/aspera/fasp/error_info.rb +11 -8
- data/lib/aspera/fasp/installation.rb +80 -80
- data/lib/aspera/fasp/listener.rb +2 -2
- data/lib/aspera/fasp/parameters.rb +103 -92
- data/lib/aspera/fasp/parameters.yaml +313 -214
- data/lib/aspera/fasp/resume_policy.rb +10 -10
- data/lib/aspera/fasp/transfer_spec.rb +22 -2
- data/lib/aspera/fasp/uri.rb +7 -7
- data/lib/aspera/faspex_gw.rb +80 -159
- data/lib/aspera/faspex_postproc.rb +77 -0
- data/lib/aspera/hash_ext.rb +3 -3
- data/lib/aspera/id_generator.rb +5 -5
- data/lib/aspera/keychain/encrypted_hash.rb +23 -28
- data/lib/aspera/keychain/macos_security.rb +21 -20
- data/lib/aspera/log.rb +13 -13
- data/lib/aspera/nagios.rb +24 -23
- data/lib/aspera/node.rb +217 -38
- data/lib/aspera/oauth.rb +78 -74
- data/lib/aspera/open_application.rb +19 -11
- data/lib/aspera/persistency_action_once.rb +4 -4
- data/lib/aspera/persistency_folder.rb +13 -13
- data/lib/aspera/preview/file_types.rb +8 -8
- data/lib/aspera/preview/generator.rb +67 -67
- data/lib/aspera/preview/utils.rb +27 -27
- data/lib/aspera/proxy_auto_config.js +63 -63
- data/lib/aspera/proxy_auto_config.rb +19 -19
- data/lib/aspera/rest.rb +65 -67
- data/lib/aspera/rest_call_error.rb +2 -1
- data/lib/aspera/rest_error_analyzer.rb +22 -21
- data/lib/aspera/rest_errors_aspera.rb +16 -16
- data/lib/aspera/secret_hider.rb +17 -14
- data/lib/aspera/ssh.rb +15 -14
- data/lib/aspera/sync.rb +177 -62
- data/lib/aspera/temp_file_manager.rb +2 -2
- data/lib/aspera/uri_reader.rb +4 -4
- data/lib/aspera/web_auth.rb +13 -64
- data/lib/aspera/web_server_simple.rb +76 -0
- data.tar.gz.sig +0 -0
- metadata +11 -6
- metadata.gz.sig +0 -0
@@ -20,36 +20,36 @@ module Aspera
|
|
20
20
|
@parameters = DEFAULTS.dup
|
21
21
|
if !params.nil?
|
22
22
|
raise "expecting Hash (or nil), but have #{params.class}" unless params.is_a?(Hash)
|
23
|
-
params.each do |k,v|
|
24
|
-
raise "unknown resume parameter: #{k}, expect one of #{DEFAULTS.keys.map(&:to_s).join(',')}" unless DEFAULTS.
|
23
|
+
params.each do |k, v|
|
24
|
+
raise "unknown resume parameter: #{k}, expect one of #{DEFAULTS.keys.map(&:to_s).join(',')}" unless DEFAULTS.key?(k)
|
25
25
|
raise "#{k} must be Integer" unless v.is_a?(Integer)
|
26
26
|
@parameters[k] = v
|
27
27
|
end
|
28
28
|
end
|
29
|
-
Log.log.debug
|
29
|
+
Log.log.debug{"resume params=#{@parameters}"}
|
30
30
|
end
|
31
31
|
|
32
32
|
# calls block a number of times (resumes) until success or limit reached
|
33
33
|
# this is re-entrant, one resumer can handle multiple transfers in //
|
34
34
|
def execute_with_resume
|
35
|
-
raise 'block
|
35
|
+
raise 'block mandatory' unless block_given?
|
36
36
|
# maximum of retry
|
37
37
|
remaining_resumes = @parameters[:iter_max]
|
38
38
|
sleep_seconds = @parameters[:sleep_initial]
|
39
|
-
Log.log.debug
|
40
|
-
# try to send the file until ascp is
|
39
|
+
Log.log.debug{"retries=#{remaining_resumes}"}
|
40
|
+
# try to send the file until ascp is successful
|
41
41
|
loop do
|
42
|
-
Log.log.debug('transfer starting')
|
42
|
+
Log.log.debug('transfer starting')
|
43
43
|
begin
|
44
44
|
# call provided block
|
45
45
|
yield
|
46
46
|
break
|
47
47
|
rescue Fasp::Error => e
|
48
|
-
Log.log.warn
|
48
|
+
Log.log.warn{"An error occurred: #{e.message}"}
|
49
49
|
# failure in ascp
|
50
50
|
if e.retryable?
|
51
51
|
# exit if we exceed the max number of retry
|
52
|
-
raise Fasp::Error,'Maximum number of retry reached' if remaining_resumes <= 0
|
52
|
+
raise Fasp::Error, 'Maximum number of retry reached' if remaining_resumes <= 0
|
53
53
|
else
|
54
54
|
# give one chance only to non retryable errors
|
55
55
|
unless remaining_resumes.eql?(@parameters[:iter_max])
|
@@ -61,7 +61,7 @@ module Aspera
|
|
61
61
|
|
62
62
|
# take this retry in account
|
63
63
|
remaining_resumes -= 1
|
64
|
-
Log.log.warn
|
64
|
+
Log.log.warn{"resuming in #{sleep_seconds} seconds (retry left:#{remaining_resumes})"}
|
65
65
|
|
66
66
|
# wait a bit before retrying, maybe network condition will be better
|
67
67
|
sleep(sleep_seconds)
|
@@ -16,10 +16,30 @@ module Aspera
|
|
16
16
|
'fasp_port' => UDP_PORT
|
17
17
|
}.freeze
|
18
18
|
# define constants for enums of parameters: <paramater>_<enum>, e.g. CIPHER_AES_128
|
19
|
-
Aspera::Fasp::Parameters.description.each do |k,v|
|
19
|
+
Aspera::Fasp::Parameters.description.each do |k, v|
|
20
20
|
next unless v[:enum].is_a?(Array)
|
21
21
|
v[:enum].each do |enum|
|
22
|
-
TransferSpec.const_set("#{k.to_s.upcase}_#{enum.upcase.gsub(/[^A-Z0-9]/,'_')}", enum.freeze)
|
22
|
+
TransferSpec.const_set("#{k.to_s.upcase}_#{enum.upcase.gsub(/[^A-Z0-9]/, '_')}", enum.freeze)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
class << self
|
26
|
+
def action_to_direction(tspec, command)
|
27
|
+
raise 'transfer spec must be a Hash' unless tspec.is_a?(Hash)
|
28
|
+
tspec['direction'] = case command.to_sym
|
29
|
+
when :upload then DIRECTION_SEND
|
30
|
+
when :download then DIRECTION_RECEIVE
|
31
|
+
else raise 'Error: upload or download only'
|
32
|
+
end
|
33
|
+
return tspec
|
34
|
+
end
|
35
|
+
|
36
|
+
def action(tspec)
|
37
|
+
raise 'transfer spec must be a Hash' unless tspec.is_a?(Hash)
|
38
|
+
return case tspec['direction']
|
39
|
+
when DIRECTION_SEND then :upload
|
40
|
+
when DIRECTION_RECEIVE then :download
|
41
|
+
else raise 'Error: upload or download only'
|
42
|
+
end
|
23
43
|
end
|
24
44
|
end
|
25
45
|
end
|
data/lib/aspera/fasp/uri.rb
CHANGED
@@ -5,10 +5,10 @@ require 'aspera/command_line_builder'
|
|
5
5
|
|
6
6
|
module Aspera
|
7
7
|
module Fasp
|
8
|
-
# translates a "faspe:" URI (used in Faspex) into transfer spec hash
|
8
|
+
# translates a "faspe:" URI (used in Faspex 4) into transfer spec hash
|
9
9
|
class Uri
|
10
10
|
def initialize(fasplink)
|
11
|
-
@fasp_uri = URI.parse(fasplink.gsub(' ','%20'))
|
11
|
+
@fasp_uri = URI.parse(fasplink.gsub(' ', '%20'))
|
12
12
|
# TODO: check scheme is faspe
|
13
13
|
end
|
14
14
|
|
@@ -34,16 +34,16 @@ module Aspera
|
|
34
34
|
when 'minrate' then result_ts['min_rate_kbps'] = value.to_i
|
35
35
|
when 'port' then result_ts['fasp_port'] = value.to_i
|
36
36
|
when 'bwcap' then result_ts['target_rate_cap_kbps'] = value.to_i
|
37
|
-
when 'enc' then result_ts['cipher'] = value.gsub(/^aes/,'aes-').gsub(/cfb$/,'-cfb').gsub(/gcm$/,'-gcm').gsub(/--/,'-')
|
37
|
+
when 'enc' then result_ts['cipher'] = value.gsub(/^aes/, 'aes-').gsub(/cfb$/, '-cfb').gsub(/gcm$/, '-gcm').gsub(/--/, '-')
|
38
38
|
when 'tags64' then result_ts['tags'] = JSON.parse(Base64.strict_decode64(value))
|
39
39
|
when 'createpath' then result_ts['create_dir'] = CommandLineBuilder.yes_to_true(value)
|
40
40
|
when 'fallback' then result_ts['http_fallback'] = CommandLineBuilder.yes_to_true(value)
|
41
41
|
when 'lockpolicy' then result_ts['lock_rate_policy'] = CommandLineBuilder.yes_to_true(value)
|
42
42
|
when 'lockminrate' then result_ts['lock_min_rate'] = CommandLineBuilder.yes_to_true(value)
|
43
|
-
when 'auth' then Log.log.debug
|
44
|
-
when 'v' then Log.log.debug
|
45
|
-
when 'protect' then Log.log.debug
|
46
|
-
else Log.log.warn
|
43
|
+
when 'auth' then Log.log.debug{"ignoring auth #{name}=#{value}"} # TODO: translate into transfer spec ? yes/no
|
44
|
+
when 'v' then Log.log.debug{"ignoring v #{name}=#{value}"} # TODO: translate into transfer spec ? 2
|
45
|
+
when 'protect' then Log.log.debug{"ignoring protect #{name}=#{value}"} # TODO: translate into transfer spec ?
|
46
|
+
else Log.log.warn{"URI parameter ignored: #{name} = #{value}"}
|
47
47
|
end
|
48
48
|
end
|
49
49
|
return result_ts
|
data/lib/aspera/faspex_gw.rb
CHANGED
@@ -1,174 +1,95 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require 'aspera/web_server_simple'
|
3
4
|
require 'aspera/log'
|
4
|
-
require 'aspera/aoc'
|
5
|
-
require 'aspera/fasp/transfer_spec'
|
6
|
-
require 'aspera/cli/main'
|
7
|
-
require 'webrick'
|
8
|
-
require 'webrick/https'
|
9
|
-
require 'securerandom'
|
10
|
-
require 'openssl'
|
11
5
|
require 'json'
|
12
6
|
|
13
7
|
module Aspera
|
14
8
|
# this class answers the Faspex /send API and creates a package on Aspera on Cloud
|
15
|
-
class
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
# {"delivery":{"use_encryption_at_rest":false,"note":"note","sources":[{"paths":["file1"]}],"title":"my title","recipients":["email1"],"send_upload_result":true}}
|
25
|
-
# {
|
26
|
-
# "delivery"=>{
|
27
|
-
# "use_encryption_at_rest"=>false,
|
28
|
-
# "note"=>"note",
|
29
|
-
# "sources"=>[{"paths"=>["file1"]}],
|
30
|
-
# "title"=>"my title",
|
31
|
-
# "recipients"=>["email1"],
|
32
|
-
# "send_upload_result"=>true
|
33
|
-
# }
|
34
|
-
# }
|
35
|
-
def process_faspex_send(request, response)
|
36
|
-
raise 'no payload' if request.body.nil?
|
37
|
-
|
38
|
-
faspex_pkg_parameters = JSON.parse(request.body)
|
39
|
-
faspex_pkg_delivery = faspex_pkg_parameters['delivery']
|
40
|
-
Log.log.debug("faspex pkg create parameters=#{faspex_pkg_parameters}")
|
41
|
-
|
42
|
-
# get recipient ids
|
43
|
-
files_pkg_recipients = []
|
44
|
-
faspex_pkg_delivery['recipients'].each do |recipient_email|
|
45
|
-
user_lookup = @aoc_api_user.read('contacts',
|
46
|
-
{ 'current_workspace_id' => @aoc_workspace_id, 'q' => recipient_email })[:data]
|
47
|
-
raise StandardError,
|
48
|
-
"no such unique user: #{recipient_email} / #{user_lookup}" unless !user_lookup.nil? && user_lookup.length.eql?(1)
|
49
|
-
|
50
|
-
recipient_user_info = user_lookup.first
|
51
|
-
files_pkg_recipients.push({
|
52
|
-
'id' => recipient_user_info['source_id'],
|
53
|
-
'type' => recipient_user_info['source_type']
|
54
|
-
})
|
55
|
-
end
|
56
|
-
|
57
|
-
# create a new package with one file
|
58
|
-
the_package = @aoc_api_user.create('packages', {
|
59
|
-
'file_names' => faspex_pkg_delivery['sources'][0]['paths'],
|
60
|
-
'name' => faspex_pkg_delivery['title'],
|
61
|
-
'note' => faspex_pkg_delivery['note'],
|
62
|
-
'recipients' => files_pkg_recipients,
|
63
|
-
'workspace_id' => @aoc_workspace_id
|
64
|
-
})[:data]
|
65
|
-
|
66
|
-
# get node information for the node on which package must be created
|
67
|
-
node_info = @aoc_api_user.read("nodes/#{the_package['node_id']}")[:data]
|
68
|
-
|
69
|
-
# get transfer token (for node)
|
70
|
-
node_auth_bearer_token = @aoc_api_user.oauth_token(scope: AoC.node_scope(node_info['access_key'],
|
71
|
-
AoC::SCOPE_NODE_USER))
|
72
|
-
|
73
|
-
# tell Files what to expect in package: 1 transfer (can also be done after transfer)
|
74
|
-
@aoc_api_user.update("packages/#{the_package['id']}", { 'sent' => true, 'transfers_expected' => 1 })
|
75
|
-
|
76
|
-
# to return an error:
|
77
|
-
# response.status=400
|
78
|
-
# return 'ERROR HERE'
|
79
|
-
|
80
|
-
# TODO: check about xfer_*
|
81
|
-
ts_tags = {
|
82
|
-
'aspera' => {
|
83
|
-
'files' => { 'package_id' => the_package['id'], 'package_operation' => 'upload' },
|
84
|
-
'node' => { 'access_key' => node_info['access_key'], 'file_id' => the_package['contents_file_id'] },
|
85
|
-
'xfer_id' => SecureRandom.uuid,
|
86
|
-
'xfer_retry' => 3600
|
87
|
-
}
|
88
|
-
}
|
89
|
-
# this transfer spec is for transfer to AoC
|
90
|
-
faspex_transfer_spec = {
|
91
|
-
'direction' => 'send',
|
92
|
-
'remote_host' => node_info['host'],
|
93
|
-
'remote_user' => Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER,
|
94
|
-
'ssh_port' => Fasp::TransferSpec::SSH_PORT,
|
95
|
-
'fasp_port' => Fasp::TransferSpec::UDP_PORT,
|
96
|
-
'tags' => ts_tags,
|
97
|
-
'token' => node_auth_bearer_token,
|
98
|
-
'paths' => [{ 'destination' => '/' }],
|
99
|
-
'cookie' => 'unused',
|
100
|
-
'create_dir' => true,
|
101
|
-
'rate_policy' => 'fair',
|
102
|
-
'rate_policy_allowed' => 'fixed',
|
103
|
-
'min_rate_cap_kbps' => nil,
|
104
|
-
'min_rate_kbps' => 0,
|
105
|
-
'target_rate_percentage' => nil,
|
106
|
-
'lock_target_rate' => nil,
|
107
|
-
'fasp_url' => 'unused',
|
108
|
-
'lock_min_rate' => true,
|
109
|
-
'lock_rate_policy' => true,
|
110
|
-
'source_root' => '',
|
111
|
-
'content_protection' => nil,
|
112
|
-
'target_rate_cap_kbps' => 20_000, # TODO: is this value useful ?
|
113
|
-
'target_rate_kbps' => 10_000, # TODO: get from where?
|
114
|
-
'cipher' => 'aes-128',
|
115
|
-
'cipher_allowed' => nil,
|
116
|
-
'http_fallback' => false,
|
117
|
-
'http_fallback_port' => nil,
|
118
|
-
'https_fallback_port' => nil,
|
119
|
-
'destination_root' => '/'
|
120
|
-
}
|
121
|
-
# but we place it in a Faspex package creation response
|
122
|
-
faspex_package_create_result = {
|
123
|
-
'links' => { 'status' => 'unused' },
|
124
|
-
'xfer_sessions' => [faspex_transfer_spec]
|
125
|
-
}
|
126
|
-
Log.log.info("faspex_package_create_result=#{faspex_package_create_result}")
|
127
|
-
response.status = 200
|
128
|
-
response.content_type = 'application/json'
|
129
|
-
response.body = JSON.generate(faspex_package_create_result)
|
130
|
-
end
|
131
|
-
|
132
|
-
def do_GET(request, response)
|
133
|
-
case request.path
|
134
|
-
when '/aspera/faspex/send'
|
135
|
-
process_faspex_send(request, response)
|
136
|
-
else
|
137
|
-
response.status = 400
|
138
|
-
'ERROR HERE'
|
139
|
-
end
|
140
|
-
end
|
141
|
-
end # FxGwServlet
|
9
|
+
class Faspex4GWServlet < WEBrick::HTTPServlet::AbstractServlet
|
10
|
+
# @param app_api [Aspera::AoC]
|
11
|
+
# @param app_context [String]
|
12
|
+
def initialize(server, app_api, app_context)
|
13
|
+
super(server)
|
14
|
+
# typed: Aspera::AoC
|
15
|
+
@app_api = app_api
|
16
|
+
@app_context = app_context
|
17
|
+
end
|
142
18
|
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
19
|
+
# Map Faspex 4 /send API to AoC package create
|
20
|
+
# parameters from user to Faspex API call
|
21
|
+
# https://developer.ibm.com/apis/catalog/aspera--aspera-faspex-client-sdk/Sending%20Packages%20(API%20v.3)
|
22
|
+
def faspex4_send_to_aoc(faspex_pkg_parameters)
|
23
|
+
faspex_pkg_delivery = faspex_pkg_parameters['delivery']
|
24
|
+
package_data = {
|
25
|
+
# 'file_names' => faspex_pkg_delivery['sources'][0]['paths'],
|
26
|
+
'name' => faspex_pkg_delivery['title'],
|
27
|
+
'note' => faspex_pkg_delivery['note'],
|
28
|
+
'recipients' => faspex_pkg_delivery['recipients'],
|
29
|
+
'workspace_id' => @app_context
|
30
|
+
}
|
31
|
+
created_package = @app_api.create_package_simple(package_data, true, @new_user_option)
|
32
|
+
# but we place it in a Faspex package creation response
|
33
|
+
return {
|
34
|
+
'links' => { 'status' => 'unused' },
|
35
|
+
'xfer_sessions' => [created_package[:spec]]
|
36
|
+
}
|
154
37
|
end
|
155
38
|
|
156
|
-
def
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
39
|
+
def faspex4_send_to_faspex5(faspex_pkg_parameters)
|
40
|
+
faspex_pkg_delivery = faspex_pkg_parameters['delivery']
|
41
|
+
package_data = {
|
42
|
+
'title' => faspex_pkg_delivery['title'],
|
43
|
+
'note' => faspex_pkg_delivery['note'],
|
44
|
+
'recipients' => faspex_pkg_delivery['recipients'].map{|name|{'name'=>name}}
|
45
|
+
}
|
46
|
+
package = @app_api.create('packages', package_data)[:data]
|
47
|
+
# TODO: option to send from remote source or httpgw
|
48
|
+
transfer_spec = @app_api.call(
|
49
|
+
operation: 'POST',
|
50
|
+
subpath: "packages/#{package['id']}/transfer_spec/upload",
|
51
|
+
headers: {'Accept' => 'application/json'},
|
52
|
+
url_params: {transfer_type: Aspera::Cli::Plugins::Faspex5::TRANSFER_CONNECT},
|
53
|
+
json_params: {paths: [{'destination'=>'/'}]}
|
54
|
+
)[:data]
|
55
|
+
transfer_spec.delete('authentication')
|
56
|
+
# but we place it in a Faspex package creation response
|
57
|
+
return {
|
58
|
+
'links' => { 'status' => 'unused' },
|
59
|
+
'xfer_sessions' => [transfer_spec]
|
162
60
|
}
|
163
|
-
Log.log.info("Server started on port #{webrick_options[:Port]}")
|
164
|
-
@server = WEBrick::HTTPServer.new(webrick_options)
|
165
|
-
@server.mount('/aspera/faspex', FxGwServlet, a_aoc_api_user, a_workspace_id)
|
166
|
-
@server.mount('/newuser', NewUserServlet)
|
167
|
-
trap('INT') { @server.shutdown }
|
168
61
|
end
|
169
62
|
|
170
|
-
def
|
171
|
-
|
63
|
+
def do_POST(request, response)
|
64
|
+
case request.path
|
65
|
+
when '/aspera/faspex/send'
|
66
|
+
begin
|
67
|
+
raise 'no payload' if request.body.nil?
|
68
|
+
faspex_pkg_parameters = JSON.parse(request.body)
|
69
|
+
Log.log.debug{"faspex pkg create parameters=#{faspex_pkg_parameters}"}
|
70
|
+
faspex_package_create_result =
|
71
|
+
if @app_api.is_a?(Aspera::AoC)
|
72
|
+
faspex4_send_to_aoc(faspex_pkg_parameters)
|
73
|
+
elsif @app_api.is_a?(Aspera::Rest)
|
74
|
+
faspex4_send_to_faspex5(faspex_pkg_parameters)
|
75
|
+
else
|
76
|
+
raise "No such adapter: #{@app_api.class}"
|
77
|
+
end
|
78
|
+
Log.log.info{"faspex_package_create_result=#{faspex_package_create_result}"}
|
79
|
+
response.status = 200
|
80
|
+
response.content_type = 'application/json'
|
81
|
+
response.body = JSON.generate(faspex_package_create_result)
|
82
|
+
rescue => e
|
83
|
+
response.status = 500
|
84
|
+
response['Content-Type'] = 'application/json'
|
85
|
+
response.body = {error: e.message}.to_json
|
86
|
+
Log.log.error(e.message)
|
87
|
+
end
|
88
|
+
else
|
89
|
+
response.status = 400
|
90
|
+
response['Content-Type'] = 'application/json'
|
91
|
+
response.body = {error: 'Bad request'}.to_json
|
92
|
+
end
|
172
93
|
end
|
173
|
-
end #
|
94
|
+
end # Faspex4GWServlet
|
174
95
|
end # AsperaLm
|
@@ -0,0 +1,77 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'English'
|
4
|
+
require 'aspera/web_server_simple'
|
5
|
+
require 'aspera/log'
|
6
|
+
require 'json'
|
7
|
+
require 'timeout'
|
8
|
+
|
9
|
+
module Aspera
|
10
|
+
# this class answers the Faspex /send API and creates a package on Aspera on Cloud
|
11
|
+
class Faspex4PostProcServlet < WEBrick::HTTPServlet::AbstractServlet
|
12
|
+
ALLOWED_PARAMETERS = %i[root script_folder fail_on_error timeout_seconds].freeze
|
13
|
+
def initialize(server, parameters)
|
14
|
+
raise 'parameters must be Hash' unless parameters.is_a?(Hash)
|
15
|
+
@parameters = parameters.symbolize_keys
|
16
|
+
Log.dump(:postproc_parameters, @parameters)
|
17
|
+
raise "unexpected key in parameters config: only: #{ALLOWED_PARAMETERS.join(', ')}" if @parameters.keys.any?{|k|!ALLOWED_PARAMETERS.include?(k)}
|
18
|
+
@parameters[:script_folder] ||= '.'
|
19
|
+
@parameters[:fail_on_error] ||= false
|
20
|
+
@parameters[:timeout_seconds] ||= 60
|
21
|
+
super(server)
|
22
|
+
Log.log.debug{"Faspex4PostProcServlet initialized"}
|
23
|
+
end
|
24
|
+
|
25
|
+
def do_POST(request, response)
|
26
|
+
Log.log.debug{"request=#{request.path}"}
|
27
|
+
begin
|
28
|
+
# only accept requests on the root
|
29
|
+
if !request.path.start_with?(@parameters[:root])
|
30
|
+
response.status = 400
|
31
|
+
response['Content-Type'] = 'application/json'
|
32
|
+
response.body = {status: 'error', message: 'Request outside domain'}.to_json
|
33
|
+
return
|
34
|
+
end
|
35
|
+
if request.body.nil?
|
36
|
+
response.status = 400
|
37
|
+
response['Content-Type'] = 'application/json'
|
38
|
+
response.body = {status: 'error', message: 'Empty request'}.to_json
|
39
|
+
return
|
40
|
+
end
|
41
|
+
# build script path by removing domain, and adding script folder
|
42
|
+
script_file = request.path[@parameters[:root].size .. ]
|
43
|
+
Log.log.debug{"script file=#{script_file}"}
|
44
|
+
script_path = File.join(@parameters[:script_folder], script_file)
|
45
|
+
Log.log.debug{"script=#{script_path}"}
|
46
|
+
webhook_parameters = JSON.parse(request.body)
|
47
|
+
Log.dump(:webhook_parameters, webhook_parameters)
|
48
|
+
# env expects only strings
|
49
|
+
environment = webhook_parameters.each_with_object({}) { |(k, v), h| h[k] = v.to_s }
|
50
|
+
post_proc_pid = Process.spawn(environment, [script_path, script_path])
|
51
|
+
Log.log.debug{"pid=#{post_proc_pid}"}
|
52
|
+
raise 'no pid' if post_proc_pid.nil?
|
53
|
+
# "wait" for process to avoid zombie
|
54
|
+
Timeout.timeout(@parameters[:timeout_seconds]) do
|
55
|
+
Process.wait(post_proc_pid)
|
56
|
+
post_proc_pid = nil
|
57
|
+
end
|
58
|
+
process_status = $CHILD_STATUS
|
59
|
+
raise "script #{script_path} failed with code #{process_status.exitstatus}" if !process_status.success? && @parameters[:fail_on_error]
|
60
|
+
response.status = 200
|
61
|
+
response.content_type = 'application/json'
|
62
|
+
response.body = JSON.generate({status: 'success', script: script_path, exit_code: process_status.exitstatus})
|
63
|
+
Log.log.debug{'Script executed successfully'}
|
64
|
+
rescue => e
|
65
|
+
Log.log.error("Script failed: #{e.class}:#{e.message}")
|
66
|
+
if !post_proc_pid.nil?
|
67
|
+
Process.kill('SIGKILL', post_proc_pid)
|
68
|
+
Process.wait(post_proc_pid)
|
69
|
+
Log.log.error("Killed process: #{post_proc_pid}")
|
70
|
+
end
|
71
|
+
response.status = 500
|
72
|
+
response['Content-Type'] = 'application/json'
|
73
|
+
response.body = {status: 'error', script: script_path, message: e.message}.to_json
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end # Faspex4PostProcServlet
|
77
|
+
end # AsperaLm
|
data/lib/aspera/hash_ext.rb
CHANGED
@@ -2,11 +2,11 @@
|
|
2
2
|
|
3
3
|
class ::Hash
|
4
4
|
def deep_merge(second)
|
5
|
-
merge(second){|_key,v1,v2|Hash === v1 && Hash === v2 ? v1.deep_merge(v2) : v2}
|
5
|
+
merge(second){|_key, v1, v2|Hash === v1 && Hash === v2 ? v1.deep_merge(v2) : v2}
|
6
6
|
end
|
7
7
|
|
8
8
|
def deep_merge!(second)
|
9
|
-
merge!(second){|_key,v1,v2|Hash === v1 && Hash === v2 ? v1.deep_merge!(v2) : v2}
|
9
|
+
merge!(second){|_key, v1, v2|Hash === v1 && Hash === v2 ? v1.deep_merge!(v2) : v2}
|
10
10
|
end
|
11
11
|
end
|
12
12
|
|
@@ -14,7 +14,7 @@ end
|
|
14
14
|
unless Hash.method_defined?(:transform_keys)
|
15
15
|
class Hash
|
16
16
|
def transform_keys
|
17
|
-
return each_with_object({}){|(k,v),memo|memo[yield(k)]=v} if block_given?
|
17
|
+
return each_with_object({}){|(k, v), memo|memo[yield(k)] = v} if block_given?
|
18
18
|
raise 'missing block'
|
19
19
|
end
|
20
20
|
end
|
data/lib/aspera/id_generator.rb
CHANGED
@@ -7,7 +7,7 @@ module Aspera
|
|
7
7
|
ID_SEPARATOR = '_'
|
8
8
|
WINDOWS_PROTECTED_CHAR = %r{[/:"<>\\*?]}.freeze
|
9
9
|
PROTECTED_CHAR_REPLACE = '_'
|
10
|
-
private_constant :ID_SEPARATOR
|
10
|
+
private_constant :ID_SEPARATOR, :PROTECTED_CHAR_REPLACE, :WINDOWS_PROTECTED_CHAR
|
11
11
|
class << self
|
12
12
|
def from_list(object_id)
|
13
13
|
if object_id.is_a?(Array)
|
@@ -16,10 +16,10 @@ module Aspera
|
|
16
16
|
end.join(ID_SEPARATOR)
|
17
17
|
end
|
18
18
|
raise 'id must be a String' unless object_id.is_a?(String)
|
19
|
-
return object_id
|
20
|
-
gsub(WINDOWS_PROTECTED_CHAR,PROTECTED_CHAR_REPLACE)
|
21
|
-
gsub('.',PROTECTED_CHAR_REPLACE)
|
22
|
-
downcase
|
19
|
+
return object_id
|
20
|
+
.gsub(WINDOWS_PROTECTED_CHAR, PROTECTED_CHAR_REPLACE) # remove windows forbidden chars
|
21
|
+
.gsub('.', PROTECTED_CHAR_REPLACE) # keep dot for extension only (nicer)
|
22
|
+
.downcase
|
23
23
|
end
|
24
24
|
end
|
25
25
|
end
|
@@ -1,6 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require 'aspera/hash_ext'
|
4
|
+
require 'aspera/environment'
|
4
5
|
require 'symmetric_encryption/core'
|
5
6
|
require 'yaml'
|
6
7
|
|
@@ -8,65 +9,59 @@ module Aspera
|
|
8
9
|
module Keychain
|
9
10
|
# Manage secrets in a simple Hash
|
10
11
|
class EncryptedHash
|
11
|
-
CIPHER_NAME='aes-256-cbc'
|
12
|
-
|
13
|
-
def initialize(path,current_password)
|
14
|
-
@path=path
|
15
|
-
self.password=current_password
|
12
|
+
CIPHER_NAME = 'aes-256-cbc'
|
13
|
+
CONTENT_KEYS = %i[label username password url description].freeze
|
14
|
+
def initialize(path, current_password)
|
15
|
+
@path = path
|
16
|
+
self.password = current_password
|
16
17
|
raise 'path to vault file shall be String' unless @path.is_a?(String)
|
17
|
-
@all_secrets=File.exist?(@path) ? YAML.load_stream(@cipher.decrypt(File.read(@path))).first : {}
|
18
|
+
@all_secrets = File.exist?(@path) ? YAML.load_stream(@cipher.decrypt(File.read(@path))).first : {}
|
18
19
|
end
|
19
20
|
|
20
21
|
def password=(new_password)
|
21
|
-
|
22
|
-
|
23
|
-
key
|
24
|
-
|
25
|
-
|
22
|
+
# number of bits in second position
|
23
|
+
key_bytes = CIPHER_NAME.split('-')[1].to_i / Environment::BITS_PER_BYTE
|
24
|
+
# derive key from passphrase, add trailing zeros
|
25
|
+
key = "#{new_password}#{"\x0" * key_bytes}"[0..(key_bytes - 1)]
|
26
|
+
Log.log.debug{"key=[#{key}],#{key.length}"}
|
27
|
+
SymmetricEncryption.cipher = @cipher = SymmetricEncryption::Cipher.new(cipher_name: CIPHER_NAME, key: key, encoding: :none)
|
26
28
|
end
|
27
29
|
|
28
30
|
def save
|
29
|
-
File.write(@path, @cipher.encrypt(YAML.dump(@all_secrets)),encoding: 'BINARY')
|
31
|
+
File.write(@path, @cipher.encrypt(YAML.dump(@all_secrets)), encoding: 'BINARY')
|
30
32
|
end
|
31
33
|
|
32
34
|
def set(options)
|
33
35
|
raise 'options shall be Hash' unless options.is_a?(Hash)
|
34
|
-
unsupported = options.keys -
|
36
|
+
unsupported = options.keys - CONTENT_KEYS
|
37
|
+
options.each_value {|v| raise 'value must be String' unless v.is_a?(String)}
|
35
38
|
raise "unsupported options: #{unsupported}" unless unsupported.empty?
|
36
39
|
label = options.delete(:label)
|
37
|
-
raise "secret #{label} already exist, delete first" if @all_secrets.
|
40
|
+
raise "secret #{label} already exist, delete first" if @all_secrets.key?(label)
|
38
41
|
@all_secrets[label] = options.symbolize_keys
|
39
42
|
save
|
40
43
|
end
|
41
44
|
|
42
45
|
def list
|
43
46
|
result = []
|
44
|
-
@all_secrets.each do |label,values|
|
47
|
+
@all_secrets.each do |label, values|
|
45
48
|
normal = values.symbolize_keys
|
46
49
|
normal[:label] = label
|
47
|
-
|
50
|
+
CONTENT_KEYS.each{|k|normal[k] = '' unless normal.key?(k)}
|
48
51
|
result.push(normal)
|
49
52
|
end
|
50
53
|
return result
|
51
54
|
end
|
52
55
|
|
53
|
-
def delete(
|
54
|
-
raise 'options shall be Hash' unless options.is_a?(Hash)
|
55
|
-
unsupported = options.keys - %i[label]
|
56
|
-
raise "unsupported options: #{unsupported}" unless unsupported.empty?
|
57
|
-
label=options[:label]
|
56
|
+
def delete(label:)
|
58
57
|
@all_secrets.delete(label)
|
59
58
|
save
|
60
59
|
end
|
61
60
|
|
62
|
-
def get(
|
63
|
-
raise
|
64
|
-
unsupported = options.keys - %i[label]
|
65
|
-
raise "unsupported options: #{unsupported}" unless unsupported.empty?
|
66
|
-
label=options[:label]
|
61
|
+
def get(label:, exception: true)
|
62
|
+
raise "Label not found: #{label}" unless @all_secrets.key?(label) || !exception
|
67
63
|
result = @all_secrets[label].clone
|
68
|
-
|
69
|
-
result[:label]=label
|
64
|
+
result[:label] = label if result.is_a?(Hash)
|
70
65
|
return result
|
71
66
|
end
|
72
67
|
end
|