aspera-cli 4.10.0 → 4.12.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (97) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/BUGS.md +19 -0
  4. data/CHANGELOG.md +528 -0
  5. data/CONTRIBUTING.md +143 -0
  6. data/README.md +977 -589
  7. data/bin/ascli +4 -4
  8. data/bin/asession +12 -12
  9. data/docs/test_env.conf +29 -19
  10. data/examples/aoc.rb +6 -6
  11. data/examples/dascli +18 -16
  12. data/examples/faspex4.rb +15 -15
  13. data/examples/node.rb +12 -12
  14. data/examples/proxy.pac +2 -2
  15. data/examples/server.rb +12 -12
  16. data/lib/aspera/aoc.rb +344 -272
  17. data/lib/aspera/ascmd.rb +56 -54
  18. data/lib/aspera/ats_api.rb +4 -4
  19. data/lib/aspera/cli/basic_auth_plugin.rb +15 -12
  20. data/lib/aspera/cli/extended_value.rb +9 -9
  21. data/lib/aspera/cli/{formater.rb → formatter.rb} +69 -69
  22. data/lib/aspera/cli/listener/line_dump.rb +1 -1
  23. data/lib/aspera/cli/listener/logger.rb +1 -1
  24. data/lib/aspera/cli/listener/progress.rb +5 -6
  25. data/lib/aspera/cli/listener/progress_multi.rb +16 -21
  26. data/lib/aspera/cli/main.rb +72 -73
  27. data/lib/aspera/cli/manager.rb +112 -112
  28. data/lib/aspera/cli/plugin.rb +68 -48
  29. data/lib/aspera/cli/plugins/alee.rb +4 -4
  30. data/lib/aspera/cli/plugins/aoc.rb +322 -720
  31. data/lib/aspera/cli/plugins/ats.rb +50 -52
  32. data/lib/aspera/cli/plugins/bss.rb +10 -10
  33. data/lib/aspera/cli/plugins/config.rb +514 -410
  34. data/lib/aspera/cli/plugins/console.rb +12 -12
  35. data/lib/aspera/cli/plugins/cos.rb +18 -20
  36. data/lib/aspera/cli/plugins/faspex.rb +134 -136
  37. data/lib/aspera/cli/plugins/faspex5.rb +235 -70
  38. data/lib/aspera/cli/plugins/node.rb +378 -309
  39. data/lib/aspera/cli/plugins/orchestrator.rb +52 -49
  40. data/lib/aspera/cli/plugins/preview.rb +129 -120
  41. data/lib/aspera/cli/plugins/server.rb +137 -83
  42. data/lib/aspera/cli/plugins/shares.rb +77 -52
  43. data/lib/aspera/cli/plugins/sync.rb +13 -33
  44. data/lib/aspera/cli/transfer_agent.rb +61 -61
  45. data/lib/aspera/cli/version.rb +2 -1
  46. data/lib/aspera/colors.rb +3 -3
  47. data/lib/aspera/command_line_builder.rb +78 -74
  48. data/lib/aspera/cos_node.rb +31 -29
  49. data/lib/aspera/data_repository.rb +1 -1
  50. data/lib/aspera/environment.rb +30 -28
  51. data/lib/aspera/fasp/agent_base.rb +17 -15
  52. data/lib/aspera/fasp/agent_connect.rb +34 -32
  53. data/lib/aspera/fasp/agent_direct.rb +70 -73
  54. data/lib/aspera/fasp/agent_httpgw.rb +79 -74
  55. data/lib/aspera/fasp/agent_node.rb +26 -26
  56. data/lib/aspera/fasp/agent_trsdk.rb +20 -20
  57. data/lib/aspera/fasp/error.rb +3 -2
  58. data/lib/aspera/fasp/error_info.rb +11 -8
  59. data/lib/aspera/fasp/installation.rb +80 -80
  60. data/lib/aspera/fasp/listener.rb +2 -2
  61. data/lib/aspera/fasp/parameters.rb +103 -92
  62. data/lib/aspera/fasp/parameters.yaml +313 -214
  63. data/lib/aspera/fasp/resume_policy.rb +10 -10
  64. data/lib/aspera/fasp/transfer_spec.rb +22 -2
  65. data/lib/aspera/fasp/uri.rb +7 -7
  66. data/lib/aspera/faspex_gw.rb +80 -159
  67. data/lib/aspera/faspex_postproc.rb +77 -0
  68. data/lib/aspera/hash_ext.rb +3 -3
  69. data/lib/aspera/id_generator.rb +5 -5
  70. data/lib/aspera/keychain/encrypted_hash.rb +23 -28
  71. data/lib/aspera/keychain/macos_security.rb +21 -20
  72. data/lib/aspera/log.rb +13 -13
  73. data/lib/aspera/nagios.rb +24 -23
  74. data/lib/aspera/node.rb +217 -38
  75. data/lib/aspera/oauth.rb +78 -74
  76. data/lib/aspera/open_application.rb +19 -11
  77. data/lib/aspera/persistency_action_once.rb +4 -4
  78. data/lib/aspera/persistency_folder.rb +13 -13
  79. data/lib/aspera/preview/file_types.rb +8 -8
  80. data/lib/aspera/preview/generator.rb +67 -67
  81. data/lib/aspera/preview/utils.rb +27 -27
  82. data/lib/aspera/proxy_auto_config.js +63 -63
  83. data/lib/aspera/proxy_auto_config.rb +19 -19
  84. data/lib/aspera/rest.rb +65 -67
  85. data/lib/aspera/rest_call_error.rb +2 -1
  86. data/lib/aspera/rest_error_analyzer.rb +22 -21
  87. data/lib/aspera/rest_errors_aspera.rb +16 -16
  88. data/lib/aspera/secret_hider.rb +17 -14
  89. data/lib/aspera/ssh.rb +15 -14
  90. data/lib/aspera/sync.rb +177 -62
  91. data/lib/aspera/temp_file_manager.rb +2 -2
  92. data/lib/aspera/uri_reader.rb +4 -4
  93. data/lib/aspera/web_auth.rb +13 -64
  94. data/lib/aspera/web_server_simple.rb +76 -0
  95. data.tar.gz.sig +0 -0
  96. metadata +11 -6
  97. metadata.gz.sig +0 -0
@@ -20,36 +20,36 @@ module Aspera
20
20
  @parameters = DEFAULTS.dup
21
21
  if !params.nil?
22
22
  raise "expecting Hash (or nil), but have #{params.class}" unless params.is_a?(Hash)
23
- params.each do |k,v|
24
- raise "unknown resume parameter: #{k}, expect one of #{DEFAULTS.keys.map(&:to_s).join(',')}" unless DEFAULTS.has_key?(k)
23
+ params.each do |k, v|
24
+ raise "unknown resume parameter: #{k}, expect one of #{DEFAULTS.keys.map(&:to_s).join(',')}" unless DEFAULTS.key?(k)
25
25
  raise "#{k} must be Integer" unless v.is_a?(Integer)
26
26
  @parameters[k] = v
27
27
  end
28
28
  end
29
- Log.log.debug("resume params=#{@parameters}")
29
+ Log.log.debug{"resume params=#{@parameters}"}
30
30
  end
31
31
 
32
32
  # calls block a number of times (resumes) until success or limit reached
33
33
  # this is re-entrant, one resumer can handle multiple transfers in //
34
34
  def execute_with_resume
35
- raise 'block manndatory' unless block_given?
35
+ raise 'block mandatory' unless block_given?
36
36
  # maximum of retry
37
37
  remaining_resumes = @parameters[:iter_max]
38
38
  sleep_seconds = @parameters[:sleep_initial]
39
- Log.log.debug("retries=#{remaining_resumes}")
40
- # try to send the file until ascp is succesful
39
+ Log.log.debug{"retries=#{remaining_resumes}"}
40
+ # try to send the file until ascp is successful
41
41
  loop do
42
- Log.log.debug('transfer starting');
42
+ Log.log.debug('transfer starting')
43
43
  begin
44
44
  # call provided block
45
45
  yield
46
46
  break
47
47
  rescue Fasp::Error => e
48
- Log.log.warn("An error occurred: #{e.message}");
48
+ Log.log.warn{"An error occurred: #{e.message}"}
49
49
  # failure in ascp
50
50
  if e.retryable?
51
51
  # exit if we exceed the max number of retry
52
- raise Fasp::Error,'Maximum number of retry reached' if remaining_resumes <= 0
52
+ raise Fasp::Error, 'Maximum number of retry reached' if remaining_resumes <= 0
53
53
  else
54
54
  # give one chance only to non retryable errors
55
55
  unless remaining_resumes.eql?(@parameters[:iter_max])
@@ -61,7 +61,7 @@ module Aspera
61
61
 
62
62
  # take this retry in account
63
63
  remaining_resumes -= 1
64
- Log.log.warn("resuming in #{sleep_seconds} seconds (retry left:#{remaining_resumes})");
64
+ Log.log.warn{"resuming in #{sleep_seconds} seconds (retry left:#{remaining_resumes})"}
65
65
 
66
66
  # wait a bit before retrying, maybe network condition will be better
67
67
  sleep(sleep_seconds)
@@ -16,10 +16,30 @@ module Aspera
16
16
  'fasp_port' => UDP_PORT
17
17
  }.freeze
18
18
  # define constants for enums of parameters: <paramater>_<enum>, e.g. CIPHER_AES_128
19
- Aspera::Fasp::Parameters.description.each do |k,v|
19
+ Aspera::Fasp::Parameters.description.each do |k, v|
20
20
  next unless v[:enum].is_a?(Array)
21
21
  v[:enum].each do |enum|
22
- TransferSpec.const_set("#{k.to_s.upcase}_#{enum.upcase.gsub(/[^A-Z0-9]/,'_')}", enum.freeze)
22
+ TransferSpec.const_set("#{k.to_s.upcase}_#{enum.upcase.gsub(/[^A-Z0-9]/, '_')}", enum.freeze)
23
+ end
24
+ end
25
+ class << self
26
+ def action_to_direction(tspec, command)
27
+ raise 'transfer spec must be a Hash' unless tspec.is_a?(Hash)
28
+ tspec['direction'] = case command.to_sym
29
+ when :upload then DIRECTION_SEND
30
+ when :download then DIRECTION_RECEIVE
31
+ else raise 'Error: upload or download only'
32
+ end
33
+ return tspec
34
+ end
35
+
36
+ def action(tspec)
37
+ raise 'transfer spec must be a Hash' unless tspec.is_a?(Hash)
38
+ return case tspec['direction']
39
+ when DIRECTION_SEND then :upload
40
+ when DIRECTION_RECEIVE then :download
41
+ else raise 'Error: upload or download only'
42
+ end
23
43
  end
24
44
  end
25
45
  end
@@ -5,10 +5,10 @@ require 'aspera/command_line_builder'
5
5
 
6
6
  module Aspera
7
7
  module Fasp
8
- # translates a "faspe:" URI (used in Faspex) into transfer spec hash
8
+ # translates a "faspe:" URI (used in Faspex 4) into transfer spec hash
9
9
  class Uri
10
10
  def initialize(fasplink)
11
- @fasp_uri = URI.parse(fasplink.gsub(' ','%20'))
11
+ @fasp_uri = URI.parse(fasplink.gsub(' ', '%20'))
12
12
  # TODO: check scheme is faspe
13
13
  end
14
14
 
@@ -34,16 +34,16 @@ module Aspera
34
34
  when 'minrate' then result_ts['min_rate_kbps'] = value.to_i
35
35
  when 'port' then result_ts['fasp_port'] = value.to_i
36
36
  when 'bwcap' then result_ts['target_rate_cap_kbps'] = value.to_i
37
- when 'enc' then result_ts['cipher'] = value.gsub(/^aes/,'aes-').gsub(/cfb$/,'-cfb').gsub(/gcm$/,'-gcm').gsub(/--/,'-')
37
+ when 'enc' then result_ts['cipher'] = value.gsub(/^aes/, 'aes-').gsub(/cfb$/, '-cfb').gsub(/gcm$/, '-gcm').gsub(/--/, '-')
38
38
  when 'tags64' then result_ts['tags'] = JSON.parse(Base64.strict_decode64(value))
39
39
  when 'createpath' then result_ts['create_dir'] = CommandLineBuilder.yes_to_true(value)
40
40
  when 'fallback' then result_ts['http_fallback'] = CommandLineBuilder.yes_to_true(value)
41
41
  when 'lockpolicy' then result_ts['lock_rate_policy'] = CommandLineBuilder.yes_to_true(value)
42
42
  when 'lockminrate' then result_ts['lock_min_rate'] = CommandLineBuilder.yes_to_true(value)
43
- when 'auth' then Log.log.debug("ignoring auth #{name}=#{value}") # TODO: translate into transfer spec ? yes/no
44
- when 'v' then Log.log.debug("ignoring v #{name}=#{value}") # TODO: translate into transfer spec ? 2
45
- when 'protect' then Log.log.debug("ignoring protect #{name}=#{value}") # TODO: translate into transfer spec ?
46
- else Log.log.warn("URI parameter ignored: #{name} = #{value}")
43
+ when 'auth' then Log.log.debug{"ignoring auth #{name}=#{value}"} # TODO: translate into transfer spec ? yes/no
44
+ when 'v' then Log.log.debug{"ignoring v #{name}=#{value}"} # TODO: translate into transfer spec ? 2
45
+ when 'protect' then Log.log.debug{"ignoring protect #{name}=#{value}"} # TODO: translate into transfer spec ?
46
+ else Log.log.warn{"URI parameter ignored: #{name} = #{value}"}
47
47
  end
48
48
  end
49
49
  return result_ts
@@ -1,174 +1,95 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require 'aspera/web_server_simple'
3
4
  require 'aspera/log'
4
- require 'aspera/aoc'
5
- require 'aspera/fasp/transfer_spec'
6
- require 'aspera/cli/main'
7
- require 'webrick'
8
- require 'webrick/https'
9
- require 'securerandom'
10
- require 'openssl'
11
5
  require 'json'
12
6
 
13
7
  module Aspera
14
8
  # this class answers the Faspex /send API and creates a package on Aspera on Cloud
15
- class FaspexGW
16
- class FxGwServlet < WEBrick::HTTPServlet::AbstractServlet
17
- def initialize(_server, a_aoc_api_user, a_workspace_id)
18
- super
19
- @aoc_api_user = a_aoc_api_user
20
- @aoc_workspace_id = a_workspace_id
21
- end
22
-
23
- # parameters from user to Faspex API call
24
- # {"delivery":{"use_encryption_at_rest":false,"note":"note","sources":[{"paths":["file1"]}],"title":"my title","recipients":["email1"],"send_upload_result":true}}
25
- # {
26
- # "delivery"=>{
27
- # "use_encryption_at_rest"=>false,
28
- # "note"=>"note",
29
- # "sources"=>[{"paths"=>["file1"]}],
30
- # "title"=>"my title",
31
- # "recipients"=>["email1"],
32
- # "send_upload_result"=>true
33
- # }
34
- # }
35
- def process_faspex_send(request, response)
36
- raise 'no payload' if request.body.nil?
37
-
38
- faspex_pkg_parameters = JSON.parse(request.body)
39
- faspex_pkg_delivery = faspex_pkg_parameters['delivery']
40
- Log.log.debug("faspex pkg create parameters=#{faspex_pkg_parameters}")
41
-
42
- # get recipient ids
43
- files_pkg_recipients = []
44
- faspex_pkg_delivery['recipients'].each do |recipient_email|
45
- user_lookup = @aoc_api_user.read('contacts',
46
- { 'current_workspace_id' => @aoc_workspace_id, 'q' => recipient_email })[:data]
47
- raise StandardError,
48
- "no such unique user: #{recipient_email} / #{user_lookup}" unless !user_lookup.nil? && user_lookup.length.eql?(1)
49
-
50
- recipient_user_info = user_lookup.first
51
- files_pkg_recipients.push({
52
- 'id' => recipient_user_info['source_id'],
53
- 'type' => recipient_user_info['source_type']
54
- })
55
- end
56
-
57
- # create a new package with one file
58
- the_package = @aoc_api_user.create('packages', {
59
- 'file_names' => faspex_pkg_delivery['sources'][0]['paths'],
60
- 'name' => faspex_pkg_delivery['title'],
61
- 'note' => faspex_pkg_delivery['note'],
62
- 'recipients' => files_pkg_recipients,
63
- 'workspace_id' => @aoc_workspace_id
64
- })[:data]
65
-
66
- # get node information for the node on which package must be created
67
- node_info = @aoc_api_user.read("nodes/#{the_package['node_id']}")[:data]
68
-
69
- # get transfer token (for node)
70
- node_auth_bearer_token = @aoc_api_user.oauth_token(scope: AoC.node_scope(node_info['access_key'],
71
- AoC::SCOPE_NODE_USER))
72
-
73
- # tell Files what to expect in package: 1 transfer (can also be done after transfer)
74
- @aoc_api_user.update("packages/#{the_package['id']}", { 'sent' => true, 'transfers_expected' => 1 })
75
-
76
- # to return an error:
77
- # response.status=400
78
- # return 'ERROR HERE'
79
-
80
- # TODO: check about xfer_*
81
- ts_tags = {
82
- 'aspera' => {
83
- 'files' => { 'package_id' => the_package['id'], 'package_operation' => 'upload' },
84
- 'node' => { 'access_key' => node_info['access_key'], 'file_id' => the_package['contents_file_id'] },
85
- 'xfer_id' => SecureRandom.uuid,
86
- 'xfer_retry' => 3600
87
- }
88
- }
89
- # this transfer spec is for transfer to AoC
90
- faspex_transfer_spec = {
91
- 'direction' => 'send',
92
- 'remote_host' => node_info['host'],
93
- 'remote_user' => Fasp::TransferSpec::ACCESS_KEY_TRANSFER_USER,
94
- 'ssh_port' => Fasp::TransferSpec::SSH_PORT,
95
- 'fasp_port' => Fasp::TransferSpec::UDP_PORT,
96
- 'tags' => ts_tags,
97
- 'token' => node_auth_bearer_token,
98
- 'paths' => [{ 'destination' => '/' }],
99
- 'cookie' => 'unused',
100
- 'create_dir' => true,
101
- 'rate_policy' => 'fair',
102
- 'rate_policy_allowed' => 'fixed',
103
- 'min_rate_cap_kbps' => nil,
104
- 'min_rate_kbps' => 0,
105
- 'target_rate_percentage' => nil,
106
- 'lock_target_rate' => nil,
107
- 'fasp_url' => 'unused',
108
- 'lock_min_rate' => true,
109
- 'lock_rate_policy' => true,
110
- 'source_root' => '',
111
- 'content_protection' => nil,
112
- 'target_rate_cap_kbps' => 20_000, # TODO: is this value useful ?
113
- 'target_rate_kbps' => 10_000, # TODO: get from where?
114
- 'cipher' => 'aes-128',
115
- 'cipher_allowed' => nil,
116
- 'http_fallback' => false,
117
- 'http_fallback_port' => nil,
118
- 'https_fallback_port' => nil,
119
- 'destination_root' => '/'
120
- }
121
- # but we place it in a Faspex package creation response
122
- faspex_package_create_result = {
123
- 'links' => { 'status' => 'unused' },
124
- 'xfer_sessions' => [faspex_transfer_spec]
125
- }
126
- Log.log.info("faspex_package_create_result=#{faspex_package_create_result}")
127
- response.status = 200
128
- response.content_type = 'application/json'
129
- response.body = JSON.generate(faspex_package_create_result)
130
- end
131
-
132
- def do_GET(request, response)
133
- case request.path
134
- when '/aspera/faspex/send'
135
- process_faspex_send(request, response)
136
- else
137
- response.status = 400
138
- 'ERROR HERE'
139
- end
140
- end
141
- end # FxGwServlet
9
+ class Faspex4GWServlet < WEBrick::HTTPServlet::AbstractServlet
10
+ # @param app_api [Aspera::AoC]
11
+ # @param app_context [String]
12
+ def initialize(server, app_api, app_context)
13
+ super(server)
14
+ # typed: Aspera::AoC
15
+ @app_api = app_api
16
+ @app_context = app_context
17
+ end
142
18
 
143
- class NewUserServlet < WEBrick::HTTPServlet::AbstractServlet
144
- def do_GET(request, response)
145
- case request.path
146
- when '/newuser'
147
- response.status = 200
148
- response.content_type = 'text/html'
149
- response.body = '<html><body>hello world</body></html>'
150
- else
151
- raise "unsupported path: [#{request.path}]"
152
- end
153
- end
19
+ # Map Faspex 4 /send API to AoC package create
20
+ # parameters from user to Faspex API call
21
+ # https://developer.ibm.com/apis/catalog/aspera--aspera-faspex-client-sdk/Sending%20Packages%20(API%20v.3)
22
+ def faspex4_send_to_aoc(faspex_pkg_parameters)
23
+ faspex_pkg_delivery = faspex_pkg_parameters['delivery']
24
+ package_data = {
25
+ # 'file_names' => faspex_pkg_delivery['sources'][0]['paths'],
26
+ 'name' => faspex_pkg_delivery['title'],
27
+ 'note' => faspex_pkg_delivery['note'],
28
+ 'recipients' => faspex_pkg_delivery['recipients'],
29
+ 'workspace_id' => @app_context
30
+ }
31
+ created_package = @app_api.create_package_simple(package_data, true, @new_user_option)
32
+ # but we place it in a Faspex package creation response
33
+ return {
34
+ 'links' => { 'status' => 'unused' },
35
+ 'xfer_sessions' => [created_package[:spec]]
36
+ }
154
37
  end
155
38
 
156
- def initialize(a_aoc_api_user, a_workspace_id)
157
- webrick_options = {
158
- Port: 9443,
159
- Logger: Log.log,
160
- SSLEnable: true,
161
- SSLCertName: [['CN', WEBrick::Utils.getservername]]
39
+ def faspex4_send_to_faspex5(faspex_pkg_parameters)
40
+ faspex_pkg_delivery = faspex_pkg_parameters['delivery']
41
+ package_data = {
42
+ 'title' => faspex_pkg_delivery['title'],
43
+ 'note' => faspex_pkg_delivery['note'],
44
+ 'recipients' => faspex_pkg_delivery['recipients'].map{|name|{'name'=>name}}
45
+ }
46
+ package = @app_api.create('packages', package_data)[:data]
47
+ # TODO: option to send from remote source or httpgw
48
+ transfer_spec = @app_api.call(
49
+ operation: 'POST',
50
+ subpath: "packages/#{package['id']}/transfer_spec/upload",
51
+ headers: {'Accept' => 'application/json'},
52
+ url_params: {transfer_type: Aspera::Cli::Plugins::Faspex5::TRANSFER_CONNECT},
53
+ json_params: {paths: [{'destination'=>'/'}]}
54
+ )[:data]
55
+ transfer_spec.delete('authentication')
56
+ # but we place it in a Faspex package creation response
57
+ return {
58
+ 'links' => { 'status' => 'unused' },
59
+ 'xfer_sessions' => [transfer_spec]
162
60
  }
163
- Log.log.info("Server started on port #{webrick_options[:Port]}")
164
- @server = WEBrick::HTTPServer.new(webrick_options)
165
- @server.mount('/aspera/faspex', FxGwServlet, a_aoc_api_user, a_workspace_id)
166
- @server.mount('/newuser', NewUserServlet)
167
- trap('INT') { @server.shutdown }
168
61
  end
169
62
 
170
- def start_server
171
- @server.start
63
+ def do_POST(request, response)
64
+ case request.path
65
+ when '/aspera/faspex/send'
66
+ begin
67
+ raise 'no payload' if request.body.nil?
68
+ faspex_pkg_parameters = JSON.parse(request.body)
69
+ Log.log.debug{"faspex pkg create parameters=#{faspex_pkg_parameters}"}
70
+ faspex_package_create_result =
71
+ if @app_api.is_a?(Aspera::AoC)
72
+ faspex4_send_to_aoc(faspex_pkg_parameters)
73
+ elsif @app_api.is_a?(Aspera::Rest)
74
+ faspex4_send_to_faspex5(faspex_pkg_parameters)
75
+ else
76
+ raise "No such adapter: #{@app_api.class}"
77
+ end
78
+ Log.log.info{"faspex_package_create_result=#{faspex_package_create_result}"}
79
+ response.status = 200
80
+ response.content_type = 'application/json'
81
+ response.body = JSON.generate(faspex_package_create_result)
82
+ rescue => e
83
+ response.status = 500
84
+ response['Content-Type'] = 'application/json'
85
+ response.body = {error: e.message}.to_json
86
+ Log.log.error(e.message)
87
+ end
88
+ else
89
+ response.status = 400
90
+ response['Content-Type'] = 'application/json'
91
+ response.body = {error: 'Bad request'}.to_json
92
+ end
172
93
  end
173
- end # FaspexGW
94
+ end # Faspex4GWServlet
174
95
  end # AsperaLm
@@ -0,0 +1,77 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'English'
4
+ require 'aspera/web_server_simple'
5
+ require 'aspera/log'
6
+ require 'json'
7
+ require 'timeout'
8
+
9
+ module Aspera
10
+ # this class answers the Faspex /send API and creates a package on Aspera on Cloud
11
+ class Faspex4PostProcServlet < WEBrick::HTTPServlet::AbstractServlet
12
+ ALLOWED_PARAMETERS = %i[root script_folder fail_on_error timeout_seconds].freeze
13
+ def initialize(server, parameters)
14
+ raise 'parameters must be Hash' unless parameters.is_a?(Hash)
15
+ @parameters = parameters.symbolize_keys
16
+ Log.dump(:postproc_parameters, @parameters)
17
+ raise "unexpected key in parameters config: only: #{ALLOWED_PARAMETERS.join(', ')}" if @parameters.keys.any?{|k|!ALLOWED_PARAMETERS.include?(k)}
18
+ @parameters[:script_folder] ||= '.'
19
+ @parameters[:fail_on_error] ||= false
20
+ @parameters[:timeout_seconds] ||= 60
21
+ super(server)
22
+ Log.log.debug{"Faspex4PostProcServlet initialized"}
23
+ end
24
+
25
+ def do_POST(request, response)
26
+ Log.log.debug{"request=#{request.path}"}
27
+ begin
28
+ # only accept requests on the root
29
+ if !request.path.start_with?(@parameters[:root])
30
+ response.status = 400
31
+ response['Content-Type'] = 'application/json'
32
+ response.body = {status: 'error', message: 'Request outside domain'}.to_json
33
+ return
34
+ end
35
+ if request.body.nil?
36
+ response.status = 400
37
+ response['Content-Type'] = 'application/json'
38
+ response.body = {status: 'error', message: 'Empty request'}.to_json
39
+ return
40
+ end
41
+ # build script path by removing domain, and adding script folder
42
+ script_file = request.path[@parameters[:root].size .. ]
43
+ Log.log.debug{"script file=#{script_file}"}
44
+ script_path = File.join(@parameters[:script_folder], script_file)
45
+ Log.log.debug{"script=#{script_path}"}
46
+ webhook_parameters = JSON.parse(request.body)
47
+ Log.dump(:webhook_parameters, webhook_parameters)
48
+ # env expects only strings
49
+ environment = webhook_parameters.each_with_object({}) { |(k, v), h| h[k] = v.to_s }
50
+ post_proc_pid = Process.spawn(environment, [script_path, script_path])
51
+ Log.log.debug{"pid=#{post_proc_pid}"}
52
+ raise 'no pid' if post_proc_pid.nil?
53
+ # "wait" for process to avoid zombie
54
+ Timeout.timeout(@parameters[:timeout_seconds]) do
55
+ Process.wait(post_proc_pid)
56
+ post_proc_pid = nil
57
+ end
58
+ process_status = $CHILD_STATUS
59
+ raise "script #{script_path} failed with code #{process_status.exitstatus}" if !process_status.success? && @parameters[:fail_on_error]
60
+ response.status = 200
61
+ response.content_type = 'application/json'
62
+ response.body = JSON.generate({status: 'success', script: script_path, exit_code: process_status.exitstatus})
63
+ Log.log.debug{'Script executed successfully'}
64
+ rescue => e
65
+ Log.log.error("Script failed: #{e.class}:#{e.message}")
66
+ if !post_proc_pid.nil?
67
+ Process.kill('SIGKILL', post_proc_pid)
68
+ Process.wait(post_proc_pid)
69
+ Log.log.error("Killed process: #{post_proc_pid}")
70
+ end
71
+ response.status = 500
72
+ response['Content-Type'] = 'application/json'
73
+ response.body = {status: 'error', script: script_path, message: e.message}.to_json
74
+ end
75
+ end
76
+ end # Faspex4PostProcServlet
77
+ end # AsperaLm
@@ -2,11 +2,11 @@
2
2
 
3
3
  class ::Hash
4
4
  def deep_merge(second)
5
- merge(second){|_key,v1,v2|Hash === v1 && Hash === v2 ? v1.deep_merge(v2) : v2}
5
+ merge(second){|_key, v1, v2|Hash === v1 && Hash === v2 ? v1.deep_merge(v2) : v2}
6
6
  end
7
7
 
8
8
  def deep_merge!(second)
9
- merge!(second){|_key,v1,v2|Hash === v1 && Hash === v2 ? v1.deep_merge!(v2) : v2}
9
+ merge!(second){|_key, v1, v2|Hash === v1 && Hash === v2 ? v1.deep_merge!(v2) : v2}
10
10
  end
11
11
  end
12
12
 
@@ -14,7 +14,7 @@ end
14
14
  unless Hash.method_defined?(:transform_keys)
15
15
  class Hash
16
16
  def transform_keys
17
- return each_with_object({}){|(k,v),memo|memo[yield(k)]=v} if block_given?
17
+ return each_with_object({}){|(k, v), memo|memo[yield(k)] = v} if block_given?
18
18
  raise 'missing block'
19
19
  end
20
20
  end
@@ -7,7 +7,7 @@ module Aspera
7
7
  ID_SEPARATOR = '_'
8
8
  WINDOWS_PROTECTED_CHAR = %r{[/:"<>\\*?]}.freeze
9
9
  PROTECTED_CHAR_REPLACE = '_'
10
- private_constant :ID_SEPARATOR,:PROTECTED_CHAR_REPLACE,:WINDOWS_PROTECTED_CHAR
10
+ private_constant :ID_SEPARATOR, :PROTECTED_CHAR_REPLACE, :WINDOWS_PROTECTED_CHAR
11
11
  class << self
12
12
  def from_list(object_id)
13
13
  if object_id.is_a?(Array)
@@ -16,10 +16,10 @@ module Aspera
16
16
  end.join(ID_SEPARATOR)
17
17
  end
18
18
  raise 'id must be a String' unless object_id.is_a?(String)
19
- return object_id.
20
- gsub(WINDOWS_PROTECTED_CHAR,PROTECTED_CHAR_REPLACE). # remove windows forbidden chars
21
- gsub('.',PROTECTED_CHAR_REPLACE). # keep dot for extension only (nicer)
22
- downcase
19
+ return object_id
20
+ .gsub(WINDOWS_PROTECTED_CHAR, PROTECTED_CHAR_REPLACE) # remove windows forbidden chars
21
+ .gsub('.', PROTECTED_CHAR_REPLACE) # keep dot for extension only (nicer)
22
+ .downcase
23
23
  end
24
24
  end
25
25
  end
@@ -1,6 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require 'aspera/hash_ext'
4
+ require 'aspera/environment'
4
5
  require 'symmetric_encryption/core'
5
6
  require 'yaml'
6
7
 
@@ -8,65 +9,59 @@ module Aspera
8
9
  module Keychain
9
10
  # Manage secrets in a simple Hash
10
11
  class EncryptedHash
11
- CIPHER_NAME='aes-256-cbc'
12
- ACCEPTED_KEYS = %i[label username password url description].freeze
13
- def initialize(path,current_password)
14
- @path=path
15
- self.password=current_password
12
+ CIPHER_NAME = 'aes-256-cbc'
13
+ CONTENT_KEYS = %i[label username password url description].freeze
14
+ def initialize(path, current_password)
15
+ @path = path
16
+ self.password = current_password
16
17
  raise 'path to vault file shall be String' unless @path.is_a?(String)
17
- @all_secrets=File.exist?(@path) ? YAML.load_stream(@cipher.decrypt(File.read(@path))).first : {}
18
+ @all_secrets = File.exist?(@path) ? YAML.load_stream(@cipher.decrypt(File.read(@path))).first : {}
18
19
  end
19
20
 
20
21
  def password=(new_password)
21
- key_bytes=CIPHER_NAME.split('-')[1].to_i/8
22
- # derive key from passphrase
23
- key="#{new_password}#{"\x0"*key_bytes}"[0..(key_bytes-1)]
24
- Log.log.debug("key=[#{key}],#{key.length}")
25
- SymmetricEncryption.cipher=@cipher = SymmetricEncryption::Cipher.new(cipher_name: CIPHER_NAME,key: key,encoding: :none)
22
+ # number of bits in second position
23
+ key_bytes = CIPHER_NAME.split('-')[1].to_i / Environment::BITS_PER_BYTE
24
+ # derive key from passphrase, add trailing zeros
25
+ key = "#{new_password}#{"\x0" * key_bytes}"[0..(key_bytes - 1)]
26
+ Log.log.debug{"key=[#{key}],#{key.length}"}
27
+ SymmetricEncryption.cipher = @cipher = SymmetricEncryption::Cipher.new(cipher_name: CIPHER_NAME, key: key, encoding: :none)
26
28
  end
27
29
 
28
30
  def save
29
- File.write(@path, @cipher.encrypt(YAML.dump(@all_secrets)),encoding: 'BINARY')
31
+ File.write(@path, @cipher.encrypt(YAML.dump(@all_secrets)), encoding: 'BINARY')
30
32
  end
31
33
 
32
34
  def set(options)
33
35
  raise 'options shall be Hash' unless options.is_a?(Hash)
34
- unsupported = options.keys - ACCEPTED_KEYS
36
+ unsupported = options.keys - CONTENT_KEYS
37
+ options.each_value {|v| raise 'value must be String' unless v.is_a?(String)}
35
38
  raise "unsupported options: #{unsupported}" unless unsupported.empty?
36
39
  label = options.delete(:label)
37
- raise "secret #{label} already exist, delete first" if @all_secrets.has_key?(label)
40
+ raise "secret #{label} already exist, delete first" if @all_secrets.key?(label)
38
41
  @all_secrets[label] = options.symbolize_keys
39
42
  save
40
43
  end
41
44
 
42
45
  def list
43
46
  result = []
44
- @all_secrets.each do |label,values|
47
+ @all_secrets.each do |label, values|
45
48
  normal = values.symbolize_keys
46
49
  normal[:label] = label
47
- ACCEPTED_KEYS.each{|k|normal[k] = '' unless normal.has_key?(k)}
50
+ CONTENT_KEYS.each{|k|normal[k] = '' unless normal.key?(k)}
48
51
  result.push(normal)
49
52
  end
50
53
  return result
51
54
  end
52
55
 
53
- def delete(options)
54
- raise 'options shall be Hash' unless options.is_a?(Hash)
55
- unsupported = options.keys - %i[label]
56
- raise "unsupported options: #{unsupported}" unless unsupported.empty?
57
- label=options[:label]
56
+ def delete(label:)
58
57
  @all_secrets.delete(label)
59
58
  save
60
59
  end
61
60
 
62
- def get(options)
63
- raise 'options shall be Hash' unless options.is_a?(Hash)
64
- unsupported = options.keys - %i[label]
65
- raise "unsupported options: #{unsupported}" unless unsupported.empty?
66
- label=options[:label]
61
+ def get(label:, exception: true)
62
+ raise "Label not found: #{label}" unless @all_secrets.key?(label) || !exception
67
63
  result = @all_secrets[label].clone
68
- raise "no such entry #{label}" if result.nil?
69
- result[:label]=label
64
+ result[:label] = label if result.is_a?(Hash)
70
65
  return result
71
66
  end
72
67
  end