aspera-cli 4.27.2 → 4.27.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (145) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +133 -0
  4. data/CONTRIBUTING.md +5 -2
  5. data/bin/ascli +3 -1
  6. data/docs/README.md +1006 -766
  7. data/lib/aspera/agent/base.rb +7 -2
  8. data/lib/aspera/agent/connect.rb +6 -8
  9. data/lib/aspera/agent/desktop.rb +2 -6
  10. data/lib/aspera/agent/direct.rb +52 -22
  11. data/lib/aspera/agent/node.rb +9 -8
  12. data/lib/aspera/agent/transferd.rb +2 -2
  13. data/lib/aspera/api/alee.rb +1 -1
  14. data/lib/aspera/api/aoc.rb +14 -12
  15. data/lib/aspera/api/ats.rb +1 -1
  16. data/lib/aspera/api/cos_node.rb +2 -2
  17. data/lib/aspera/api/faspex.rb +11 -7
  18. data/lib/aspera/api/httpgw.rb +38 -35
  19. data/lib/aspera/api/node.rb +39 -34
  20. data/lib/aspera/ascmd.rb +3 -1
  21. data/lib/aspera/ascp/installation.rb +63 -28
  22. data/lib/aspera/ascp/management.rb +1 -0
  23. data/lib/aspera/assert.rb +4 -0
  24. data/lib/aspera/cli/ascp_actions.rb +20 -41
  25. data/lib/aspera/cli/async_transfer_store.rb +12 -11
  26. data/lib/aspera/cli/bootstrapper.rb +14 -16
  27. data/lib/aspera/cli/command_line.rb +252 -0
  28. data/lib/aspera/cli/command_registry.rb +215 -37
  29. data/lib/aspera/cli/command_spec.rb +104 -15
  30. data/lib/aspera/cli/completion/ascli.bash +12 -0
  31. data/lib/aspera/cli/completion/ascli.fish +16 -0
  32. data/lib/aspera/cli/completion/ascli.zsh +19 -0
  33. data/lib/aspera/cli/context.rb +3 -0
  34. data/lib/aspera/cli/deprecation.rb +37 -0
  35. data/lib/aspera/cli/extended_value.rb +6 -3
  36. data/lib/aspera/cli/formatter.rb +94 -80
  37. data/lib/aspera/cli/gem_checker.rb +1 -1
  38. data/lib/aspera/cli/hints.rb +7 -6
  39. data/lib/aspera/cli/http.rb +22 -34
  40. data/lib/aspera/cli/info.rb +3 -0
  41. data/lib/aspera/cli/mcp_tool.rb +47 -83
  42. data/lib/aspera/cli/option_declarator.rb +33 -42
  43. data/lib/aspera/cli/option_registry.rb +69 -0
  44. data/lib/aspera/cli/option_types.rb +105 -0
  45. data/lib/aspera/cli/option_value.rb +278 -0
  46. data/lib/aspera/cli/options.schema.yaml +124 -15
  47. data/lib/aspera/cli/parser.rb +333 -862
  48. data/lib/aspera/cli/plugins/alee.rb +7 -4
  49. data/lib/aspera/cli/plugins/aoc.rb +545 -518
  50. data/lib/aspera/cli/plugins/ats.rb +59 -80
  51. data/lib/aspera/cli/plugins/base.rb +221 -265
  52. data/lib/aspera/cli/plugins/basic_auth.rb +2 -10
  53. data/lib/aspera/cli/plugins/config.rb +263 -184
  54. data/lib/aspera/cli/plugins/console.rb +103 -39
  55. data/lib/aspera/cli/plugins/cos.rb +6 -23
  56. data/lib/aspera/cli/plugins/factory.rb +3 -0
  57. data/lib/aspera/cli/plugins/faspex5.rb +204 -182
  58. data/lib/aspera/cli/plugins/faspio.rb +6 -11
  59. data/lib/aspera/cli/plugins/httpgw.rb +8 -11
  60. data/lib/aspera/cli/plugins/mcp.rb +20 -55
  61. data/lib/aspera/cli/plugins/node.rb +300 -327
  62. data/lib/aspera/cli/plugins/orchestrator.rb +152 -110
  63. data/lib/aspera/cli/plugins/preview.rb +96 -105
  64. data/lib/aspera/cli/plugins/server.rb +78 -53
  65. data/lib/aspera/cli/plugins/shares.rb +80 -131
  66. data/lib/aspera/cli/preset_actions.rb +44 -27
  67. data/lib/aspera/cli/preset_manager.rb +44 -19
  68. data/lib/aspera/cli/prompt.rb +36 -0
  69. data/lib/aspera/cli/result.rb +42 -36
  70. data/lib/aspera/cli/runner.rb +32 -59
  71. data/lib/aspera/cli/special_values.rb +5 -0
  72. data/lib/aspera/cli/sync_actions.rb +51 -46
  73. data/lib/aspera/cli/terminal_formatter.rb +9 -3
  74. data/lib/aspera/cli/transfer_actions.rb +14 -9
  75. data/lib/aspera/cli/transfer_agent.rb +34 -38
  76. data/lib/aspera/cli/transfer_progress.rb +290 -55
  77. data/lib/aspera/cli/vault_manager.rb +0 -17
  78. data/lib/aspera/cli/version.rb +1 -1
  79. data/lib/aspera/cli/wizard.rb +5 -3
  80. data/lib/aspera/coverage.rb +1 -1
  81. data/lib/aspera/environment.rb +35 -5
  82. data/lib/aspera/faspex_gw.rb +2 -1
  83. data/lib/aspera/faspex_postproc.rb +1 -0
  84. data/lib/aspera/graphql.rb +5 -5
  85. data/lib/aspera/json_rpc/client.rb +5 -5
  86. data/lib/aspera/keychain/encrypted_hash.rb +2 -2
  87. data/lib/aspera/keychain/factory.rb +2 -1
  88. data/lib/aspera/keychain/one_password_api.rb +1 -1
  89. data/lib/aspera/link_header.rb +2 -2
  90. data/lib/aspera/log.rb +47 -27
  91. data/lib/aspera/markdown.rb +2 -0
  92. data/lib/aspera/mime.rb +25 -0
  93. data/lib/aspera/node_emulator.rb +759 -0
  94. data/lib/aspera/oauth/base.rb +37 -26
  95. data/lib/aspera/oauth/factory.rb +7 -3
  96. data/lib/aspera/oauth/generic.rb +1 -1
  97. data/lib/aspera/oauth/json_credentials.rb +34 -0
  98. data/lib/aspera/oauth/jwt.rb +4 -5
  99. data/lib/aspera/oauth/web.rb +9 -8
  100. data/lib/aspera/oauth.rb +1 -0
  101. data/lib/aspera/persistency_folder.rb +1 -3
  102. data/lib/aspera/preview/file_types.rb +4 -4
  103. data/lib/aspera/preview/generator.rb +11 -1
  104. data/lib/aspera/preview/options.schema.yaml +119 -0
  105. data/lib/aspera/preview/terminal.rb +4 -3
  106. data/lib/aspera/preview/utils.rb +9 -6
  107. data/lib/aspera/products/connect.rb +1 -1
  108. data/lib/aspera/rainbow.rb +7 -0
  109. data/lib/aspera/rest/aspera_errors.rb +72 -0
  110. data/lib/aspera/rest/call_error.rb +27 -0
  111. data/lib/aspera/rest/client.rb +523 -0
  112. data/lib/aspera/rest/error_analyzer.rb +113 -0
  113. data/lib/aspera/rest/list.rb +149 -0
  114. data/lib/aspera/rest/parameters.rb +55 -0
  115. data/lib/aspera/rest/util.rb +176 -0
  116. data/lib/aspera/rest.rb +7 -621
  117. data/lib/aspera/schema/IBM Aspera Console-enhanced.yaml +1125 -0
  118. data/lib/aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml +0 -20
  119. data/lib/aspera/schema/IBM Aspera Orchestrator API-v1.yaml +1784 -0
  120. data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +1230 -137
  121. data/lib/aspera/schema/IBM Aspera on Cloud Automation API-1.0.5-enhanced.yaml +2395 -0
  122. data/lib/aspera/schema/IBM_Aspera_Shares.yaml +14 -13
  123. data/lib/aspera/schema/documentation.rb +13 -3
  124. data/lib/aspera/schema/reader.rb +12 -18
  125. data/lib/aspera/schema/registry.rb +23 -1
  126. data/lib/aspera/schema/validator.rb +92 -0
  127. data/lib/aspera/secret_hider.rb +36 -25
  128. data/lib/aspera/string_ext.rb +15 -0
  129. data/lib/aspera/temp_file_manager.rb +6 -5
  130. data/lib/aspera/transfer/parameters.rb +2 -0
  131. data/lib/aspera/transfer/spec.rb +1 -0
  132. data/lib/aspera/uri_reader.rb +11 -11
  133. data/lib/aspera/web_auth/index.html +147 -0
  134. data/lib/aspera/web_auth/server.rb +81 -0
  135. data.tar.gz.sig +0 -0
  136. metadata +43 -9
  137. metadata.gz.sig +0 -0
  138. data/lib/aspera/colors.rb +0 -79
  139. data/lib/aspera/node_simulator.rb +0 -344
  140. data/lib/aspera/preview/options.rb +0 -45
  141. data/lib/aspera/rest_call_error.rb +0 -25
  142. data/lib/aspera/rest_error_analyzer.rb +0 -111
  143. data/lib/aspera/rest_errors_aspera.rb +0 -58
  144. data/lib/aspera/rest_list.rb +0 -136
  145. data/lib/aspera/web_auth.rb +0 -211
@@ -3,7 +3,10 @@
3
3
  require 'aspera/oauth/factory'
4
4
  require 'aspera/log'
5
5
  require 'aspera/assert'
6
+ require 'aspera/mime'
6
7
  require 'date'
8
+ require 'aspera/rainbow'
9
+ using Rainbow
7
10
 
8
11
  module Aspera
9
12
  module OAuth
@@ -31,7 +34,7 @@ module Aspera
31
34
  Aspera.assert_type(params, Hash)
32
35
  Aspera.assert_type(cache_ids, Array)
33
36
  # This is the OAuth API
34
- @api = Rest.new(**rest_params)
37
+ @api = Rest::Client.new(**rest_params)
35
38
  @params = params.dup.freeze
36
39
  @path_token = path_token
37
40
  @token_field = token_field
@@ -47,37 +50,40 @@ module Aspera
47
50
  # Sub path to generate token
48
51
  attr_reader :path_token
49
52
 
53
+ # Parameters for token creation call
54
+ # @param include_secret [Boolean] Add secret in default call parameters
55
+ # @param other_params [Hash] Other parameters to merge
56
+ # @return [Hash] Token creation parameters
57
+ def base_parameters(include_secret: false, **other_params)
58
+ call_params = @params.dup
59
+ call_params.delete(:client_secret) unless include_secret
60
+ call_params.merge(other_params)
61
+ end
62
+
50
63
  # Helper method to create token as per RFC
51
- # @return [Net::HTTPResponse] raw HTTP response with token
52
- # @raise [RestCallError] if not 2XX code
53
- def create_token_call(creation_params)
54
- Log.log.debug { 'Generating a new token'.bg_green }
64
+ # @param other_params [Hash] Additional parameter to base parameters
65
+ # @return [Net::HTTPResponse] Raw HTTP response with token
66
+ # @raise [Rest::CallError] If not 2XX code
67
+ def create_token_base(include_secret: false, **other_params)
68
+ Log.log.debug { 'Generating a new token'.bg(:green) }
69
+ creation_params = base_parameters(include_secret: include_secret, **other_params)
55
70
  return @api.create(@path_token, nil, query: creation_params, ret: :resp) if @use_query
56
71
  return @api.create(@path_token, creation_params, content_type: Mime::WWW, ret: :resp)
57
72
  end
58
73
 
59
- # Create base parameters for token creation calls
60
- # @param add_secret [Boolean] Add secret in default call parameters
61
- # @return [Hash] Optional general parameters
62
- def base_params(add_secret: false)
63
- call_params = @params.dup
64
- call_params.delete(:client_secret) unless add_secret
65
- return call_params
66
- end
67
-
68
- # @return [String] value suitable for Authorization header
74
+ # @return [String] Value suitable for Authorization header: adds `Bearer `
69
75
  def authorization(**kwargs)
70
76
  return OAuth::Factory.bearer_authorization(token(**kwargs))
71
77
  end
72
78
 
73
- # get an OAuth v2 token (generated, cached, refreshed)
74
- # call token() to get a token.
75
- # if a token is expired (api returns 4xx), call again token(refresh: true)
79
+ # Get a OAuth v2 token (generated, cached, refreshed)
80
+ # If a token is expired (api returns 4xx), call again token(refresh: true)
76
81
  # @param cache [Boolean] set to false to disable cache
77
82
  # @param refresh [Boolean] set to true to force refresh or re-generation (if previous failed)
83
+ # @return [String] The bearer token
78
84
  def token(cache: true, refresh: false)
79
- # get token info from cache (or nil), decoded with date and expiration status
80
- token_info = Factory.instance.get_token_info(@token_cache_id) if cache
85
+ # Get token info from cache (or nil), decoded with date and expiration status
86
+ token_info = Factory.instance.get_token_info(@token_cache_id, token_field: @token_field) if cache
81
87
  token_data = nil
82
88
  unless token_info.nil?
83
89
  token_data = token_info[:data]
@@ -90,17 +96,21 @@ module Aspera
90
96
  Log.log.trace1 { "refresh: #{refresh} expired: #{token_info[:expired]}" }
91
97
  refresh_token = nil
92
98
  if token_data.key?('refresh_token') && !token_data['refresh_token'].eql?('not_supported')
93
- # save possible refresh token, before deleting the cache
99
+ # Save possible refresh token, before deleting the cache
94
100
  refresh_token = token_data['refresh_token']
95
101
  end
96
- # delete cache
102
+ # Delete cache
97
103
  Factory.instance.persist_mgr.delete(@token_cache_id)
98
104
  token_data = nil
99
- # lets try the existing refresh token
105
+ # Lets try the existing refresh token
100
106
  # NOTE: AoC admin token has no refresh, and lives by default 1800secs
101
107
  if !refresh_token.nil?
102
108
  begin
103
- http = create_token_call(base_params(add_secret: true).merge(grant_type: 'refresh_token', refresh_token: refresh_token))
109
+ http = create_token_base(
110
+ include_secret: true,
111
+ grant_type: 'refresh_token',
112
+ refresh_token: refresh_token
113
+ )
104
114
  # Save only if success
105
115
  json_data = http.body
106
116
  token_data = JSON.parse(json_data)
@@ -116,13 +126,14 @@ module Aspera
116
126
  # no cache, nor refresh: generate a token
117
127
  if token_data.nil?
118
128
  # Call the method-specific token creation
119
- # which returns the result of create_token_call
129
+ # which returns the result of create_token_base
120
130
  json_data = create_token.body
121
131
  token_data = JSON.parse(json_data)
122
132
  Factory.instance.persist_mgr.put(@token_cache_id, json_data)
133
+ Factory.instance.get_token_info(@token_cache_id, token_field: @token_field)
123
134
  end
124
135
  Aspera.assert(token_data.key?(@token_field)) { "API error: No such field in answer: #{@token_field}" } unless token_data.nil?
125
- # ok we shall have a token here
136
+ # Ok we shall have a token here
126
137
  return token_data[@token_field]
127
138
  end
128
139
  end
@@ -2,6 +2,7 @@
2
2
 
3
3
  require 'aspera/id_generator'
4
4
  require 'aspera/assert'
5
+ require 'aspera/string_ext'
5
6
  require 'singleton'
6
7
  require 'base64'
7
8
  module Aspera
@@ -120,7 +121,8 @@ module Aspera
120
121
  data.each.map do |k, v|
121
122
  info = {id: k}
122
123
  begin; info.merge!(JSON.parse(v)); rescue StandardError; nil; end
123
- d = decode_token(info.delete(TOKEN_FIELD))
124
+ token_val = info.delete(TOKEN_FIELD) || info.delete('token')
125
+ d = decode_token(token_val)
124
126
  info.merge(d) if d
125
127
  info
126
128
  end
@@ -128,13 +130,15 @@ module Aspera
128
130
 
129
131
  # Get token information from cache
130
132
  # @param id [String] identifier of token
133
+ # @param token_field [String, nil] Field containing the token, or nil to auto-detect
131
134
  # @return [Hash] token internal information , including Date object for `expiration_date`
132
- def get_token_info(id)
135
+ def get_token_info(id, token_field: nil)
133
136
  token_raw_string = persist_mgr.get(id)
134
137
  return if token_raw_string.nil?
135
138
  token_data = JSON.parse(token_raw_string)
136
139
  Aspera.assert_type(token_data, Hash)
137
- decoded_token = decode_token(token_data[TOKEN_FIELD])
140
+ token_value = token_data[token_field] || token_data[TOKEN_FIELD] || token_data['token']
141
+ decoded_token = decode_token(token_value)
138
142
  info = {data: token_data}
139
143
  if decoded_token.is_a?(Hash)
140
144
  info[:decoded] = decoded_token
@@ -23,7 +23,7 @@ module Aspera
23
23
  end
24
24
 
25
25
  def create_token
26
- return create_token_call(base_params.merge(@create_params))
26
+ create_token_base(**@create_params)
27
27
  end
28
28
  end
29
29
  Factory.instance.register_token_creator(Generic)
@@ -0,0 +1,34 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'aspera/oauth/base'
4
+ require 'digest'
5
+
6
+ module Aspera
7
+ module OAuth
8
+ # Token creator using JSON credentials (e.g., username/password or apikey) sent to a login endpoint
9
+ class JsonCredentials < Base
10
+ # @param json [Hash] Body parameters to send as JSON (e.g. {username:, password:} or {apikey:})
11
+ # @param generic_params [Hash] Generic parameters for OAuth::Base
12
+ def initialize(
13
+ json:,
14
+ **generic_params
15
+ )
16
+ cache_id = json[:username] || Digest::SHA256.hexdigest(json[:apikey] || json.to_s)[0..23]
17
+ super(**generic_params, cache_ids: [cache_id])
18
+ @body = json
19
+ end
20
+
21
+ def create_token
22
+ api.call(
23
+ operation: 'POST',
24
+ subpath: path_token,
25
+ content_type: Mime::JSON,
26
+ body: @body,
27
+ headers: {'Accept' => Mime::JSON},
28
+ ret: :resp
29
+ )
30
+ end
31
+ end
32
+ Factory.instance.register_token_creator(JsonCredentials)
33
+ end
34
+ end
@@ -1,6 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require 'aspera/oauth/base'
4
+ require 'aspera/environment'
4
5
  require 'aspera/assert'
5
6
  require 'securerandom'
6
7
  require 'openssl'
@@ -17,10 +18,8 @@ module Aspera
17
18
  class << self
18
19
  def generate_rsa_private_key(path:, length: DEFAULT_PRIV_KEY_LENGTH)
19
20
  priv_key = OpenSSL::PKey::RSA.new(length)
20
- File.write(path, priv_key.to_s)
21
- File.write("#{path}.pub", priv_key.public_key.to_s)
22
- Environment.restrict_file_access(path)
23
- Environment.restrict_file_access("#{path}.pub")
21
+ Environment.write_file_restricted(path, force: true) { priv_key.to_s }
22
+ Environment.write_file_restricted("#{path}.pub", force: true) { priv_key.public_key.to_s }
24
23
  nil
25
24
  end
26
25
  end
@@ -59,7 +58,7 @@ module Aspera
59
58
  }.merge(@additional_payload)
60
59
  Log.dump(:jwt_payload, jwt_payload)
61
60
  assertion = JWT.encode(jwt_payload, @private_key_obj, 'RS256', @headers)
62
- return create_token_call(base_params.merge(grant_type: GRANT_TYPE, assertion: assertion))
61
+ create_token_base(grant_type: GRANT_TYPE, assertion: assertion)
63
62
  end
64
63
  end
65
64
  Factory.instance.register_token_creator(Jwt)
@@ -2,7 +2,7 @@
2
2
 
3
3
  require 'aspera/oauth/base'
4
4
  require 'aspera/environment'
5
- require 'aspera/web_auth'
5
+ require 'aspera/web_auth/server'
6
6
  require 'aspera/assert'
7
7
  module Aspera
8
8
  module OAuth
@@ -32,23 +32,24 @@ module Aspera
32
32
  random_state = SecureRandom.uuid
33
33
  login_page_url = Rest.build_uri(
34
34
  "#{api.base_url}/#{@path_authorize}",
35
- base_params.merge(response_type: 'code', redirect_uri: @redirect_uri, state: random_state)
35
+ base_parameters(response_type: 'code', redirect_uri: @redirect_uri, state: random_state)
36
36
  )
37
37
  # here, we need a human to authorize on a web page
38
38
  Log.log.debug { "login_page_url=#{login_page_url}" }
39
39
  # start a web server to receive request code
40
- web_server = WebAuth.new(@redirect_uri, self.class.additional_info)
40
+ web_server = WebAuth::Server.new(@redirect_uri, self.class.additional_info)
41
41
  # start browser on login page
42
42
  Environment.instance.open_uri(login_page_url)
43
43
  # wait for code in request
44
44
  received_params = web_server.received_request
45
45
  Aspera.assert(random_state.eql?(received_params['state'])) { 'wrong received state' }
46
46
  # exchange code for token
47
- return create_token_call(base_params(add_secret: true).merge(
48
- grant_type: 'authorization_code',
49
- code: received_params['code'],
50
- redirect_uri: @redirect_uri
51
- ))
47
+ create_token_base(
48
+ include_secret: true,
49
+ grant_type: 'authorization_code',
50
+ code: received_params['code'],
51
+ redirect_uri: @redirect_uri
52
+ )
52
53
  end
53
54
  end
54
55
  Factory.instance.register_token_creator(Web)
data/lib/aspera/oauth.rb CHANGED
@@ -6,3 +6,4 @@ require 'aspera/oauth/generic'
6
6
  require 'aspera/oauth/jwt'
7
7
  require 'aspera/oauth/web'
8
8
  require 'aspera/oauth/url_json'
9
+ require 'aspera/oauth/json_credentials'
@@ -44,9 +44,7 @@ module Aspera
44
44
  Aspera.assert_type(value, String)
45
45
  persist_filepath = id_to_filepath(object_id)
46
46
  Log.log.debug { "persistency saving: #{persist_filepath}" }
47
- FileUtils.rm_f(persist_filepath)
48
- File.write(persist_filepath, value)
49
- Environment.restrict_file_access(persist_filepath)
47
+ Environment.write_file_restricted(persist_filepath, force: true) { value }
50
48
  @cache[object_id] = value
51
49
  nil
52
50
  end
@@ -57,11 +57,11 @@ module Aspera
57
57
 
58
58
  private_constant :SUPPORTED_MIME_TYPES
59
59
 
60
- # @attr use_mimemagic [Boolean] `true` to use mimemagic to determine real mime type based on file content
61
- attr_accessor :use_mimemagic
60
+ # @attr detect_mime [Boolean] `true` to determine real mime type based on file content
61
+ attr_accessor :detect_mime
62
62
 
63
63
  def initialize
64
- @use_mimemagic = false
64
+ @detect_mime = false
65
65
  end
66
66
 
67
67
  # @param mimetype [String] mime type
@@ -81,7 +81,7 @@ module Aspera
81
81
  # @return [Symbol] file type, one of enum CONVERSION_TYPES
82
82
  # @raise [RuntimeError] if no conversion type found
83
83
  def conversion_type(filepath, mimetype)
84
- Log.log.debug { "conversion_type(#{filepath},mime=#{mimetype},magic=#{@use_mimemagic})" }
84
+ Log.log.debug { "conversion_type(#{filepath},mime=#{mimetype},magic=#{@detect_mime})" }
85
85
  # Default type or empty means no type
86
86
  mimetype = TYPE_NOT_FOUND if mimetype.nil? || (mimetype.is_a?(String) && mimetype.empty?)
87
87
  mimetype = Marcel::MimeType.for(Pathname.new(filepath), name: File.basename(filepath), declared_type: mimetype)
@@ -4,9 +4,9 @@
4
4
  # spellchecker:ignore soffice pauseframes libx264 trunc bufsize muxer apng libmp3lame maxrate posterize movflags faststart
5
5
  # spellchecker:ignore palettegen paletteuse pointsize bordercolor repage lanczos unoconv optipng reencode conv transframes
6
6
 
7
- require 'aspera/preview/options'
8
7
  require 'aspera/preview/utils'
9
8
  require 'aspera/preview/file_types'
9
+ require 'aspera/schema/registry'
10
10
  require 'aspera/log'
11
11
  require 'aspera/assert'
12
12
 
@@ -20,6 +20,9 @@ module Aspera
20
20
  # List of valid ffmpeg option keys for reencode configuration.
21
21
  FFMPEG_OPTIONS_LIST = %w[in out].freeze
22
22
 
23
+ # Generator options, members defined in `options.schema.yaml`.
24
+ Options = Struct.new(*Schema::Registry.instance.reader(Schema::Registry::PREVIEW_OPTIONS).current['properties'].keys.map(&:to_sym))
25
+
23
26
  # CLI needs to know conversion type to know if need skip it.
24
27
  # One of CONVERSION_TYPES.
25
28
  attr_reader :conversion_type, :destination
@@ -251,6 +254,7 @@ module Aspera
251
254
  tmp_pdf_file = File.join(this_tmpdir, "#{File.basename(@source, File.extname(@source))}.pdf")
252
255
  case @options.office_conversion
253
256
  when :unoconv
257
+ # `unoconv` is a python wrapper over `soffice`
254
258
  Utils.silent_execute(
255
259
  :unoconv,
256
260
  '-f', 'pdf',
@@ -261,6 +265,12 @@ module Aspera
261
265
  Utils.silent_execute(
262
266
  :soffice,
263
267
  '--headless',
268
+ '--invisible',
269
+ '--nocrashreport',
270
+ '--nodefault',
271
+ '--nofirststartwizard',
272
+ '--nologo',
273
+ '--norestore',
264
274
  '--convert-to', 'pdf',
265
275
  '--outdir', File.dirname(tmp_pdf_file),
266
276
  @source
@@ -0,0 +1,119 @@
1
+ $schema: https://json-schema.org/draft/2020-12/schema
2
+ $id: https://github.com/IBM/aspera-cli/tree/main/lib/aspera/preview/options.schema.yaml
3
+ title: Preview generator options
4
+ description: >-
5
+ Options of the preview generator: tools and parameters used to generate PNG thumbnails and MP4 video previews.
6
+ $comment: >-
7
+ Used to declare the options of the `preview` plugin: values of `enum` properties are `Symbol`.
8
+ Scaling uses `ffmpeg` scale filter arguments, see: https://trac.ffmpeg.org/wiki/Scaling
9
+ (`iw`/`ih`: input width/height, `-x`: keep aspect ratio, with a value multiple of `x`).
10
+ type: object
11
+ additionalProperties: false
12
+ properties:
13
+ office_conversion:
14
+ description: 'PNG thumbnail of office document: conversion tool.'
15
+ type: string
16
+ enum: [soffice, unoconv]
17
+ default: soffice
18
+ max_size:
19
+ description: 'Maximum size (in bytes) of a preview file: a warning is logged if exceeded.'
20
+ type: integer
21
+ minimum: 1
22
+ default: 16777216
23
+ thumb_vid_scale:
24
+ description: 'PNG thumbnail of video: frame size, as `ffmpeg` scale filter argument.'
25
+ type: string
26
+ default: '-1:min(ih,100)'
27
+ thumb_vid_fraction:
28
+ description: 'PNG thumbnail of video (`fixed`): position of the snapshot, as a fraction of the video duration.'
29
+ type: number
30
+ minimum: 0
31
+ maximum: 1
32
+ default: 0.1
33
+ thumb_img_size:
34
+ description: 'PNG thumbnail of image, PDF, office document or text: size (in pixels).'
35
+ type: integer
36
+ minimum: 1
37
+ default: 800
38
+ thumb_text_font:
39
+ description: 'PNG thumbnail of text: font name, as listed by `magick identify -list font`.'
40
+ type: string
41
+ default: Courier
42
+ video_conversion:
43
+ description: >-
44
+ MP4 video preview: generation method.
45
+ `reencode`: re-encode the beginning of the video,
46
+ `blend`: key frames with transitions,
47
+ `clips`: concatenation of short clips.
48
+ type: string
49
+ enum: [reencode, blend, clips]
50
+ default: reencode
51
+ video_png_conv:
52
+ description: >-
53
+ PNG thumbnail of video: generation method.
54
+ `fixed`: single frame,
55
+ `animated`: animated PNG.
56
+ type: string
57
+ enum: [fixed, animated]
58
+ default: fixed
59
+ video_scale:
60
+ description: 'MP4 video preview: frame size, as `ffmpeg` scale filter argument.'
61
+ type: string
62
+ default: 'min(iw,360):-2'
63
+ video_start_sec:
64
+ description: 'Video: start offset (in seconds) of MP4 video preview and of `animated` PNG thumbnail.'
65
+ type: integer
66
+ minimum: 0
67
+ default: 10
68
+ video_codec:
69
+ description: >-
70
+ MP4 video preview (`reencode`): `ffmpeg` video codec, e.g. `libx264`, `h264_videotoolbox`, `h264_nvenc`.
71
+ Default: first available H.264 encoder.
72
+ type: string
73
+ reencode_ffmpeg:
74
+ description: >-
75
+ MP4 video preview (`reencode`): `ffmpeg` options replacing the default ones.
76
+ type: object
77
+ additionalProperties: false
78
+ properties:
79
+ in:
80
+ description: Input options.
81
+ type: array
82
+ items:
83
+ type: [string, number]
84
+ out:
85
+ description: Output options.
86
+ type: array
87
+ items:
88
+ type: [string, number]
89
+ default: {}
90
+ blend_keyframes:
91
+ description: 'MP4 video preview (`blend`): number of key frames.'
92
+ type: integer
93
+ minimum: 1
94
+ default: 30
95
+ blend_pauseframes:
96
+ description: 'MP4 video preview (`blend`): number of pause frames (repetitions of each key frame).'
97
+ type: integer
98
+ minimum: 0
99
+ default: 3
100
+ blend_transframes:
101
+ description: 'MP4 video preview (`blend`): number of transition frames between key frames.'
102
+ type: integer
103
+ minimum: 0
104
+ default: 5
105
+ blend_fps:
106
+ description: 'MP4 video preview (`blend`): frames per second.'
107
+ type: integer
108
+ minimum: 1
109
+ default: 15
110
+ clips_count:
111
+ description: 'MP4 video preview (`clips`): number of clips.'
112
+ type: integer
113
+ minimum: 1
114
+ default: 5
115
+ clips_length:
116
+ description: 'Video: length (in seconds) of each clip of MP4 video preview (`clips`), and of `animated` PNG thumbnail.'
117
+ type: integer
118
+ minimum: 1
119
+ default: 5
@@ -2,11 +2,12 @@
2
2
 
3
3
  # cspell:words Magick MAGICKCORE ITERM mintty winsize termcap
4
4
 
5
- require 'rainbow'
6
5
  require 'base64'
7
6
  require 'io/console'
8
7
  require 'aspera/log'
9
8
  require 'aspera/environment'
9
+ require 'aspera/rainbow'
10
+ using Rainbow
10
11
 
11
12
  module Aspera
12
13
  module Preview
@@ -155,9 +156,9 @@ module Aspera
155
156
  row_data.each_with_index do |pixel_rgb, col|
156
157
  text_pixels.push("\n") if col.eql?(0) && !row.eql?(0)
157
158
  if double
158
- text_pixels.push(Rainbow('▄').background(pixel_rgb).foreground(pixel_colors[row + 1][col]))
159
+ text_pixels.push('▄'.bg(pixel_rgb).fg(pixel_colors[row + 1][col]))
159
160
  else
160
- text_pixels.push(Rainbow(' ').background(pixel_rgb))
161
+ text_pixels.push(' '.bg(pixel_rgb))
161
162
  end
162
163
  end
163
164
  end
@@ -25,6 +25,9 @@ module Aspera
25
25
  private_constant :EXTERNAL_TOOLS, :TEMP_FORMAT, :FFMPEG_DEFAULT_PARAMS, :H264_ENCODER_PREFERENCE
26
26
 
27
27
  class << self
28
+ # @param tool [Symbol] either `unoconv` or `soffice`
29
+ attr_accessor :office_tool
30
+
28
31
  # Return the first H.264 encoder available in the local ffmpeg installation.
29
32
  # Result is memoized after the first call.
30
33
  # @return [String] encoder name (e.g. 'libx264', 'libopenh264')
@@ -32,8 +35,10 @@ module Aspera
32
35
  def available_h264_encoder
33
36
  return @available_h264_encoder if defined?(@available_h264_encoder)
34
37
  stdout, = execute(:ffmpeg, '-encoders', mode: :capture, exception: false)
35
- @available_h264_encoder = H264_ENCODER_PREFERENCE.find { |enc| stdout.include?(enc) } ||
36
- raise("No supported H.264 encoder found in ffmpeg. Available: #{stdout.lines.grep(/h264/i).map(&:strip).join(', ')}")
38
+ available = stdout.lines.grep(/h264/i).map { |l| l.split[1] }
39
+ @available_h264_encoder = H264_ENCODER_PREFERENCE.find { |enc| available.include?(enc) }
40
+ Aspera.assert(@available_h264_encoder) { "No supported H.264 encoder found in ffmpeg. Available: #{available.join(', ')}" }
41
+ @available_h264_encoder
37
42
  end
38
43
 
39
44
  # Check that external tools can be executed.
@@ -42,10 +47,8 @@ module Aspera
42
47
  # @return [nil]
43
48
  def check_tools(skip_types = [])
44
49
  tools_to_check = EXTERNAL_TOOLS.dup
45
- if skip_types.include?(:office)
46
- tools_to_check.delete(:unoconv)
47
- tools_to_check.delete(:soffice)
48
- end
50
+ tools_to_check.delete(:unoconv) if skip_types.include?(:office) || office_tool.eql?(:soffice)
51
+ tools_to_check.delete(:soffice) if skip_types.include?(:office) || office_tool.eql?(:unoconv)
49
52
  # Check for binaries
50
53
  tools_to_check.each do |command_sym|
51
54
  silent_execute(command_sym, '-h')
@@ -53,7 +53,7 @@ module Aspera
53
53
 
54
54
  # Base URL for CDN of Connect
55
55
  def cdn_api
56
- Rest.new(base_url: CDN_BASE_URL)
56
+ Rest::Client.new(base_url: CDN_BASE_URL)
57
57
  end
58
58
 
59
59
  # Retrieve structure from cloud (CDN) with all versions available
@@ -0,0 +1,7 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Load Rainbow with terminal detection (tty, TERM=dumb, CLICOLOR_FORCE)
4
+ # and its String refinement.
5
+ # Each file using colors must still activate it with: using Rainbow
6
+ require 'rainbow'
7
+ require 'rainbow/refinement'
@@ -0,0 +1,72 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'aspera/rest/error_analyzer'
4
+ require 'aspera/log'
5
+ require 'xmlsimple'
6
+
7
+ module Aspera
8
+ module Rest
9
+ # REST error handlers for various Aspera REST APIs
10
+ class AsperaErrors
11
+ class << self
12
+ # handlers should probably be defined by plugins for modularity
13
+ def register_handlers
14
+ Log.log.debug('registering Aspera REST error handlers')
15
+ # Faspex 4: both user_message and internal_message, and code 200
16
+ # example: missing meta data on package creation
17
+ ErrorAnalyzer.instance.add_simple_handler(name: 'Type 1: error:user_message', path: %w[error user_message], always: true)
18
+ ErrorAnalyzer.instance.add_simple_handler(name: 'Type 2: error:description', path: %w[error description])
19
+ ErrorAnalyzer.instance.add_simple_handler(name: 'Type 3: error:internal_message', path: %w[error internal_message])
20
+ ErrorAnalyzer.instance.add_simple_handler(name: 'Type 5', path: ['error_description'])
21
+ ErrorAnalyzer.instance.add_simple_handler(name: 'Type 6', path: ['message'])
22
+ ErrorAnalyzer.instance.add_simple_handler(name: 'Type 6', path: ['failure'], always: true)
23
+ # AoC Automation
24
+ ErrorAnalyzer.instance.add_simple_handler(name: 'AoC Automation', path: ['error'])
25
+ ErrorAnalyzer.instance.add_handler('Type 7: errors[]') do |type, context|
26
+ next unless context[:data].is_a?(Hash) && context[:data]['errors'].is_a?(Hash)
27
+ # special for Shares: false positive ? (update global transfer_settings)
28
+ next if context[:data].key?('min_connect_version')
29
+ context[:data]['errors'].each do |k, v|
30
+ ErrorAnalyzer.add_error(context, type, "#{k}: #{v}")
31
+ end
32
+ end
33
+ # call to upload_setup and download_setup of node api
34
+ ErrorAnalyzer.instance.add_handler('T8:node: *_setup') do |type, context|
35
+ next unless context[:data].is_a?(Hash)
36
+ d_t_s = context[:data]['transfer_specs']
37
+ next unless d_t_s.is_a?(Array)
38
+ d_t_s.each do |res|
39
+ r_err = res.dig(*%w[transfer_spec error]) || res['error']
40
+ next unless r_err.is_a?(Hash)
41
+ ErrorAnalyzer.add_error(context, type, r_err.values.join(': '))
42
+ end
43
+ end
44
+ ErrorAnalyzer.instance.add_simple_handler(name: 'T9:IBM cloud IAM', path: ['errorMessage'])
45
+ ErrorAnalyzer.instance.add_simple_handler(name: 'T10:faspex v4', path: ['user_message'])
46
+ ErrorAnalyzer.instance.add_handler('Orchestrator') do |type, context|
47
+ next if context[:response].code.start_with?('2')
48
+ data = context[:data]
49
+ if data.is_a?(String) && data.start_with?('<?xml')
50
+ parsed = XmlSimple.xml_in(data, {'ForceArray' => false}) rescue nil
51
+ if parsed.is_a?(Hash) && parsed['content'].is_a?(String)
52
+ ErrorAnalyzer.add_error(context, type, parsed['content'].gsub(/\A"|"\z/, ''))
53
+ parsed.each do |k, v|
54
+ next if k.eql?('content')
55
+ ErrorAnalyzer.add_error(context, "#{type}(sub)", "#{k}: #{v}") if [String, Integer].include?(v.class)
56
+ end
57
+ end
58
+ next
59
+ end
60
+ next unless data.is_a?(Hash)
61
+ work_order = data['work_order']
62
+ next unless work_order.is_a?(Hash)
63
+ ErrorAnalyzer.add_error(context, type, work_order['statusDetails'])
64
+ data['missing_parameters']&.each do |param|
65
+ ErrorAnalyzer.add_error(context, type, "missing parameter: #{param}")
66
+ end
67
+ end
68
+ end
69
+ end
70
+ end
71
+ end
72
+ end
@@ -0,0 +1,27 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Aspera
4
+ module Rest
5
+ # raised on error after REST call
6
+ class CallError < StandardError
7
+ def request
8
+ @context[:request]
9
+ end
10
+
11
+ def response
12
+ @context[:response]
13
+ end
14
+
15
+ def data
16
+ @context[:data]
17
+ end
18
+
19
+ # @param context [Hash,String] with keys :messages, :request, :response, :data
20
+ def initialize(context)
21
+ context = {messages: [context]} if context.is_a?(String)
22
+ @context = context
23
+ super(@context[:messages].join("\n"))
24
+ end
25
+ end
26
+ end
27
+ end