aspera-cli 4.27.2 → 4.27.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +133 -0
- data/CONTRIBUTING.md +5 -2
- data/bin/ascli +3 -1
- data/docs/README.md +1006 -766
- data/lib/aspera/agent/base.rb +7 -2
- data/lib/aspera/agent/connect.rb +6 -8
- data/lib/aspera/agent/desktop.rb +2 -6
- data/lib/aspera/agent/direct.rb +52 -22
- data/lib/aspera/agent/node.rb +9 -8
- data/lib/aspera/agent/transferd.rb +2 -2
- data/lib/aspera/api/alee.rb +1 -1
- data/lib/aspera/api/aoc.rb +14 -12
- data/lib/aspera/api/ats.rb +1 -1
- data/lib/aspera/api/cos_node.rb +2 -2
- data/lib/aspera/api/faspex.rb +11 -7
- data/lib/aspera/api/httpgw.rb +38 -35
- data/lib/aspera/api/node.rb +39 -34
- data/lib/aspera/ascmd.rb +3 -1
- data/lib/aspera/ascp/installation.rb +63 -28
- data/lib/aspera/ascp/management.rb +1 -0
- data/lib/aspera/assert.rb +4 -0
- data/lib/aspera/cli/ascp_actions.rb +20 -41
- data/lib/aspera/cli/async_transfer_store.rb +12 -11
- data/lib/aspera/cli/bootstrapper.rb +14 -16
- data/lib/aspera/cli/command_line.rb +252 -0
- data/lib/aspera/cli/command_registry.rb +215 -37
- data/lib/aspera/cli/command_spec.rb +104 -15
- data/lib/aspera/cli/completion/ascli.bash +12 -0
- data/lib/aspera/cli/completion/ascli.fish +16 -0
- data/lib/aspera/cli/completion/ascli.zsh +19 -0
- data/lib/aspera/cli/context.rb +3 -0
- data/lib/aspera/cli/deprecation.rb +37 -0
- data/lib/aspera/cli/extended_value.rb +6 -3
- data/lib/aspera/cli/formatter.rb +94 -80
- data/lib/aspera/cli/gem_checker.rb +1 -1
- data/lib/aspera/cli/hints.rb +7 -6
- data/lib/aspera/cli/http.rb +22 -34
- data/lib/aspera/cli/info.rb +3 -0
- data/lib/aspera/cli/mcp_tool.rb +47 -83
- data/lib/aspera/cli/option_declarator.rb +33 -42
- data/lib/aspera/cli/option_registry.rb +69 -0
- data/lib/aspera/cli/option_types.rb +105 -0
- data/lib/aspera/cli/option_value.rb +278 -0
- data/lib/aspera/cli/options.schema.yaml +124 -15
- data/lib/aspera/cli/parser.rb +333 -862
- data/lib/aspera/cli/plugins/alee.rb +7 -4
- data/lib/aspera/cli/plugins/aoc.rb +545 -518
- data/lib/aspera/cli/plugins/ats.rb +59 -80
- data/lib/aspera/cli/plugins/base.rb +221 -265
- data/lib/aspera/cli/plugins/basic_auth.rb +2 -10
- data/lib/aspera/cli/plugins/config.rb +263 -184
- data/lib/aspera/cli/plugins/console.rb +103 -39
- data/lib/aspera/cli/plugins/cos.rb +6 -23
- data/lib/aspera/cli/plugins/factory.rb +3 -0
- data/lib/aspera/cli/plugins/faspex5.rb +204 -182
- data/lib/aspera/cli/plugins/faspio.rb +6 -11
- data/lib/aspera/cli/plugins/httpgw.rb +8 -11
- data/lib/aspera/cli/plugins/mcp.rb +20 -55
- data/lib/aspera/cli/plugins/node.rb +300 -327
- data/lib/aspera/cli/plugins/orchestrator.rb +152 -110
- data/lib/aspera/cli/plugins/preview.rb +96 -105
- data/lib/aspera/cli/plugins/server.rb +78 -53
- data/lib/aspera/cli/plugins/shares.rb +80 -131
- data/lib/aspera/cli/preset_actions.rb +44 -27
- data/lib/aspera/cli/preset_manager.rb +44 -19
- data/lib/aspera/cli/prompt.rb +36 -0
- data/lib/aspera/cli/result.rb +42 -36
- data/lib/aspera/cli/runner.rb +32 -59
- data/lib/aspera/cli/special_values.rb +5 -0
- data/lib/aspera/cli/sync_actions.rb +51 -46
- data/lib/aspera/cli/terminal_formatter.rb +9 -3
- data/lib/aspera/cli/transfer_actions.rb +14 -9
- data/lib/aspera/cli/transfer_agent.rb +34 -38
- data/lib/aspera/cli/transfer_progress.rb +290 -55
- data/lib/aspera/cli/vault_manager.rb +0 -17
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/cli/wizard.rb +5 -3
- data/lib/aspera/coverage.rb +1 -1
- data/lib/aspera/environment.rb +35 -5
- data/lib/aspera/faspex_gw.rb +2 -1
- data/lib/aspera/faspex_postproc.rb +1 -0
- data/lib/aspera/graphql.rb +5 -5
- data/lib/aspera/json_rpc/client.rb +5 -5
- data/lib/aspera/keychain/encrypted_hash.rb +2 -2
- data/lib/aspera/keychain/factory.rb +2 -1
- data/lib/aspera/keychain/one_password_api.rb +1 -1
- data/lib/aspera/link_header.rb +2 -2
- data/lib/aspera/log.rb +47 -27
- data/lib/aspera/markdown.rb +2 -0
- data/lib/aspera/mime.rb +25 -0
- data/lib/aspera/node_emulator.rb +759 -0
- data/lib/aspera/oauth/base.rb +37 -26
- data/lib/aspera/oauth/factory.rb +7 -3
- data/lib/aspera/oauth/generic.rb +1 -1
- data/lib/aspera/oauth/json_credentials.rb +34 -0
- data/lib/aspera/oauth/jwt.rb +4 -5
- data/lib/aspera/oauth/web.rb +9 -8
- data/lib/aspera/oauth.rb +1 -0
- data/lib/aspera/persistency_folder.rb +1 -3
- data/lib/aspera/preview/file_types.rb +4 -4
- data/lib/aspera/preview/generator.rb +11 -1
- data/lib/aspera/preview/options.schema.yaml +119 -0
- data/lib/aspera/preview/terminal.rb +4 -3
- data/lib/aspera/preview/utils.rb +9 -6
- data/lib/aspera/products/connect.rb +1 -1
- data/lib/aspera/rainbow.rb +7 -0
- data/lib/aspera/rest/aspera_errors.rb +72 -0
- data/lib/aspera/rest/call_error.rb +27 -0
- data/lib/aspera/rest/client.rb +523 -0
- data/lib/aspera/rest/error_analyzer.rb +113 -0
- data/lib/aspera/rest/list.rb +149 -0
- data/lib/aspera/rest/parameters.rb +55 -0
- data/lib/aspera/rest/util.rb +176 -0
- data/lib/aspera/rest.rb +7 -621
- data/lib/aspera/schema/IBM Aspera Console-enhanced.yaml +1125 -0
- data/lib/aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml +0 -20
- data/lib/aspera/schema/IBM Aspera Orchestrator API-v1.yaml +1784 -0
- data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +1230 -137
- data/lib/aspera/schema/IBM Aspera on Cloud Automation API-1.0.5-enhanced.yaml +2395 -0
- data/lib/aspera/schema/IBM_Aspera_Shares.yaml +14 -13
- data/lib/aspera/schema/documentation.rb +13 -3
- data/lib/aspera/schema/reader.rb +12 -18
- data/lib/aspera/schema/registry.rb +23 -1
- data/lib/aspera/schema/validator.rb +92 -0
- data/lib/aspera/secret_hider.rb +36 -25
- data/lib/aspera/string_ext.rb +15 -0
- data/lib/aspera/temp_file_manager.rb +6 -5
- data/lib/aspera/transfer/parameters.rb +2 -0
- data/lib/aspera/transfer/spec.rb +1 -0
- data/lib/aspera/uri_reader.rb +11 -11
- data/lib/aspera/web_auth/index.html +147 -0
- data/lib/aspera/web_auth/server.rb +81 -0
- data.tar.gz.sig +0 -0
- metadata +43 -9
- metadata.gz.sig +0 -0
- data/lib/aspera/colors.rb +0 -79
- data/lib/aspera/node_simulator.rb +0 -344
- data/lib/aspera/preview/options.rb +0 -45
- data/lib/aspera/rest_call_error.rb +0 -25
- data/lib/aspera/rest_error_analyzer.rb +0 -111
- data/lib/aspera/rest_errors_aspera.rb +0 -58
- data/lib/aspera/rest_list.rb +0 -136
- data/lib/aspera/web_auth.rb +0 -211
data/lib/aspera/oauth/base.rb
CHANGED
|
@@ -3,7 +3,10 @@
|
|
|
3
3
|
require 'aspera/oauth/factory'
|
|
4
4
|
require 'aspera/log'
|
|
5
5
|
require 'aspera/assert'
|
|
6
|
+
require 'aspera/mime'
|
|
6
7
|
require 'date'
|
|
8
|
+
require 'aspera/rainbow'
|
|
9
|
+
using Rainbow
|
|
7
10
|
|
|
8
11
|
module Aspera
|
|
9
12
|
module OAuth
|
|
@@ -31,7 +34,7 @@ module Aspera
|
|
|
31
34
|
Aspera.assert_type(params, Hash)
|
|
32
35
|
Aspera.assert_type(cache_ids, Array)
|
|
33
36
|
# This is the OAuth API
|
|
34
|
-
@api = Rest.new(**rest_params)
|
|
37
|
+
@api = Rest::Client.new(**rest_params)
|
|
35
38
|
@params = params.dup.freeze
|
|
36
39
|
@path_token = path_token
|
|
37
40
|
@token_field = token_field
|
|
@@ -47,37 +50,40 @@ module Aspera
|
|
|
47
50
|
# Sub path to generate token
|
|
48
51
|
attr_reader :path_token
|
|
49
52
|
|
|
53
|
+
# Parameters for token creation call
|
|
54
|
+
# @param include_secret [Boolean] Add secret in default call parameters
|
|
55
|
+
# @param other_params [Hash] Other parameters to merge
|
|
56
|
+
# @return [Hash] Token creation parameters
|
|
57
|
+
def base_parameters(include_secret: false, **other_params)
|
|
58
|
+
call_params = @params.dup
|
|
59
|
+
call_params.delete(:client_secret) unless include_secret
|
|
60
|
+
call_params.merge(other_params)
|
|
61
|
+
end
|
|
62
|
+
|
|
50
63
|
# Helper method to create token as per RFC
|
|
51
|
-
# @
|
|
52
|
-
# @
|
|
53
|
-
|
|
54
|
-
|
|
64
|
+
# @param other_params [Hash] Additional parameter to base parameters
|
|
65
|
+
# @return [Net::HTTPResponse] Raw HTTP response with token
|
|
66
|
+
# @raise [Rest::CallError] If not 2XX code
|
|
67
|
+
def create_token_base(include_secret: false, **other_params)
|
|
68
|
+
Log.log.debug { 'Generating a new token'.bg(:green) }
|
|
69
|
+
creation_params = base_parameters(include_secret: include_secret, **other_params)
|
|
55
70
|
return @api.create(@path_token, nil, query: creation_params, ret: :resp) if @use_query
|
|
56
71
|
return @api.create(@path_token, creation_params, content_type: Mime::WWW, ret: :resp)
|
|
57
72
|
end
|
|
58
73
|
|
|
59
|
-
#
|
|
60
|
-
# @param add_secret [Boolean] Add secret in default call parameters
|
|
61
|
-
# @return [Hash] Optional general parameters
|
|
62
|
-
def base_params(add_secret: false)
|
|
63
|
-
call_params = @params.dup
|
|
64
|
-
call_params.delete(:client_secret) unless add_secret
|
|
65
|
-
return call_params
|
|
66
|
-
end
|
|
67
|
-
|
|
68
|
-
# @return [String] value suitable for Authorization header
|
|
74
|
+
# @return [String] Value suitable for Authorization header: adds `Bearer `
|
|
69
75
|
def authorization(**kwargs)
|
|
70
76
|
return OAuth::Factory.bearer_authorization(token(**kwargs))
|
|
71
77
|
end
|
|
72
78
|
|
|
73
|
-
#
|
|
74
|
-
#
|
|
75
|
-
# if a token is expired (api returns 4xx), call again token(refresh: true)
|
|
79
|
+
# Get a OAuth v2 token (generated, cached, refreshed)
|
|
80
|
+
# If a token is expired (api returns 4xx), call again token(refresh: true)
|
|
76
81
|
# @param cache [Boolean] set to false to disable cache
|
|
77
82
|
# @param refresh [Boolean] set to true to force refresh or re-generation (if previous failed)
|
|
83
|
+
# @return [String] The bearer token
|
|
78
84
|
def token(cache: true, refresh: false)
|
|
79
|
-
#
|
|
80
|
-
token_info = Factory.instance.get_token_info(@token_cache_id) if cache
|
|
85
|
+
# Get token info from cache (or nil), decoded with date and expiration status
|
|
86
|
+
token_info = Factory.instance.get_token_info(@token_cache_id, token_field: @token_field) if cache
|
|
81
87
|
token_data = nil
|
|
82
88
|
unless token_info.nil?
|
|
83
89
|
token_data = token_info[:data]
|
|
@@ -90,17 +96,21 @@ module Aspera
|
|
|
90
96
|
Log.log.trace1 { "refresh: #{refresh} expired: #{token_info[:expired]}" }
|
|
91
97
|
refresh_token = nil
|
|
92
98
|
if token_data.key?('refresh_token') && !token_data['refresh_token'].eql?('not_supported')
|
|
93
|
-
#
|
|
99
|
+
# Save possible refresh token, before deleting the cache
|
|
94
100
|
refresh_token = token_data['refresh_token']
|
|
95
101
|
end
|
|
96
|
-
#
|
|
102
|
+
# Delete cache
|
|
97
103
|
Factory.instance.persist_mgr.delete(@token_cache_id)
|
|
98
104
|
token_data = nil
|
|
99
|
-
#
|
|
105
|
+
# Lets try the existing refresh token
|
|
100
106
|
# NOTE: AoC admin token has no refresh, and lives by default 1800secs
|
|
101
107
|
if !refresh_token.nil?
|
|
102
108
|
begin
|
|
103
|
-
http =
|
|
109
|
+
http = create_token_base(
|
|
110
|
+
include_secret: true,
|
|
111
|
+
grant_type: 'refresh_token',
|
|
112
|
+
refresh_token: refresh_token
|
|
113
|
+
)
|
|
104
114
|
# Save only if success
|
|
105
115
|
json_data = http.body
|
|
106
116
|
token_data = JSON.parse(json_data)
|
|
@@ -116,13 +126,14 @@ module Aspera
|
|
|
116
126
|
# no cache, nor refresh: generate a token
|
|
117
127
|
if token_data.nil?
|
|
118
128
|
# Call the method-specific token creation
|
|
119
|
-
# which returns the result of
|
|
129
|
+
# which returns the result of create_token_base
|
|
120
130
|
json_data = create_token.body
|
|
121
131
|
token_data = JSON.parse(json_data)
|
|
122
132
|
Factory.instance.persist_mgr.put(@token_cache_id, json_data)
|
|
133
|
+
Factory.instance.get_token_info(@token_cache_id, token_field: @token_field)
|
|
123
134
|
end
|
|
124
135
|
Aspera.assert(token_data.key?(@token_field)) { "API error: No such field in answer: #{@token_field}" } unless token_data.nil?
|
|
125
|
-
#
|
|
136
|
+
# Ok we shall have a token here
|
|
126
137
|
return token_data[@token_field]
|
|
127
138
|
end
|
|
128
139
|
end
|
data/lib/aspera/oauth/factory.rb
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
require 'aspera/id_generator'
|
|
4
4
|
require 'aspera/assert'
|
|
5
|
+
require 'aspera/string_ext'
|
|
5
6
|
require 'singleton'
|
|
6
7
|
require 'base64'
|
|
7
8
|
module Aspera
|
|
@@ -120,7 +121,8 @@ module Aspera
|
|
|
120
121
|
data.each.map do |k, v|
|
|
121
122
|
info = {id: k}
|
|
122
123
|
begin; info.merge!(JSON.parse(v)); rescue StandardError; nil; end
|
|
123
|
-
|
|
124
|
+
token_val = info.delete(TOKEN_FIELD) || info.delete('token')
|
|
125
|
+
d = decode_token(token_val)
|
|
124
126
|
info.merge(d) if d
|
|
125
127
|
info
|
|
126
128
|
end
|
|
@@ -128,13 +130,15 @@ module Aspera
|
|
|
128
130
|
|
|
129
131
|
# Get token information from cache
|
|
130
132
|
# @param id [String] identifier of token
|
|
133
|
+
# @param token_field [String, nil] Field containing the token, or nil to auto-detect
|
|
131
134
|
# @return [Hash] token internal information , including Date object for `expiration_date`
|
|
132
|
-
def get_token_info(id)
|
|
135
|
+
def get_token_info(id, token_field: nil)
|
|
133
136
|
token_raw_string = persist_mgr.get(id)
|
|
134
137
|
return if token_raw_string.nil?
|
|
135
138
|
token_data = JSON.parse(token_raw_string)
|
|
136
139
|
Aspera.assert_type(token_data, Hash)
|
|
137
|
-
|
|
140
|
+
token_value = token_data[token_field] || token_data[TOKEN_FIELD] || token_data['token']
|
|
141
|
+
decoded_token = decode_token(token_value)
|
|
138
142
|
info = {data: token_data}
|
|
139
143
|
if decoded_token.is_a?(Hash)
|
|
140
144
|
info[:decoded] = decoded_token
|
data/lib/aspera/oauth/generic.rb
CHANGED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'aspera/oauth/base'
|
|
4
|
+
require 'digest'
|
|
5
|
+
|
|
6
|
+
module Aspera
|
|
7
|
+
module OAuth
|
|
8
|
+
# Token creator using JSON credentials (e.g., username/password or apikey) sent to a login endpoint
|
|
9
|
+
class JsonCredentials < Base
|
|
10
|
+
# @param json [Hash] Body parameters to send as JSON (e.g. {username:, password:} or {apikey:})
|
|
11
|
+
# @param generic_params [Hash] Generic parameters for OAuth::Base
|
|
12
|
+
def initialize(
|
|
13
|
+
json:,
|
|
14
|
+
**generic_params
|
|
15
|
+
)
|
|
16
|
+
cache_id = json[:username] || Digest::SHA256.hexdigest(json[:apikey] || json.to_s)[0..23]
|
|
17
|
+
super(**generic_params, cache_ids: [cache_id])
|
|
18
|
+
@body = json
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def create_token
|
|
22
|
+
api.call(
|
|
23
|
+
operation: 'POST',
|
|
24
|
+
subpath: path_token,
|
|
25
|
+
content_type: Mime::JSON,
|
|
26
|
+
body: @body,
|
|
27
|
+
headers: {'Accept' => Mime::JSON},
|
|
28
|
+
ret: :resp
|
|
29
|
+
)
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
Factory.instance.register_token_creator(JsonCredentials)
|
|
33
|
+
end
|
|
34
|
+
end
|
data/lib/aspera/oauth/jwt.rb
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require 'aspera/oauth/base'
|
|
4
|
+
require 'aspera/environment'
|
|
4
5
|
require 'aspera/assert'
|
|
5
6
|
require 'securerandom'
|
|
6
7
|
require 'openssl'
|
|
@@ -17,10 +18,8 @@ module Aspera
|
|
|
17
18
|
class << self
|
|
18
19
|
def generate_rsa_private_key(path:, length: DEFAULT_PRIV_KEY_LENGTH)
|
|
19
20
|
priv_key = OpenSSL::PKey::RSA.new(length)
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
Environment.restrict_file_access(path)
|
|
23
|
-
Environment.restrict_file_access("#{path}.pub")
|
|
21
|
+
Environment.write_file_restricted(path, force: true) { priv_key.to_s }
|
|
22
|
+
Environment.write_file_restricted("#{path}.pub", force: true) { priv_key.public_key.to_s }
|
|
24
23
|
nil
|
|
25
24
|
end
|
|
26
25
|
end
|
|
@@ -59,7 +58,7 @@ module Aspera
|
|
|
59
58
|
}.merge(@additional_payload)
|
|
60
59
|
Log.dump(:jwt_payload, jwt_payload)
|
|
61
60
|
assertion = JWT.encode(jwt_payload, @private_key_obj, 'RS256', @headers)
|
|
62
|
-
|
|
61
|
+
create_token_base(grant_type: GRANT_TYPE, assertion: assertion)
|
|
63
62
|
end
|
|
64
63
|
end
|
|
65
64
|
Factory.instance.register_token_creator(Jwt)
|
data/lib/aspera/oauth/web.rb
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
require 'aspera/oauth/base'
|
|
4
4
|
require 'aspera/environment'
|
|
5
|
-
require 'aspera/web_auth'
|
|
5
|
+
require 'aspera/web_auth/server'
|
|
6
6
|
require 'aspera/assert'
|
|
7
7
|
module Aspera
|
|
8
8
|
module OAuth
|
|
@@ -32,23 +32,24 @@ module Aspera
|
|
|
32
32
|
random_state = SecureRandom.uuid
|
|
33
33
|
login_page_url = Rest.build_uri(
|
|
34
34
|
"#{api.base_url}/#{@path_authorize}",
|
|
35
|
-
|
|
35
|
+
base_parameters(response_type: 'code', redirect_uri: @redirect_uri, state: random_state)
|
|
36
36
|
)
|
|
37
37
|
# here, we need a human to authorize on a web page
|
|
38
38
|
Log.log.debug { "login_page_url=#{login_page_url}" }
|
|
39
39
|
# start a web server to receive request code
|
|
40
|
-
web_server = WebAuth.new(@redirect_uri, self.class.additional_info)
|
|
40
|
+
web_server = WebAuth::Server.new(@redirect_uri, self.class.additional_info)
|
|
41
41
|
# start browser on login page
|
|
42
42
|
Environment.instance.open_uri(login_page_url)
|
|
43
43
|
# wait for code in request
|
|
44
44
|
received_params = web_server.received_request
|
|
45
45
|
Aspera.assert(random_state.eql?(received_params['state'])) { 'wrong received state' }
|
|
46
46
|
# exchange code for token
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
47
|
+
create_token_base(
|
|
48
|
+
include_secret: true,
|
|
49
|
+
grant_type: 'authorization_code',
|
|
50
|
+
code: received_params['code'],
|
|
51
|
+
redirect_uri: @redirect_uri
|
|
52
|
+
)
|
|
52
53
|
end
|
|
53
54
|
end
|
|
54
55
|
Factory.instance.register_token_creator(Web)
|
data/lib/aspera/oauth.rb
CHANGED
|
@@ -44,9 +44,7 @@ module Aspera
|
|
|
44
44
|
Aspera.assert_type(value, String)
|
|
45
45
|
persist_filepath = id_to_filepath(object_id)
|
|
46
46
|
Log.log.debug { "persistency saving: #{persist_filepath}" }
|
|
47
|
-
|
|
48
|
-
File.write(persist_filepath, value)
|
|
49
|
-
Environment.restrict_file_access(persist_filepath)
|
|
47
|
+
Environment.write_file_restricted(persist_filepath, force: true) { value }
|
|
50
48
|
@cache[object_id] = value
|
|
51
49
|
nil
|
|
52
50
|
end
|
|
@@ -57,11 +57,11 @@ module Aspera
|
|
|
57
57
|
|
|
58
58
|
private_constant :SUPPORTED_MIME_TYPES
|
|
59
59
|
|
|
60
|
-
# @attr
|
|
61
|
-
attr_accessor :
|
|
60
|
+
# @attr detect_mime [Boolean] `true` to determine real mime type based on file content
|
|
61
|
+
attr_accessor :detect_mime
|
|
62
62
|
|
|
63
63
|
def initialize
|
|
64
|
-
@
|
|
64
|
+
@detect_mime = false
|
|
65
65
|
end
|
|
66
66
|
|
|
67
67
|
# @param mimetype [String] mime type
|
|
@@ -81,7 +81,7 @@ module Aspera
|
|
|
81
81
|
# @return [Symbol] file type, one of enum CONVERSION_TYPES
|
|
82
82
|
# @raise [RuntimeError] if no conversion type found
|
|
83
83
|
def conversion_type(filepath, mimetype)
|
|
84
|
-
Log.log.debug { "conversion_type(#{filepath},mime=#{mimetype},magic=#{@
|
|
84
|
+
Log.log.debug { "conversion_type(#{filepath},mime=#{mimetype},magic=#{@detect_mime})" }
|
|
85
85
|
# Default type or empty means no type
|
|
86
86
|
mimetype = TYPE_NOT_FOUND if mimetype.nil? || (mimetype.is_a?(String) && mimetype.empty?)
|
|
87
87
|
mimetype = Marcel::MimeType.for(Pathname.new(filepath), name: File.basename(filepath), declared_type: mimetype)
|
|
@@ -4,9 +4,9 @@
|
|
|
4
4
|
# spellchecker:ignore soffice pauseframes libx264 trunc bufsize muxer apng libmp3lame maxrate posterize movflags faststart
|
|
5
5
|
# spellchecker:ignore palettegen paletteuse pointsize bordercolor repage lanczos unoconv optipng reencode conv transframes
|
|
6
6
|
|
|
7
|
-
require 'aspera/preview/options'
|
|
8
7
|
require 'aspera/preview/utils'
|
|
9
8
|
require 'aspera/preview/file_types'
|
|
9
|
+
require 'aspera/schema/registry'
|
|
10
10
|
require 'aspera/log'
|
|
11
11
|
require 'aspera/assert'
|
|
12
12
|
|
|
@@ -20,6 +20,9 @@ module Aspera
|
|
|
20
20
|
# List of valid ffmpeg option keys for reencode configuration.
|
|
21
21
|
FFMPEG_OPTIONS_LIST = %w[in out].freeze
|
|
22
22
|
|
|
23
|
+
# Generator options, members defined in `options.schema.yaml`.
|
|
24
|
+
Options = Struct.new(*Schema::Registry.instance.reader(Schema::Registry::PREVIEW_OPTIONS).current['properties'].keys.map(&:to_sym))
|
|
25
|
+
|
|
23
26
|
# CLI needs to know conversion type to know if need skip it.
|
|
24
27
|
# One of CONVERSION_TYPES.
|
|
25
28
|
attr_reader :conversion_type, :destination
|
|
@@ -251,6 +254,7 @@ module Aspera
|
|
|
251
254
|
tmp_pdf_file = File.join(this_tmpdir, "#{File.basename(@source, File.extname(@source))}.pdf")
|
|
252
255
|
case @options.office_conversion
|
|
253
256
|
when :unoconv
|
|
257
|
+
# `unoconv` is a python wrapper over `soffice`
|
|
254
258
|
Utils.silent_execute(
|
|
255
259
|
:unoconv,
|
|
256
260
|
'-f', 'pdf',
|
|
@@ -261,6 +265,12 @@ module Aspera
|
|
|
261
265
|
Utils.silent_execute(
|
|
262
266
|
:soffice,
|
|
263
267
|
'--headless',
|
|
268
|
+
'--invisible',
|
|
269
|
+
'--nocrashreport',
|
|
270
|
+
'--nodefault',
|
|
271
|
+
'--nofirststartwizard',
|
|
272
|
+
'--nologo',
|
|
273
|
+
'--norestore',
|
|
264
274
|
'--convert-to', 'pdf',
|
|
265
275
|
'--outdir', File.dirname(tmp_pdf_file),
|
|
266
276
|
@source
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
$schema: https://json-schema.org/draft/2020-12/schema
|
|
2
|
+
$id: https://github.com/IBM/aspera-cli/tree/main/lib/aspera/preview/options.schema.yaml
|
|
3
|
+
title: Preview generator options
|
|
4
|
+
description: >-
|
|
5
|
+
Options of the preview generator: tools and parameters used to generate PNG thumbnails and MP4 video previews.
|
|
6
|
+
$comment: >-
|
|
7
|
+
Used to declare the options of the `preview` plugin: values of `enum` properties are `Symbol`.
|
|
8
|
+
Scaling uses `ffmpeg` scale filter arguments, see: https://trac.ffmpeg.org/wiki/Scaling
|
|
9
|
+
(`iw`/`ih`: input width/height, `-x`: keep aspect ratio, with a value multiple of `x`).
|
|
10
|
+
type: object
|
|
11
|
+
additionalProperties: false
|
|
12
|
+
properties:
|
|
13
|
+
office_conversion:
|
|
14
|
+
description: 'PNG thumbnail of office document: conversion tool.'
|
|
15
|
+
type: string
|
|
16
|
+
enum: [soffice, unoconv]
|
|
17
|
+
default: soffice
|
|
18
|
+
max_size:
|
|
19
|
+
description: 'Maximum size (in bytes) of a preview file: a warning is logged if exceeded.'
|
|
20
|
+
type: integer
|
|
21
|
+
minimum: 1
|
|
22
|
+
default: 16777216
|
|
23
|
+
thumb_vid_scale:
|
|
24
|
+
description: 'PNG thumbnail of video: frame size, as `ffmpeg` scale filter argument.'
|
|
25
|
+
type: string
|
|
26
|
+
default: '-1:min(ih,100)'
|
|
27
|
+
thumb_vid_fraction:
|
|
28
|
+
description: 'PNG thumbnail of video (`fixed`): position of the snapshot, as a fraction of the video duration.'
|
|
29
|
+
type: number
|
|
30
|
+
minimum: 0
|
|
31
|
+
maximum: 1
|
|
32
|
+
default: 0.1
|
|
33
|
+
thumb_img_size:
|
|
34
|
+
description: 'PNG thumbnail of image, PDF, office document or text: size (in pixels).'
|
|
35
|
+
type: integer
|
|
36
|
+
minimum: 1
|
|
37
|
+
default: 800
|
|
38
|
+
thumb_text_font:
|
|
39
|
+
description: 'PNG thumbnail of text: font name, as listed by `magick identify -list font`.'
|
|
40
|
+
type: string
|
|
41
|
+
default: Courier
|
|
42
|
+
video_conversion:
|
|
43
|
+
description: >-
|
|
44
|
+
MP4 video preview: generation method.
|
|
45
|
+
`reencode`: re-encode the beginning of the video,
|
|
46
|
+
`blend`: key frames with transitions,
|
|
47
|
+
`clips`: concatenation of short clips.
|
|
48
|
+
type: string
|
|
49
|
+
enum: [reencode, blend, clips]
|
|
50
|
+
default: reencode
|
|
51
|
+
video_png_conv:
|
|
52
|
+
description: >-
|
|
53
|
+
PNG thumbnail of video: generation method.
|
|
54
|
+
`fixed`: single frame,
|
|
55
|
+
`animated`: animated PNG.
|
|
56
|
+
type: string
|
|
57
|
+
enum: [fixed, animated]
|
|
58
|
+
default: fixed
|
|
59
|
+
video_scale:
|
|
60
|
+
description: 'MP4 video preview: frame size, as `ffmpeg` scale filter argument.'
|
|
61
|
+
type: string
|
|
62
|
+
default: 'min(iw,360):-2'
|
|
63
|
+
video_start_sec:
|
|
64
|
+
description: 'Video: start offset (in seconds) of MP4 video preview and of `animated` PNG thumbnail.'
|
|
65
|
+
type: integer
|
|
66
|
+
minimum: 0
|
|
67
|
+
default: 10
|
|
68
|
+
video_codec:
|
|
69
|
+
description: >-
|
|
70
|
+
MP4 video preview (`reencode`): `ffmpeg` video codec, e.g. `libx264`, `h264_videotoolbox`, `h264_nvenc`.
|
|
71
|
+
Default: first available H.264 encoder.
|
|
72
|
+
type: string
|
|
73
|
+
reencode_ffmpeg:
|
|
74
|
+
description: >-
|
|
75
|
+
MP4 video preview (`reencode`): `ffmpeg` options replacing the default ones.
|
|
76
|
+
type: object
|
|
77
|
+
additionalProperties: false
|
|
78
|
+
properties:
|
|
79
|
+
in:
|
|
80
|
+
description: Input options.
|
|
81
|
+
type: array
|
|
82
|
+
items:
|
|
83
|
+
type: [string, number]
|
|
84
|
+
out:
|
|
85
|
+
description: Output options.
|
|
86
|
+
type: array
|
|
87
|
+
items:
|
|
88
|
+
type: [string, number]
|
|
89
|
+
default: {}
|
|
90
|
+
blend_keyframes:
|
|
91
|
+
description: 'MP4 video preview (`blend`): number of key frames.'
|
|
92
|
+
type: integer
|
|
93
|
+
minimum: 1
|
|
94
|
+
default: 30
|
|
95
|
+
blend_pauseframes:
|
|
96
|
+
description: 'MP4 video preview (`blend`): number of pause frames (repetitions of each key frame).'
|
|
97
|
+
type: integer
|
|
98
|
+
minimum: 0
|
|
99
|
+
default: 3
|
|
100
|
+
blend_transframes:
|
|
101
|
+
description: 'MP4 video preview (`blend`): number of transition frames between key frames.'
|
|
102
|
+
type: integer
|
|
103
|
+
minimum: 0
|
|
104
|
+
default: 5
|
|
105
|
+
blend_fps:
|
|
106
|
+
description: 'MP4 video preview (`blend`): frames per second.'
|
|
107
|
+
type: integer
|
|
108
|
+
minimum: 1
|
|
109
|
+
default: 15
|
|
110
|
+
clips_count:
|
|
111
|
+
description: 'MP4 video preview (`clips`): number of clips.'
|
|
112
|
+
type: integer
|
|
113
|
+
minimum: 1
|
|
114
|
+
default: 5
|
|
115
|
+
clips_length:
|
|
116
|
+
description: 'Video: length (in seconds) of each clip of MP4 video preview (`clips`), and of `animated` PNG thumbnail.'
|
|
117
|
+
type: integer
|
|
118
|
+
minimum: 1
|
|
119
|
+
default: 5
|
|
@@ -2,11 +2,12 @@
|
|
|
2
2
|
|
|
3
3
|
# cspell:words Magick MAGICKCORE ITERM mintty winsize termcap
|
|
4
4
|
|
|
5
|
-
require 'rainbow'
|
|
6
5
|
require 'base64'
|
|
7
6
|
require 'io/console'
|
|
8
7
|
require 'aspera/log'
|
|
9
8
|
require 'aspera/environment'
|
|
9
|
+
require 'aspera/rainbow'
|
|
10
|
+
using Rainbow
|
|
10
11
|
|
|
11
12
|
module Aspera
|
|
12
13
|
module Preview
|
|
@@ -155,9 +156,9 @@ module Aspera
|
|
|
155
156
|
row_data.each_with_index do |pixel_rgb, col|
|
|
156
157
|
text_pixels.push("\n") if col.eql?(0) && !row.eql?(0)
|
|
157
158
|
if double
|
|
158
|
-
text_pixels.push(
|
|
159
|
+
text_pixels.push('▄'.bg(pixel_rgb).fg(pixel_colors[row + 1][col]))
|
|
159
160
|
else
|
|
160
|
-
text_pixels.push(
|
|
161
|
+
text_pixels.push(' '.bg(pixel_rgb))
|
|
161
162
|
end
|
|
162
163
|
end
|
|
163
164
|
end
|
data/lib/aspera/preview/utils.rb
CHANGED
|
@@ -25,6 +25,9 @@ module Aspera
|
|
|
25
25
|
private_constant :EXTERNAL_TOOLS, :TEMP_FORMAT, :FFMPEG_DEFAULT_PARAMS, :H264_ENCODER_PREFERENCE
|
|
26
26
|
|
|
27
27
|
class << self
|
|
28
|
+
# @param tool [Symbol] either `unoconv` or `soffice`
|
|
29
|
+
attr_accessor :office_tool
|
|
30
|
+
|
|
28
31
|
# Return the first H.264 encoder available in the local ffmpeg installation.
|
|
29
32
|
# Result is memoized after the first call.
|
|
30
33
|
# @return [String] encoder name (e.g. 'libx264', 'libopenh264')
|
|
@@ -32,8 +35,10 @@ module Aspera
|
|
|
32
35
|
def available_h264_encoder
|
|
33
36
|
return @available_h264_encoder if defined?(@available_h264_encoder)
|
|
34
37
|
stdout, = execute(:ffmpeg, '-encoders', mode: :capture, exception: false)
|
|
35
|
-
|
|
36
|
-
|
|
38
|
+
available = stdout.lines.grep(/h264/i).map { |l| l.split[1] }
|
|
39
|
+
@available_h264_encoder = H264_ENCODER_PREFERENCE.find { |enc| available.include?(enc) }
|
|
40
|
+
Aspera.assert(@available_h264_encoder) { "No supported H.264 encoder found in ffmpeg. Available: #{available.join(', ')}" }
|
|
41
|
+
@available_h264_encoder
|
|
37
42
|
end
|
|
38
43
|
|
|
39
44
|
# Check that external tools can be executed.
|
|
@@ -42,10 +47,8 @@ module Aspera
|
|
|
42
47
|
# @return [nil]
|
|
43
48
|
def check_tools(skip_types = [])
|
|
44
49
|
tools_to_check = EXTERNAL_TOOLS.dup
|
|
45
|
-
if skip_types.include?(:office)
|
|
46
|
-
|
|
47
|
-
tools_to_check.delete(:soffice)
|
|
48
|
-
end
|
|
50
|
+
tools_to_check.delete(:unoconv) if skip_types.include?(:office) || office_tool.eql?(:soffice)
|
|
51
|
+
tools_to_check.delete(:soffice) if skip_types.include?(:office) || office_tool.eql?(:unoconv)
|
|
49
52
|
# Check for binaries
|
|
50
53
|
tools_to_check.each do |command_sym|
|
|
51
54
|
silent_execute(command_sym, '-h')
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'aspera/rest/error_analyzer'
|
|
4
|
+
require 'aspera/log'
|
|
5
|
+
require 'xmlsimple'
|
|
6
|
+
|
|
7
|
+
module Aspera
|
|
8
|
+
module Rest
|
|
9
|
+
# REST error handlers for various Aspera REST APIs
|
|
10
|
+
class AsperaErrors
|
|
11
|
+
class << self
|
|
12
|
+
# handlers should probably be defined by plugins for modularity
|
|
13
|
+
def register_handlers
|
|
14
|
+
Log.log.debug('registering Aspera REST error handlers')
|
|
15
|
+
# Faspex 4: both user_message and internal_message, and code 200
|
|
16
|
+
# example: missing meta data on package creation
|
|
17
|
+
ErrorAnalyzer.instance.add_simple_handler(name: 'Type 1: error:user_message', path: %w[error user_message], always: true)
|
|
18
|
+
ErrorAnalyzer.instance.add_simple_handler(name: 'Type 2: error:description', path: %w[error description])
|
|
19
|
+
ErrorAnalyzer.instance.add_simple_handler(name: 'Type 3: error:internal_message', path: %w[error internal_message])
|
|
20
|
+
ErrorAnalyzer.instance.add_simple_handler(name: 'Type 5', path: ['error_description'])
|
|
21
|
+
ErrorAnalyzer.instance.add_simple_handler(name: 'Type 6', path: ['message'])
|
|
22
|
+
ErrorAnalyzer.instance.add_simple_handler(name: 'Type 6', path: ['failure'], always: true)
|
|
23
|
+
# AoC Automation
|
|
24
|
+
ErrorAnalyzer.instance.add_simple_handler(name: 'AoC Automation', path: ['error'])
|
|
25
|
+
ErrorAnalyzer.instance.add_handler('Type 7: errors[]') do |type, context|
|
|
26
|
+
next unless context[:data].is_a?(Hash) && context[:data]['errors'].is_a?(Hash)
|
|
27
|
+
# special for Shares: false positive ? (update global transfer_settings)
|
|
28
|
+
next if context[:data].key?('min_connect_version')
|
|
29
|
+
context[:data]['errors'].each do |k, v|
|
|
30
|
+
ErrorAnalyzer.add_error(context, type, "#{k}: #{v}")
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
# call to upload_setup and download_setup of node api
|
|
34
|
+
ErrorAnalyzer.instance.add_handler('T8:node: *_setup') do |type, context|
|
|
35
|
+
next unless context[:data].is_a?(Hash)
|
|
36
|
+
d_t_s = context[:data]['transfer_specs']
|
|
37
|
+
next unless d_t_s.is_a?(Array)
|
|
38
|
+
d_t_s.each do |res|
|
|
39
|
+
r_err = res.dig(*%w[transfer_spec error]) || res['error']
|
|
40
|
+
next unless r_err.is_a?(Hash)
|
|
41
|
+
ErrorAnalyzer.add_error(context, type, r_err.values.join(': '))
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
ErrorAnalyzer.instance.add_simple_handler(name: 'T9:IBM cloud IAM', path: ['errorMessage'])
|
|
45
|
+
ErrorAnalyzer.instance.add_simple_handler(name: 'T10:faspex v4', path: ['user_message'])
|
|
46
|
+
ErrorAnalyzer.instance.add_handler('Orchestrator') do |type, context|
|
|
47
|
+
next if context[:response].code.start_with?('2')
|
|
48
|
+
data = context[:data]
|
|
49
|
+
if data.is_a?(String) && data.start_with?('<?xml')
|
|
50
|
+
parsed = XmlSimple.xml_in(data, {'ForceArray' => false}) rescue nil
|
|
51
|
+
if parsed.is_a?(Hash) && parsed['content'].is_a?(String)
|
|
52
|
+
ErrorAnalyzer.add_error(context, type, parsed['content'].gsub(/\A"|"\z/, ''))
|
|
53
|
+
parsed.each do |k, v|
|
|
54
|
+
next if k.eql?('content')
|
|
55
|
+
ErrorAnalyzer.add_error(context, "#{type}(sub)", "#{k}: #{v}") if [String, Integer].include?(v.class)
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
next
|
|
59
|
+
end
|
|
60
|
+
next unless data.is_a?(Hash)
|
|
61
|
+
work_order = data['work_order']
|
|
62
|
+
next unless work_order.is_a?(Hash)
|
|
63
|
+
ErrorAnalyzer.add_error(context, type, work_order['statusDetails'])
|
|
64
|
+
data['missing_parameters']&.each do |param|
|
|
65
|
+
ErrorAnalyzer.add_error(context, type, "missing parameter: #{param}")
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
end
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Aspera
|
|
4
|
+
module Rest
|
|
5
|
+
# raised on error after REST call
|
|
6
|
+
class CallError < StandardError
|
|
7
|
+
def request
|
|
8
|
+
@context[:request]
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
def response
|
|
12
|
+
@context[:response]
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def data
|
|
16
|
+
@context[:data]
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
# @param context [Hash,String] with keys :messages, :request, :response, :data
|
|
20
|
+
def initialize(context)
|
|
21
|
+
context = {messages: [context]} if context.is_a?(String)
|
|
22
|
+
@context = context
|
|
23
|
+
super(@context[:messages].join("\n"))
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
end
|