aspera-cli 4.26.1 → 4.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +300 -22
- data/CONTRIBUTING.md +100 -12
- data/bin/ascli +5 -5
- data/docs/README.md +10757 -0
- data/docs/test-mcp-with-ai.md +202 -0
- data/lib/aspera/agent/base.rb +20 -6
- data/lib/aspera/agent/connect.rb +53 -16
- data/lib/aspera/agent/desktop.rb +52 -17
- data/lib/aspera/agent/direct.rb +27 -25
- data/lib/aspera/agent/factory.rb +4 -0
- data/lib/aspera/agent/httpgw.rb +2 -2
- data/lib/aspera/agent/node.rb +34 -2
- data/lib/aspera/agent/transferd.rb +39 -3
- data/lib/aspera/api/alee.rb +1 -1
- data/lib/aspera/api/aoc.rb +56 -51
- data/lib/aspera/api/cos_node.rb +2 -2
- data/lib/aspera/api/faspex.rb +22 -20
- data/lib/aspera/api/httpgw.rb +4 -4
- data/lib/aspera/api/node.rb +33 -30
- data/lib/aspera/api/queries/bss_subscription_account.graphql +18 -0
- data/lib/aspera/api/queries/bss_subscription_usage.graphql +18 -0
- data/lib/aspera/ascmd.rb +25 -23
- data/lib/aspera/ascp/installation.rb +43 -40
- data/lib/aspera/ascp/management.rb +2 -3
- data/lib/aspera/cli/ascp_actions.rb +155 -0
- data/lib/aspera/cli/async_transfer_store.rb +81 -0
- data/lib/aspera/cli/bootstrapper.rb +197 -0
- data/lib/aspera/cli/command_registry.rb +145 -0
- data/lib/aspera/cli/command_spec.rb +138 -0
- data/lib/aspera/cli/context.rb +71 -0
- data/lib/aspera/cli/error.rb +16 -0
- data/lib/aspera/cli/extended_value.rb +5 -3
- data/lib/aspera/cli/formatter.rb +122 -209
- data/lib/aspera/cli/gem_checker.rb +65 -0
- data/lib/aspera/cli/hints.rb +2 -2
- data/lib/aspera/cli/http.rb +218 -0
- data/lib/aspera/cli/info.rb +2 -0
- data/lib/aspera/cli/mailer.rb +97 -0
- data/lib/aspera/cli/mcp_tool.rb +198 -0
- data/lib/aspera/cli/options.schema.yaml +364 -7
- data/lib/aspera/cli/{manager.rb → parser.rb} +337 -144
- data/lib/aspera/cli/plugins/alee.rb +20 -22
- data/lib/aspera/cli/plugins/aoc.rb +1173 -849
- data/lib/aspera/cli/plugins/ats.rb +200 -161
- data/lib/aspera/cli/plugins/base.rb +470 -107
- data/lib/aspera/cli/plugins/basic_auth.rb +14 -4
- data/lib/aspera/cli/plugins/config.rb +434 -1032
- data/lib/aspera/cli/plugins/console.rb +106 -64
- data/lib/aspera/cli/plugins/cos.rb +44 -32
- data/lib/aspera/cli/plugins/factory.rb +7 -4
- data/lib/aspera/cli/plugins/faspex.rb +296 -259
- data/lib/aspera/cli/plugins/faspex5.rb +592 -344
- data/lib/aspera/cli/plugins/faspio.rb +49 -51
- data/lib/aspera/cli/plugins/httpgw.rb +18 -25
- data/lib/aspera/cli/plugins/mcp.rb +279 -0
- data/lib/aspera/cli/plugins/node.rb +1001 -797
- data/lib/aspera/cli/plugins/oauth.rb +7 -10
- data/lib/aspera/cli/plugins/orchestrator.rb +111 -139
- data/lib/aspera/cli/plugins/preview.rb +183 -142
- data/lib/aspera/cli/plugins/server.rb +125 -80
- data/lib/aspera/cli/plugins/shares.rb +301 -107
- data/lib/aspera/cli/preset_actions.rb +139 -0
- data/lib/aspera/cli/preset_manager.rb +236 -0
- data/lib/aspera/cli/result.rb +360 -0
- data/lib/aspera/cli/runner.rb +473 -0
- data/lib/aspera/cli/secret_finder.rb +40 -0
- data/lib/aspera/cli/special_values.rb +1 -0
- data/lib/aspera/cli/sync_actions.rb +84 -56
- data/lib/aspera/cli/terminal_formatter.rb +65 -0
- data/lib/aspera/cli/transfer_actions.rb +83 -0
- data/lib/aspera/cli/transfer_agent.rb +116 -51
- data/lib/aspera/cli/transfer_progress.rb +9 -9
- data/lib/aspera/cli/vault_manager.rb +57 -0
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/cli/wizard.rb +26 -26
- data/lib/aspera/command_line_builder.rb +23 -23
- data/lib/aspera/data_repository.rb +6 -1
- data/lib/aspera/dot_container.rb +5 -5
- data/lib/aspera/environment.rb +39 -13
- data/lib/aspera/exec_spec.rb +13 -0
- data/lib/aspera/faspex_gw.rb +2 -2
- data/lib/aspera/faspex_postproc.rb +4 -3
- data/lib/aspera/graphql.rb +35 -0
- data/lib/aspera/hash_ext.rb +6 -0
- data/lib/aspera/json_rpc/client.rb +62 -0
- data/lib/aspera/json_rpc/version.rb +7 -0
- data/lib/aspera/keychain/base.rb +1 -1
- data/lib/aspera/keychain/encrypted_hash.rb +2 -2
- data/lib/aspera/keychain/factory.rb +4 -4
- data/lib/aspera/keychain/macos_security.rb +4 -4
- data/lib/aspera/link_header.rb +82 -0
- data/lib/aspera/log.rb +23 -5
- data/lib/aspera/markdown.rb +85 -2
- data/lib/aspera/nagios.rb +2 -2
- data/lib/aspera/node_simulator.rb +3 -1
- data/lib/aspera/oauth/base.rb +10 -10
- data/lib/aspera/oauth/boot.rb +43 -0
- data/lib/aspera/oauth/factory.rb +41 -15
- data/lib/aspera/oauth/jwt.rb +4 -4
- data/lib/aspera/oauth/web.rb +4 -4
- data/lib/aspera/oauth.rb +1 -0
- data/lib/aspera/persistency_action_once.rb +10 -10
- data/lib/aspera/persistency_folder.rb +10 -3
- data/lib/aspera/preview/file_types.rb +7 -3
- data/lib/aspera/preview/generator.rb +3 -3
- data/lib/aspera/preview/utils.rb +1 -1
- data/lib/aspera/products/connect.rb +4 -1
- data/lib/aspera/products/other.rb +1 -1
- data/lib/aspera/products/transferd.rb +3 -3
- data/lib/aspera/proxy_auto_config.rb +10 -9
- data/lib/aspera/rest.rb +18 -40
- data/lib/aspera/rest_error_analyzer.rb +7 -3
- data/lib/aspera/rest_errors_aspera.rb +0 -10
- data/lib/aspera/rest_list.rb +13 -6
- data/lib/aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml +14 -4
- data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +2730 -648
- data/lib/aspera/schema/async_tables.yaml +361 -0
- data/lib/aspera/schema/documentation.rb +57 -26
- data/lib/aspera/schema/reader.rb +60 -11
- data/lib/aspera/schema/registry.rb +50 -10
- data/lib/aspera/secret_hider.rb +11 -2
- data/lib/aspera/ssl.rb +4 -4
- data/lib/aspera/sync/conf.schema.yaml +2 -2
- data/lib/aspera/sync/database.rb +14 -10
- data/lib/aspera/sync/operations.rb +15 -18
- data/lib/aspera/temp_file_manager.rb +5 -1
- data/lib/aspera/timer_limiter.rb +1 -1
- data/lib/aspera/transfer/faux_file.rb +24 -11
- data/lib/aspera/transfer/parameters.rb +27 -25
- data/lib/aspera/transfer/result.rb +74 -0
- data/lib/aspera/transfer/resumer.rb +7 -7
- data/lib/aspera/transfer/spec.rb +18 -1
- data/lib/aspera/transfer/spec.schema.yaml +26 -8
- data/lib/aspera/transfer/uri.rb +1 -1
- data/lib/aspera/uri_reader.rb +2 -2
- data/lib/aspera/web_auth.rb +2 -2
- data/lib/aspera/web_server_simple.rb +8 -5
- data/lib/aspera/yaml.rb +2 -1
- data.tar.gz.sig +0 -0
- metadata +45 -16
- metadata.gz.sig +0 -0
- data/lib/aspera/cli/main.rb +0 -480
- data/lib/aspera/json_rpc.rb +0 -51
data/lib/aspera/oauth/factory.rb
CHANGED
|
@@ -7,10 +7,14 @@ require 'base64'
|
|
|
7
7
|
module Aspera
|
|
8
8
|
module OAuth
|
|
9
9
|
# Factory to create tokens and manage their cache
|
|
10
|
+
#
|
|
11
|
+
# @!method self.instance
|
|
12
|
+
# Returns the singleton instance of Factory
|
|
13
|
+
# @return [Factory] the singleton instance
|
|
10
14
|
class Factory
|
|
11
15
|
include Singleton
|
|
12
16
|
|
|
13
|
-
#
|
|
17
|
+
# prefix for persistency of tokens (simplify garbage collect)
|
|
14
18
|
PERSIST_CATEGORY_TOKEN = 'token'
|
|
15
19
|
# prefix for bearer authorization when in header
|
|
16
20
|
SPACE_BEARER_AUTH_SCHEME = 'Bearer '
|
|
@@ -19,18 +23,23 @@ module Aspera
|
|
|
19
23
|
private_constant :PERSIST_CATEGORY_TOKEN, :SPACE_BEARER_AUTH_SCHEME
|
|
20
24
|
|
|
21
25
|
class << self
|
|
26
|
+
# Format a token for use in Authorization header
|
|
22
27
|
# @param token [String] The token alone
|
|
23
28
|
# @return [String] Value suitable for Authorization header
|
|
24
29
|
def bearer_authorization(token)
|
|
25
30
|
return "#{SPACE_BEARER_AUTH_SCHEME}#{token}"
|
|
26
31
|
end
|
|
27
32
|
|
|
28
|
-
#
|
|
33
|
+
# Check if the authorization contains a bearer token
|
|
34
|
+
# @param authorization [String] The authorization header value
|
|
35
|
+
# @return [Boolean] true if the authorization contains a bearer token, i.e. auth scheme is bearer
|
|
29
36
|
def bearer_auth?(authorization)
|
|
30
37
|
return authorization.start_with?(SPACE_BEARER_AUTH_SCHEME)
|
|
31
38
|
end
|
|
32
39
|
|
|
33
40
|
# Extract only token from Authorization (remove scheme)
|
|
41
|
+
# @param authorization [String] The authorization header value
|
|
42
|
+
# @return [String] The bearer token without the scheme prefix
|
|
34
43
|
def bearer_token(authorization)
|
|
35
44
|
Aspera.assert(bearer_auth?(authorization)){'not a bearer token, wrong prefix scheme'}
|
|
36
45
|
return authorization.delete_prefix(SPACE_BEARER_AUTH_SCHEME)
|
|
@@ -40,12 +49,14 @@ module Aspera
|
|
|
40
49
|
# @param url [String] Base URL of the OAuth server
|
|
41
50
|
# @param creator_class [Class] Class of the token creator
|
|
42
51
|
# @param params [Array] List of parameters (can be nested) to uniquely identify the token
|
|
43
|
-
# @return a unique cache identifier
|
|
52
|
+
# @return [String] a unique cache identifier
|
|
44
53
|
def cache_id(url, creator_class, *params)
|
|
45
54
|
return IdGenerator.from_list(PERSIST_CATEGORY_TOKEN, url, Factory.class_to_id(creator_class), params)
|
|
46
55
|
end
|
|
47
56
|
|
|
48
|
-
#
|
|
57
|
+
# Convert a class name to snake_case symbol
|
|
58
|
+
# @param creator_class [Class] The class to convert
|
|
59
|
+
# @return [Symbol] snake_case version of class name
|
|
49
60
|
def class_to_id(creator_class)
|
|
50
61
|
return creator_class.name.split('::').last.capital_to_snake.to_sym
|
|
51
62
|
end
|
|
@@ -53,6 +64,7 @@ module Aspera
|
|
|
53
64
|
|
|
54
65
|
private
|
|
55
66
|
|
|
67
|
+
# Initialize the factory with default parameters and empty collections
|
|
56
68
|
def initialize
|
|
57
69
|
# persistency manager
|
|
58
70
|
@persist = nil
|
|
@@ -73,12 +85,16 @@ module Aspera
|
|
|
73
85
|
|
|
74
86
|
attr_reader :parameters
|
|
75
87
|
|
|
88
|
+
# Set the persistence manager for token caching
|
|
89
|
+
# @param manager [Object] The persistence manager instance
|
|
76
90
|
def persist_mgr=(manager)
|
|
77
91
|
@persist = manager
|
|
78
92
|
# cleanup expired tokens
|
|
79
93
|
@persist.garbage_collect(PERSIST_CATEGORY_TOKEN, @parameters[:token_cache_max_age])
|
|
80
94
|
end
|
|
81
95
|
|
|
96
|
+
# Get or initialize the persistence manager
|
|
97
|
+
# @return [Object] The persistence manager instance
|
|
82
98
|
def persist_mgr
|
|
83
99
|
if @persist.nil?
|
|
84
100
|
# use OAuth::Factory.instance.persist_mgr=PersistencyFolder.new)
|
|
@@ -91,16 +107,19 @@ module Aspera
|
|
|
91
107
|
return @persist
|
|
92
108
|
end
|
|
93
109
|
|
|
94
|
-
#
|
|
110
|
+
# Delete all existing tokens in cache
|
|
111
|
+
# @return [Array<String>] List of deleted file paths
|
|
95
112
|
def flush_tokens
|
|
96
113
|
persist_mgr.garbage_collect(PERSIST_CATEGORY_TOKEN)
|
|
97
114
|
end
|
|
98
115
|
|
|
116
|
+
# Retrieve all persisted tokens with their decoded information
|
|
117
|
+
# @return [Array<Hash>] Array of token information hashes
|
|
99
118
|
def persisted_tokens
|
|
100
119
|
data = persist_mgr.current_items(PERSIST_CATEGORY_TOKEN)
|
|
101
120
|
data.each.map do |k, v|
|
|
102
121
|
info = {id: k}
|
|
103
|
-
info.merge!(JSON.parse(v)) rescue nil
|
|
122
|
+
begin; info.merge!(JSON.parse(v)); rescue StandardError; nil; end
|
|
104
123
|
d = decode_token(info.delete(TOKEN_FIELD))
|
|
105
124
|
info.merge(d) if d
|
|
106
125
|
info
|
|
@@ -134,32 +153,39 @@ module Aspera
|
|
|
134
153
|
return info
|
|
135
154
|
end
|
|
136
155
|
|
|
137
|
-
#
|
|
156
|
+
# Register a bearer token decoder for inspecting token properties
|
|
157
|
+
# @param method [Proc] The decoder lambda/proc to register
|
|
158
|
+
# @return [nil]
|
|
138
159
|
def register_decoder(method)
|
|
139
160
|
@decoders.push(method)
|
|
161
|
+
nil
|
|
140
162
|
end
|
|
141
163
|
|
|
142
|
-
#
|
|
164
|
+
# Decode a token using all registered decoders
|
|
165
|
+
# @param token [String] The token to decode
|
|
166
|
+
# @return [Hash, nil] Decoded token data or nil if no decoder succeeded
|
|
143
167
|
def decode_token(token)
|
|
144
168
|
@decoders.each do |decoder|
|
|
145
|
-
result = decoder.call(token) rescue nil
|
|
169
|
+
result = begin; decoder.call(token); rescue StandardError; nil; end
|
|
146
170
|
return result unless result.nil?
|
|
147
171
|
end
|
|
148
172
|
return
|
|
149
173
|
end
|
|
150
174
|
|
|
151
|
-
#
|
|
152
|
-
# @param
|
|
153
|
-
# @
|
|
154
|
-
# @param id_create called to generate unique id for token, for cache
|
|
175
|
+
# Register a token creation method
|
|
176
|
+
# @param creator_class [Class] The token creator class to register
|
|
177
|
+
# @return [nil]
|
|
155
178
|
def register_token_creator(creator_class)
|
|
156
179
|
Aspera.assert_type(creator_class, Class)
|
|
157
180
|
id = Factory.class_to_id(creator_class)
|
|
158
181
|
Log.log.debug{"registering creator for #{id}"}
|
|
159
182
|
@token_type_classes[id] = creator_class
|
|
183
|
+
nil
|
|
160
184
|
end
|
|
161
185
|
|
|
162
|
-
#
|
|
186
|
+
# Create a token creator instance for the specified grant method
|
|
187
|
+
# @param parameters [Hash] Parameters including :grant_method and creator-specific options
|
|
188
|
+
# @return [Object] An instance of the registered token creator class
|
|
163
189
|
def create(**parameters)
|
|
164
190
|
Aspera.assert_type(parameters, Hash)
|
|
165
191
|
id = parameters[:grant_method]
|
|
@@ -169,6 +195,6 @@ module Aspera
|
|
|
169
195
|
end
|
|
170
196
|
end
|
|
171
197
|
# JSON Web Signature (JWS) compact serialization: https://datatracker.ietf.org/doc/html/rfc7515
|
|
172
|
-
Factory.instance.register_decoder(lambda{ |token| parts = token.split('.'); Aspera.
|
|
198
|
+
Factory.instance.register_decoder(lambda{ |token| parts = token.split('.'); Aspera.assert_values(parts.length, [3]){'JWS token parts'}; JSON.parse(Base64.decode64(parts[1]))}) # rubocop:disable Style/Semicolon
|
|
173
199
|
end
|
|
174
200
|
end
|
data/lib/aspera/oauth/jwt.rb
CHANGED
|
@@ -26,9 +26,9 @@ module Aspera
|
|
|
26
26
|
end
|
|
27
27
|
DEFAULT_PRIV_KEY_LENGTH = 4096
|
|
28
28
|
GRANT_TYPE = 'urn:ietf:params:oauth:grant-type:jwt-bearer'
|
|
29
|
-
# @param private_key_obj private key object
|
|
30
|
-
# @param payload payload to be included in the JWT
|
|
31
|
-
# @param headers headers to be included in the JWT
|
|
29
|
+
# @param private_key_obj [OpenSSL::PKey::RSA] private key object
|
|
30
|
+
# @param payload [Hash] payload to be included in the JWT
|
|
31
|
+
# @param headers [Hash] headers to be included in the JWT
|
|
32
32
|
def initialize(
|
|
33
33
|
private_key_obj:,
|
|
34
34
|
payload:,
|
|
@@ -49,7 +49,7 @@ module Aspera
|
|
|
49
49
|
|
|
50
50
|
def create_token
|
|
51
51
|
require 'jwt'
|
|
52
|
-
seconds_since_epoch = Time.
|
|
52
|
+
seconds_since_epoch = Time.now.to_i
|
|
53
53
|
Log.log.debug{"seconds_since_epoch=#{seconds_since_epoch}"}
|
|
54
54
|
jwt_payload = {
|
|
55
55
|
exp: seconds_since_epoch + OAuth::Factory.instance.parameters[:jwt_expiry_offset_sec], # expiration time
|
data/lib/aspera/oauth/web.rb
CHANGED
|
@@ -11,8 +11,8 @@ module Aspera
|
|
|
11
11
|
class << self
|
|
12
12
|
attr_accessor :additional_info
|
|
13
13
|
end
|
|
14
|
-
# @param redirect_uri
|
|
15
|
-
# @param path_authorize
|
|
14
|
+
# @param redirect_uri [String] url to receive the code after auth (to be exchanged for token)
|
|
15
|
+
# @param path_authorize [String] path to login page on web app
|
|
16
16
|
def initialize(
|
|
17
17
|
redirect_uri:,
|
|
18
18
|
path_authorize: 'authorize',
|
|
@@ -22,8 +22,8 @@ module Aspera
|
|
|
22
22
|
@redirect_uri = redirect_uri
|
|
23
23
|
@path_authorize = path_authorize
|
|
24
24
|
uri = URI.parse(@redirect_uri)
|
|
25
|
-
Aspera.
|
|
26
|
-
Aspera.assert(!uri.port.nil
|
|
25
|
+
Aspera.assert_values(uri.scheme, %w[http https]){'redirect_uri scheme must be http or https'}
|
|
26
|
+
Aspera.assert(!uri.port.nil?, 'redirect_uri must have a port')
|
|
27
27
|
# TODO: we could check that host is localhost or local address, as we are going to listen locally
|
|
28
28
|
end
|
|
29
29
|
|
data/lib/aspera/oauth.rb
CHANGED
|
@@ -14,18 +14,18 @@ module Aspera
|
|
|
14
14
|
MANAGER_METHODS = %i[get put delete]
|
|
15
15
|
private_constant :DELETE_DEFAULT, :PARSE_DEFAULT, :FORMAT_DEFAULT, :MERGE_DEFAULT, :MANAGER_METHODS
|
|
16
16
|
|
|
17
|
-
# @param
|
|
18
|
-
# @param
|
|
19
|
-
# @param
|
|
20
|
-
# @param
|
|
21
|
-
# @param
|
|
22
|
-
# @param
|
|
23
|
-
# @param
|
|
17
|
+
# @param manager [Object] Mandatory Database (must respond to get/put/delete)
|
|
18
|
+
# @param data [Object] Mandatory object to persist, must be same object from begin to end (assume array by default)
|
|
19
|
+
# @param id [String] Mandatory identifier
|
|
20
|
+
# @param delete [Proc] Optional delete persistency condition
|
|
21
|
+
# @param parse [Proc] Optional parse method (default to JSON)
|
|
22
|
+
# @param format [Proc] Optional dump method (default to JSON)
|
|
23
|
+
# @param merge [Proc] Optional merge data from file to current data
|
|
24
24
|
def initialize(manager:, data:, id:, delete: DELETE_DEFAULT, parse: PARSE_DEFAULT, format: FORMAT_DEFAULT, merge: MERGE_DEFAULT)
|
|
25
25
|
Aspera.assert(MANAGER_METHODS.all?{ |i| manager.respond_to?(i)}){"Manager must answer to #{MANAGER_METHODS}"}
|
|
26
|
-
Aspera.assert(!data.nil
|
|
26
|
+
Aspera.assert(!data.nil?, 'data must not be nil')
|
|
27
27
|
Aspera.assert_type(id, String)
|
|
28
|
-
Aspera.assert(!id.empty
|
|
28
|
+
Aspera.assert(!id.empty?, 'id must not be empty')
|
|
29
29
|
Aspera.assert_type(delete, Proc)
|
|
30
30
|
Aspera.assert_type(parse, Proc)
|
|
31
31
|
Aspera.assert_type(format, Proc)
|
|
@@ -48,7 +48,7 @@ module Aspera
|
|
|
48
48
|
end
|
|
49
49
|
end
|
|
50
50
|
|
|
51
|
-
# @return internal persisted object, in order to modify its content
|
|
51
|
+
# @return [Array, Hash] internal persisted object, in order to modify its content
|
|
52
52
|
def data
|
|
53
53
|
return @persisted_object
|
|
54
54
|
end
|
|
@@ -19,7 +19,8 @@ module Aspera
|
|
|
19
19
|
end
|
|
20
20
|
|
|
21
21
|
# Get value of persisted item
|
|
22
|
-
# @
|
|
22
|
+
# @param object_id [String] Identifier of persisted item
|
|
23
|
+
# @return [String, nil] Persisted value, or nil if not found
|
|
23
24
|
def get(object_id)
|
|
24
25
|
Log.log.debug{"persistency get: #{object_id}"}
|
|
25
26
|
if @cache.key?(object_id)
|
|
@@ -59,7 +60,10 @@ module Aspera
|
|
|
59
60
|
@cache.delete(object_id)
|
|
60
61
|
end
|
|
61
62
|
|
|
62
|
-
# Delete persisted items
|
|
63
|
+
# Delete persisted items matching the given category, optionally older than a given age
|
|
64
|
+
# @param persist_category [String] Prefix used to identify files belonging to the category
|
|
65
|
+
# @param max_age_seconds [Integer, nil] If set, only delete files older than this many seconds
|
|
66
|
+
# @return [Array<String>] List of deleted file paths
|
|
63
67
|
def garbage_collect(persist_category, max_age_seconds = nil)
|
|
64
68
|
garbage_files = current_files(persist_category)
|
|
65
69
|
if !max_age_seconds.nil?
|
|
@@ -74,6 +78,9 @@ module Aspera
|
|
|
74
78
|
return garbage_files
|
|
75
79
|
end
|
|
76
80
|
|
|
81
|
+
# List all persisted files matching the given category prefix
|
|
82
|
+
# @param persist_category [String] Prefix identifying the category
|
|
83
|
+
# @return [Array<String>] List of matching file paths
|
|
77
84
|
def current_files(persist_category)
|
|
78
85
|
Dir[File.join(@folder, "#{persist_category}*#{FILE_SUFFIX}")]
|
|
79
86
|
end
|
|
@@ -84,7 +91,7 @@ module Aspera
|
|
|
84
91
|
|
|
85
92
|
private
|
|
86
93
|
|
|
87
|
-
# @param object_id String or
|
|
94
|
+
# @param object_id [String, Array] identifier or list of identifiers
|
|
88
95
|
def id_to_filepath(object_id)
|
|
89
96
|
Aspera.assert_type(object_id, String)
|
|
90
97
|
FileUtils.mkdir_p(@folder)
|
|
@@ -8,6 +8,10 @@ require 'marcel'
|
|
|
8
8
|
module Aspera
|
|
9
9
|
module Preview
|
|
10
10
|
# function conversion_type returns one of the types: CONVERSION_TYPES
|
|
11
|
+
#
|
|
12
|
+
# @!method self.instance
|
|
13
|
+
# Returns the singleton instance of FileTypes
|
|
14
|
+
# @return [FileTypes] the singleton instance
|
|
11
15
|
class FileTypes
|
|
12
16
|
include Singleton
|
|
13
17
|
|
|
@@ -74,7 +78,7 @@ module Aspera
|
|
|
74
78
|
|
|
75
79
|
# @param filepath [String] Full path to file
|
|
76
80
|
# @param mimetype [String] MIME typre provided by node API
|
|
77
|
-
# @return file type, one of enum CONVERSION_TYPES
|
|
81
|
+
# @return [Symbol] file type, one of enum CONVERSION_TYPES
|
|
78
82
|
# @raise [RuntimeError] if no conversion type found
|
|
79
83
|
def conversion_type(filepath, mimetype)
|
|
80
84
|
Log.log.debug{"conversion_type(#{filepath},mime=#{mimetype},magic=#{@use_mimemagic})"}
|
|
@@ -82,9 +86,9 @@ module Aspera
|
|
|
82
86
|
mimetype = TYPE_NOT_FOUND if mimetype.nil? || (mimetype.is_a?(String) && mimetype.empty?)
|
|
83
87
|
mimetype = Marcel::MimeType.for(Pathname.new(filepath), name: File.basename(filepath), declared_type: mimetype)
|
|
84
88
|
mimetype = 'text/plain' if mimetype.eql?(TYPE_NOT_FOUND) && ascii_text_file?(filepath)
|
|
85
|
-
|
|
89
|
+
Aspera.assert(!mimetype.eql?(TYPE_NOT_FOUND)){"no MIME type found for #{File.basename(filepath)}"}
|
|
86
90
|
conversion_type = mime_to_type(mimetype)
|
|
87
|
-
|
|
91
|
+
Aspera.assert(!conversion_type.nil?){"no conversion type found for #{File.basename(filepath)}"}
|
|
88
92
|
Log.log.trace1{"conversion_type(#{File.basename(filepath)}): #{conversion_type.class.name} [#{conversion_type}]"}
|
|
89
93
|
return conversion_type
|
|
90
94
|
end
|
|
@@ -27,10 +27,10 @@ module Aspera
|
|
|
27
27
|
# Node API MIME types are from: http://svn.apache.org/repos/asf/httpd/httpd/trunk/docs/conf/mime.types.
|
|
28
28
|
# The resulting preview file type is taken from destination file extension.
|
|
29
29
|
# Conversion methods are provided by private methods: convert_<conversion_type>_to_<preview_format>.
|
|
30
|
-
#
|
|
31
|
-
#
|
|
30
|
+
# conversion_type: one of FileTypes::CONVERSION_TYPES.
|
|
31
|
+
# preview_format: one of Generator::PREVIEW_FORMATS.
|
|
32
32
|
# The conversion video->mp4 is implemented in methods: convert_video_to_mp4_using_<video_conversion>.
|
|
33
|
-
#
|
|
33
|
+
# conversion method: one of Generator::VIDEO_CONVERSION_METHODS.
|
|
34
34
|
# @param src [String] Source file path.
|
|
35
35
|
# @param dst [String] Destination file path.
|
|
36
36
|
# @param options [Options] All conversion options.
|
data/lib/aspera/preview/utils.rb
CHANGED
|
@@ -5,6 +5,9 @@ require 'singleton'
|
|
|
5
5
|
|
|
6
6
|
module Aspera
|
|
7
7
|
module Products
|
|
8
|
+
# @!method self.instance
|
|
9
|
+
# Returns the singleton instance of Connect
|
|
10
|
+
# @return [Connect] the singleton instance
|
|
8
11
|
class Connect
|
|
9
12
|
include Singleton
|
|
10
13
|
|
|
@@ -62,7 +65,7 @@ module Aspera
|
|
|
62
65
|
Log.dump(:javascript, connect_versions_javascript)
|
|
63
66
|
# get javascript object only
|
|
64
67
|
found = connect_versions_javascript.match(/^.*? = (.*);/)
|
|
65
|
-
|
|
68
|
+
Aspera.assert(!found.nil?, type: Cli::Error){'Problem when getting connect versions from internet'}
|
|
66
69
|
all_data = JSON.parse(found[1])
|
|
67
70
|
@connect_versions = all_data['entries']
|
|
68
71
|
end
|
|
@@ -57,7 +57,7 @@ module Aspera
|
|
|
57
57
|
# Find installed products and provide paths for it.
|
|
58
58
|
# @param scan_locations [Array] Array of Hash with keys: expected, app_root, sub_bin, ascp_path, name, version
|
|
59
59
|
# @return [Array] of products found, with filled missing fields
|
|
60
|
-
# @raise
|
|
60
|
+
# @raise [RuntimeError] if no installed product found
|
|
61
61
|
def find(scan_locations)
|
|
62
62
|
product_names = []
|
|
63
63
|
found = scan_locations.select do |item|
|
|
@@ -27,9 +27,9 @@ module Aspera
|
|
|
27
27
|
sdk_directory
|
|
28
28
|
end
|
|
29
29
|
|
|
30
|
-
# @return the path to folder where SDK is or should be installed
|
|
30
|
+
# @return [String] the path to folder where SDK is or should be installed
|
|
31
31
|
def sdk_directory
|
|
32
|
-
Aspera.assert(!@sdk_dir.nil
|
|
32
|
+
Aspera.assert(!@sdk_dir.nil?, 'SDK path was not initialized')
|
|
33
33
|
@sdk_dir
|
|
34
34
|
end
|
|
35
35
|
|
|
@@ -52,7 +52,7 @@ module Aspera
|
|
|
52
52
|
end
|
|
53
53
|
end
|
|
54
54
|
end
|
|
55
|
-
Aspera.assert(!result.nil
|
|
55
|
+
Aspera.assert(!result.nil?, 'Port not found in daemon logs')
|
|
56
56
|
Log.log.debug{"Got port #{result} from log"}
|
|
57
57
|
return result
|
|
58
58
|
end
|
|
@@ -14,7 +14,7 @@ module URI
|
|
|
14
14
|
# @yieldparam url [String] The URL to find proxy for
|
|
15
15
|
# @yieldreturn [String, nil] Proxy URL or nil to fallback to original method
|
|
16
16
|
def register_proxy_finder
|
|
17
|
-
Aspera.assert(block_given
|
|
17
|
+
Aspera.assert(block_given?, 'block required for register_proxy_finder')
|
|
18
18
|
# overload the method in URI : call user's provided block and fallback to original method
|
|
19
19
|
define_method(:find_proxy){ |env_vars = ENV| yield(to_s) || find_proxy_orig(env_vars)}
|
|
20
20
|
end
|
|
@@ -44,7 +44,7 @@ module Aspera
|
|
|
44
44
|
context_ip = r_addr.to_s if r_addr.is_a?(Resolv::IPv4)
|
|
45
45
|
end
|
|
46
46
|
end
|
|
47
|
-
|
|
47
|
+
Aspera.assert(!context_ip.nil?){"DNS name not found: #{context_host}"}
|
|
48
48
|
# NOTE: Javascript code here with string inclusions
|
|
49
49
|
javascript = <<END_OF_JAVASCRIPT
|
|
50
50
|
function dnsResolve(host) {
|
|
@@ -63,7 +63,7 @@ END_OF_JAVASCRIPT
|
|
|
63
63
|
|
|
64
64
|
attr_writer :proxy_user, :proxy_pass
|
|
65
65
|
|
|
66
|
-
# @param proxy_auto_config the proxy auto config script to be evaluated
|
|
66
|
+
# @param proxy_auto_config [String] the proxy auto config script to be evaluated
|
|
67
67
|
def initialize(proxy_auto_config)
|
|
68
68
|
# user provided javascript with FindProxyForURL function
|
|
69
69
|
@proxy_auto_config = proxy_auto_config
|
|
@@ -79,7 +79,7 @@ END_OF_JAVASCRIPT
|
|
|
79
79
|
end
|
|
80
80
|
|
|
81
81
|
# execute proxy auto config script for the given URL : https://en.wikipedia.org/wiki/Proxy_auto-config
|
|
82
|
-
# @return either nil, or a String formatted following PAC standard
|
|
82
|
+
# @return [String, nil] either nil, or a String formatted following PAC standard
|
|
83
83
|
def find_proxy_for_url(service_url)
|
|
84
84
|
uri = URI.parse(service_url)
|
|
85
85
|
simple_url = "#{uri.scheme}://#{uri.host}"
|
|
@@ -99,7 +99,7 @@ END_OF_JAVASCRIPT
|
|
|
99
99
|
end
|
|
100
100
|
|
|
101
101
|
# used to replace URI::Generic.find_proxy
|
|
102
|
-
# @return Array of
|
|
102
|
+
# @return [Array<URI>] list of URIs, possibly empty
|
|
103
103
|
def get_proxies(service_url)
|
|
104
104
|
# prepare result
|
|
105
105
|
uri_list = []
|
|
@@ -114,14 +114,15 @@ END_OF_JAVASCRIPT
|
|
|
114
114
|
proxy_list_str.split(';').each do |item|
|
|
115
115
|
# strip and split by space
|
|
116
116
|
parts = item.strip.split
|
|
117
|
-
|
|
117
|
+
proxy_type = parts.shift
|
|
118
|
+
case proxy_type
|
|
118
119
|
when 'DIRECT'
|
|
119
|
-
Aspera.assert(parts.empty
|
|
120
|
+
Aspera.assert(parts.empty?, 'DIRECT has no param')
|
|
120
121
|
Log.log.debug('ignoring proxy DIRECT')
|
|
121
122
|
when 'PROXY'
|
|
122
123
|
addr_port = parts.shift
|
|
123
124
|
Aspera.assert_type(addr_port, String)
|
|
124
|
-
Aspera.assert(parts.empty
|
|
125
|
+
Aspera.assert(parts.empty?, 'PROXY shall have one param')
|
|
125
126
|
begin
|
|
126
127
|
# PAC proxy addresses are <host>:<port>
|
|
127
128
|
if /:[0-9]+$/.match?(addr_port)
|
|
@@ -136,7 +137,7 @@ END_OF_JAVASCRIPT
|
|
|
136
137
|
rescue StandardError => e
|
|
137
138
|
Log.log.warn{"PAC: cannot parse #{addr_port} #{e}"}
|
|
138
139
|
end
|
|
139
|
-
else Log.log.warn{"PAC: ignoring proxy type #{
|
|
140
|
+
else Log.log.warn{"PAC: ignoring proxy type #{proxy_type}: not supported"}
|
|
140
141
|
end
|
|
141
142
|
end
|
|
142
143
|
Log.log.debug{"Proxies: #{uri_list}"}
|
data/lib/aspera/rest.rb
CHANGED
|
@@ -30,6 +30,10 @@ module Aspera
|
|
|
30
30
|
# @param download_partial_suffix [String] suffix for partial download
|
|
31
31
|
# @param session_cb [lambda] lambda called on new HTTP session. Takes the Net::HTTP as arg. Used to change parameters on creation.
|
|
32
32
|
# @param progress_bar [Object] progress bar object called for file transfer
|
|
33
|
+
#
|
|
34
|
+
# @!method self.instance
|
|
35
|
+
# Returns the singleton instance of RestParameters
|
|
36
|
+
# @return [RestParameters] the singleton instance
|
|
33
37
|
class RestParameters
|
|
34
38
|
include Singleton
|
|
35
39
|
|
|
@@ -73,6 +77,9 @@ module Aspera
|
|
|
73
77
|
# and error are analyzed in RestErrorAnalyzer
|
|
74
78
|
class Rest
|
|
75
79
|
class << self
|
|
80
|
+
# Build a Basic authentication header value
|
|
81
|
+
# @param user [String] Username
|
|
82
|
+
# @param pass [String] Password
|
|
76
83
|
# @return [String] Basic auth token
|
|
77
84
|
def basic_authorization(user, pass); return "Basic #{Base64.strict_encode64("#{user}:#{pass}")}"; end
|
|
78
85
|
|
|
@@ -89,7 +96,8 @@ module Aspera
|
|
|
89
96
|
# Check if php style is specified.
|
|
90
97
|
# `nil` values in query result in key without value, e.g. `?a`, while empty string values result in `?a=`.
|
|
91
98
|
# @param url [String] The URL without query.
|
|
92
|
-
# @param query [Hash,Array,String] The query.
|
|
99
|
+
# @param query [Hash,Array,String] The query parameters.
|
|
100
|
+
# @return [URI] The built URI.
|
|
93
101
|
def build_uri(url, query)
|
|
94
102
|
uri = URI.parse(url)
|
|
95
103
|
Aspera.assert_values(uri.scheme, %w[http https]){'URI scheme'}
|
|
@@ -102,7 +110,7 @@ module Aspera
|
|
|
102
110
|
when Hash
|
|
103
111
|
URI.encode_www_form(h_to_query_array(query))
|
|
104
112
|
when Array
|
|
105
|
-
Aspera.assert(query.all?{ |i| i.is_a?(Array) && i.length.eql?(2)}
|
|
113
|
+
Aspera.assert(query.all?{ |i| i.is_a?(Array) && i.length.eql?(2)}, 'Query must be array of arrays of 2 elements')
|
|
106
114
|
URI.encode_www_form(query) # remove nil values
|
|
107
115
|
else Aspera.error_unexpected_value(query.class){'query type'}
|
|
108
116
|
end.gsub('%5B%5D=', '[]=')
|
|
@@ -114,6 +122,7 @@ module Aspera
|
|
|
114
122
|
# Either p=1&p=2 (default)
|
|
115
123
|
# or p[]=1&p[]=2 (if `:x_array_php_style` is set to true in query)
|
|
116
124
|
# @param query [Hash] HTTP query as hash
|
|
125
|
+
# @return [Array<Array>] Array of [key, value] pairs suitable for URI.encode_www_form
|
|
117
126
|
def h_to_query_array(query)
|
|
118
127
|
Aspera.assert_type(query, Hash)
|
|
119
128
|
suffix = query.delete(:x_array_php_style) ? '[]' : nil
|
|
@@ -151,37 +160,6 @@ module Aspera
|
|
|
151
160
|
end
|
|
152
161
|
end
|
|
153
162
|
|
|
154
|
-
# Parse Link header according to RFC 8288 to extract a specific relation
|
|
155
|
-
# @param link_header [String, nil] The Link header value
|
|
156
|
-
# @param rel [String] The relation to look for (default: 'next')
|
|
157
|
-
# @return [String, nil] The URL of the link with the specified relation, or nil
|
|
158
|
-
def parse_link_header(link_header, rel: 'next')
|
|
159
|
-
return if link_header.nil? || link_header.empty?
|
|
160
|
-
# RFC 8288: Link header format is: <URI>; param1=value1; param2=value2, <URI2>; ...
|
|
161
|
-
# We look for the link with the specified rel
|
|
162
|
-
link_header.split(',').each do |link_part|
|
|
163
|
-
link_part = link_part.strip
|
|
164
|
-
# Extract URL between < and >
|
|
165
|
-
url_match = link_part.match(/<([^>]+)>/)
|
|
166
|
-
next unless url_match
|
|
167
|
-
url = url_match[1]
|
|
168
|
-
# Extract parameters after the URL
|
|
169
|
-
params_str = link_part[url_match.end(0)..]
|
|
170
|
-
# Check if this link has the specified rel (with or without quotes, case insensitive)
|
|
171
|
-
next unless /;\s*rel\s*=\s*"?#{Regexp.escape(rel)}"?/i.match?(params_str)
|
|
172
|
-
return url
|
|
173
|
-
end
|
|
174
|
-
# Fallback: if no rel found and looking for 'next', try the first link (backward compatibility)
|
|
175
|
-
if rel.eql?('next')
|
|
176
|
-
first_link = link_header.split(',').first&.strip
|
|
177
|
-
if first_link
|
|
178
|
-
url_match = first_link.match(/<([^>]+)>/)
|
|
179
|
-
return url_match[1] if url_match
|
|
180
|
-
end
|
|
181
|
-
end
|
|
182
|
-
nil
|
|
183
|
-
end
|
|
184
|
-
|
|
185
163
|
# Start a HTTP/S session, also used for web sockets
|
|
186
164
|
# @param base_url [String] Base url of HTTP/S session
|
|
187
165
|
# @return [Net::HTTP] A started HTTP session
|
|
@@ -275,7 +253,7 @@ module Aspera
|
|
|
275
253
|
# Base common headers of API
|
|
276
254
|
attr_reader :headers
|
|
277
255
|
|
|
278
|
-
# @return creation parameters
|
|
256
|
+
# @return [Hash] creation parameters
|
|
279
257
|
def params
|
|
280
258
|
return {
|
|
281
259
|
base_url: @base_url, # String
|
|
@@ -316,7 +294,7 @@ module Aspera
|
|
|
316
294
|
# default is no auth
|
|
317
295
|
@auth_params = auth
|
|
318
296
|
Aspera.assert_type(@auth_params, Hash)
|
|
319
|
-
Aspera.assert(@auth_params.key?(:type)
|
|
297
|
+
Aspera.assert(@auth_params.key?(:type), 'no auth type defined')
|
|
320
298
|
@not_auth_codes = not_auth_codes
|
|
321
299
|
Aspera.assert_type(@not_auth_codes, Array)
|
|
322
300
|
# persistent session
|
|
@@ -330,10 +308,10 @@ module Aspera
|
|
|
330
308
|
@oauth = nil
|
|
331
309
|
end
|
|
332
310
|
|
|
333
|
-
# @return the OAuth object (create, or cached if already created)
|
|
311
|
+
# @return [OAuth::Base] the OAuth object (create, or cached if already created)
|
|
334
312
|
def oauth
|
|
335
313
|
if @oauth.nil?
|
|
336
|
-
Aspera.assert(@auth_params[:type].eql?(:oauth2)
|
|
314
|
+
Aspera.assert(@auth_params[:type].eql?(:oauth2), 'no OAuth defined')
|
|
337
315
|
oauth_parameters = @auth_params.reject{ |k, _v| k.eql?(:type)}
|
|
338
316
|
Log.dump(:oauth_parameters, oauth_parameters)
|
|
339
317
|
@oauth = OAuth::Factory.instance.create(**oauth_parameters)
|
|
@@ -350,7 +328,7 @@ module Aspera
|
|
|
350
328
|
# @param headers [Hash{String => String}] Additional headers (override Content-Type)
|
|
351
329
|
# @param save_to_file [String, nil] File path to save response body
|
|
352
330
|
# @param exception [Boolean] Whether to raise an exception on HTTP error
|
|
353
|
-
# @param ret [Symbol] One of :data, :resp, :both
|
|
331
|
+
# @param ret [Symbol] One of :data, :resp, :both - controls return value
|
|
354
332
|
# @return [Array(Hash, Net::HTTPResponse)] When `ret` is :both
|
|
355
333
|
# @return [Net::HTTPResponse] When `ret` is :resp
|
|
356
334
|
# @return [Hash] When `ret` is :data
|
|
@@ -374,7 +352,7 @@ module Aspera
|
|
|
374
352
|
Log.dump(:headers, headers, level: :trace1)
|
|
375
353
|
Aspera.assert_type(subpath, String)
|
|
376
354
|
# We must have a way to check return code
|
|
377
|
-
Aspera.assert(exception || !ret.eql?(:data))
|
|
355
|
+
Aspera.assert(exception || !ret.eql?(:data), 'ret: :data requires exception handler')
|
|
378
356
|
if headers.nil?
|
|
379
357
|
headers = @headers.clone
|
|
380
358
|
else
|
|
@@ -481,7 +459,7 @@ module Aspera
|
|
|
481
459
|
end
|
|
482
460
|
end
|
|
483
461
|
Log.log.debug{"result: code=#{result_http.code} mime=#{result_mime}"}
|
|
484
|
-
# sometimes there is a UTF8 char (e.g.
|
|
462
|
+
# sometimes there is a UTF8 char (e.g. (c) )
|
|
485
463
|
# TODO : related to mime type encoding ?
|
|
486
464
|
# result_http.body.force_encoding('UTF-8') if result_http.body.is_a?(String)
|
|
487
465
|
# Log.log.debug{"result: body=#{result_http.body}"}
|
|
@@ -7,6 +7,10 @@ require 'net/http'
|
|
|
7
7
|
|
|
8
8
|
module Aspera
|
|
9
9
|
# analyze error codes returned by REST calls and raise ruby exception
|
|
10
|
+
#
|
|
11
|
+
# @!method self.instance
|
|
12
|
+
# Returns the singleton instance of RestErrorAnalyzer
|
|
13
|
+
# @return [RestErrorAnalyzer] the singleton instance
|
|
10
14
|
class RestErrorAnalyzer
|
|
11
15
|
include Singleton
|
|
12
16
|
|
|
@@ -28,9 +32,9 @@ module Aspera
|
|
|
28
32
|
# Use this method to analyze a EST result and raise an exception
|
|
29
33
|
# Analyzes REST call response and raises a RestCallError exception
|
|
30
34
|
# if HTTP result code is not 2XX
|
|
31
|
-
# @param req [Net::HTTPRequest]
|
|
32
|
-
# @param data [Object]
|
|
33
|
-
# @param http [Net::HTTPResponse]
|
|
35
|
+
# @param req [Net::HTTPRequest] the HTTP request
|
|
36
|
+
# @param data [Object] the parsed response body
|
|
37
|
+
# @param http [Net::HTTPResponse] the raw HTTP response
|
|
34
38
|
def raise_on_error(req, data, http)
|
|
35
39
|
Log.log.debug{"raise_on_error #{req.method} #{req.path} #{http.code}"}
|
|
36
40
|
context = {
|
|
@@ -41,16 +41,6 @@ module Aspera
|
|
|
41
41
|
end
|
|
42
42
|
RestErrorAnalyzer.instance.add_simple_handler(name: 'T9:IBM cloud IAM', path: ['errorMessage'])
|
|
43
43
|
RestErrorAnalyzer.instance.add_simple_handler(name: 'T10:faspex v4', path: ['user_message'])
|
|
44
|
-
RestErrorAnalyzer.instance.add_handler('bss graphql') do |type, context|
|
|
45
|
-
next unless context[:data].is_a?(Hash)
|
|
46
|
-
d_t_s = context[:data]['errors']
|
|
47
|
-
next unless d_t_s.is_a?(Array)
|
|
48
|
-
d_t_s.each do |res|
|
|
49
|
-
r_err = res['message']
|
|
50
|
-
next unless r_err.is_a?(String)
|
|
51
|
-
RestErrorAnalyzer.add_error(context, type, r_err)
|
|
52
|
-
end
|
|
53
|
-
end
|
|
54
44
|
RestErrorAnalyzer.instance.add_handler('Orchestrator') do |type, context|
|
|
55
45
|
next if context[:response].code.start_with?('2')
|
|
56
46
|
data = context[:data]
|