aspera-cli 4.26.2 → 4.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +281 -21
- data/CONTRIBUTING.md +98 -7
- data/bin/ascli +3 -3
- data/docs/README.md +10757 -0
- data/docs/test-mcp-with-ai.md +202 -0
- data/lib/aspera/agent/base.rb +20 -6
- data/lib/aspera/agent/connect.rb +53 -16
- data/lib/aspera/agent/desktop.rb +52 -17
- data/lib/aspera/agent/direct.rb +26 -24
- data/lib/aspera/agent/httpgw.rb +1 -1
- data/lib/aspera/agent/node.rb +32 -0
- data/lib/aspera/agent/transferd.rb +39 -3
- data/lib/aspera/api/alee.rb +1 -1
- data/lib/aspera/api/aoc.rb +14 -16
- data/lib/aspera/api/cos_node.rb +1 -1
- data/lib/aspera/api/faspex.rb +15 -14
- data/lib/aspera/api/httpgw.rb +4 -4
- data/lib/aspera/api/node.rb +26 -23
- data/lib/aspera/api/queries/bss_subscription_account.graphql +18 -0
- data/lib/aspera/api/queries/bss_subscription_usage.graphql +18 -0
- data/lib/aspera/ascmd.rb +21 -19
- data/lib/aspera/ascp/installation.rb +37 -38
- data/lib/aspera/ascp/management.rb +2 -3
- data/lib/aspera/cli/ascp_actions.rb +155 -0
- data/lib/aspera/cli/async_transfer_store.rb +81 -0
- data/lib/aspera/cli/bootstrapper.rb +197 -0
- data/lib/aspera/cli/command_registry.rb +145 -0
- data/lib/aspera/cli/command_spec.rb +138 -0
- data/lib/aspera/cli/context.rb +16 -3
- data/lib/aspera/cli/error.rb +16 -0
- data/lib/aspera/cli/extended_value.rb +2 -1
- data/lib/aspera/cli/formatter.rb +41 -27
- data/lib/aspera/cli/gem_checker.rb +65 -0
- data/lib/aspera/cli/hints.rb +2 -2
- data/lib/aspera/cli/http.rb +70 -19
- data/lib/aspera/cli/info.rb +2 -0
- data/lib/aspera/cli/mailer.rb +97 -0
- data/lib/aspera/cli/mcp_tool.rb +198 -0
- data/lib/aspera/cli/options.schema.yaml +364 -7
- data/lib/aspera/cli/{manager.rb → parser.rb} +334 -142
- data/lib/aspera/cli/plugins/alee.rb +20 -22
- data/lib/aspera/cli/plugins/aoc.rb +1166 -853
- data/lib/aspera/cli/plugins/ats.rb +200 -161
- data/lib/aspera/cli/plugins/base.rb +457 -100
- data/lib/aspera/cli/plugins/basic_auth.rb +14 -4
- data/lib/aspera/cli/plugins/config.rb +428 -754
- data/lib/aspera/cli/plugins/console.rb +106 -64
- data/lib/aspera/cli/plugins/cos.rb +44 -32
- data/lib/aspera/cli/plugins/factory.rb +4 -4
- data/lib/aspera/cli/plugins/faspex.rb +294 -257
- data/lib/aspera/cli/plugins/faspex5.rb +585 -357
- data/lib/aspera/cli/plugins/faspio.rb +49 -51
- data/lib/aspera/cli/plugins/httpgw.rb +18 -25
- data/lib/aspera/cli/plugins/mcp.rb +279 -0
- data/lib/aspera/cli/plugins/node.rb +998 -794
- data/lib/aspera/cli/plugins/oauth.rb +7 -10
- data/lib/aspera/cli/plugins/orchestrator.rb +111 -134
- data/lib/aspera/cli/plugins/preview.rb +180 -139
- data/lib/aspera/cli/plugins/server.rb +124 -79
- data/lib/aspera/cli/plugins/shares.rb +301 -107
- data/lib/aspera/cli/preset_actions.rb +139 -0
- data/lib/aspera/cli/preset_manager.rb +13 -12
- data/lib/aspera/cli/result.rb +61 -11
- data/lib/aspera/cli/runner.rb +275 -141
- data/lib/aspera/cli/secret_finder.rb +40 -0
- data/lib/aspera/cli/special_values.rb +1 -0
- data/lib/aspera/cli/sync_actions.rb +83 -59
- data/lib/aspera/cli/terminal_formatter.rb +1 -1
- data/lib/aspera/cli/transfer_actions.rb +83 -0
- data/lib/aspera/cli/transfer_agent.rb +116 -51
- data/lib/aspera/cli/transfer_progress.rb +3 -3
- data/lib/aspera/cli/vault_manager.rb +57 -0
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/cli/wizard.rb +21 -20
- data/lib/aspera/command_line_builder.rb +22 -22
- data/lib/aspera/data_repository.rb +2 -1
- data/lib/aspera/dot_container.rb +4 -4
- data/lib/aspera/environment.rb +18 -11
- data/lib/aspera/exec_spec.rb +13 -0
- data/lib/aspera/faspex_gw.rb +1 -1
- data/lib/aspera/faspex_postproc.rb +4 -3
- data/lib/aspera/graphql.rb +35 -0
- data/lib/aspera/hash_ext.rb +6 -0
- data/lib/aspera/json_rpc/client.rb +62 -0
- data/lib/aspera/json_rpc/version.rb +7 -0
- data/lib/aspera/keychain/encrypted_hash.rb +1 -1
- data/lib/aspera/keychain/factory.rb +3 -3
- data/lib/aspera/keychain/macos_security.rb +3 -3
- data/lib/aspera/link_header.rb +82 -0
- data/lib/aspera/log.rb +17 -3
- data/lib/aspera/markdown.rb +85 -2
- data/lib/aspera/node_simulator.rb +3 -1
- data/lib/aspera/oauth/base.rb +5 -5
- data/lib/aspera/oauth/boot.rb +2 -2
- data/lib/aspera/oauth/factory.rb +7 -5
- data/lib/aspera/oauth/jwt.rb +4 -4
- data/lib/aspera/oauth/web.rb +2 -2
- data/lib/aspera/persistency_action_once.rb +8 -8
- data/lib/aspera/persistency_folder.rb +10 -3
- data/lib/aspera/preview/file_types.rb +3 -3
- data/lib/aspera/preview/generator.rb +3 -3
- data/lib/aspera/preview/utils.rb +1 -1
- data/lib/aspera/products/connect.rb +1 -1
- data/lib/aspera/products/other.rb +1 -1
- data/lib/aspera/products/transferd.rb +1 -1
- data/lib/aspera/proxy_auto_config.rb +7 -6
- data/lib/aspera/rest.rb +10 -36
- data/lib/aspera/rest_error_analyzer.rb +3 -3
- data/lib/aspera/rest_errors_aspera.rb +0 -10
- data/lib/aspera/rest_list.rb +3 -3
- data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +1853 -510
- data/lib/aspera/schema/documentation.rb +57 -26
- data/lib/aspera/schema/reader.rb +59 -10
- data/lib/aspera/schema/registry.rb +39 -6
- data/lib/aspera/secret_hider.rb +7 -2
- data/lib/aspera/ssl.rb +4 -4
- data/lib/aspera/sync/conf.schema.yaml +2 -2
- data/lib/aspera/sync/operations.rb +10 -13
- data/lib/aspera/temp_file_manager.rb +1 -1
- data/lib/aspera/timer_limiter.rb +1 -1
- data/lib/aspera/transfer/faux_file.rb +24 -11
- data/lib/aspera/transfer/parameters.rb +23 -21
- data/lib/aspera/transfer/result.rb +74 -0
- data/lib/aspera/transfer/resumer.rb +6 -6
- data/lib/aspera/transfer/spec.rb +18 -1
- data/lib/aspera/transfer/spec.schema.yaml +12 -3
- data/lib/aspera/uri_reader.rb +2 -2
- data/lib/aspera/web_auth.rb +2 -2
- data/lib/aspera/web_server_simple.rb +8 -5
- data/lib/aspera/yaml.rb +2 -1
- data.tar.gz.sig +0 -0
- metadata +25 -3
- metadata.gz.sig +0 -0
- data/lib/aspera/json_rpc.rb +0 -52
|
@@ -14,21 +14,45 @@ module Aspera
|
|
|
14
14
|
(1..18).map{ |i| "P(#{i})"} +
|
|
15
15
|
%w[Syncd Error Confl Pconf] +
|
|
16
16
|
(23..24).map{ |i| "P(#{i})"}).freeze
|
|
17
|
+
# When a plugin class includes SyncActions, register the :sql option
|
|
18
|
+
# in that class's DSL registry so Base#initialize auto-declares it.
|
|
17
19
|
class << self
|
|
18
|
-
def
|
|
19
|
-
|
|
20
|
+
def included(base)
|
|
21
|
+
base.option(:sql, description: 'SQL suffix appended to sqlite3 queries for admin subcommands (e.g. WHERE clause)')
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
# DSL helper: register the 7 `sync admin` leaf commands under the given parent path.
|
|
25
|
+
# Called at class-load time from any plugin that includes SyncActions.
|
|
26
|
+
# @param base [Class] the plugin class (receiver of DSL methods)
|
|
27
|
+
# @param admin_path [Symbol, Array<Symbol>] full parent path, e.g. %i[sync admin]
|
|
28
|
+
def register_sync_admin_commands(base, admin_path)
|
|
29
|
+
path_and_info_args = [{name: :path, type: String}, {name: :sync_info, type: Hash, mandatory: false, default: {}}]
|
|
30
|
+
base.commands_under(admin_path) do
|
|
31
|
+
base.command(:status, description: 'Show sync session status', arguments: path_and_info_args, action: :action_sync_admin_status)
|
|
32
|
+
base.command(:find, description: 'Find sync database files', arguments: [{name: :path, type: String}], action: :action_sync_admin_find)
|
|
33
|
+
base.command(:meta, description: 'Show sync session metadata', arguments: path_and_info_args, action: :action_sync_admin_meta)
|
|
34
|
+
base.command(:counters, description: 'Show sync counters', arguments: path_and_info_args, action: :action_sync_admin_counters)
|
|
35
|
+
base.command(:file_info, description: 'Show per-file sync state', arguments: path_and_info_args, action: :action_sync_admin_file_info)
|
|
36
|
+
base.command(:overview, description: 'Show sync database overview', arguments: path_and_info_args, action: :action_sync_admin_overview)
|
|
37
|
+
base.command(:query, description: 'Execute a raw SQL query', arguments: path_and_info_args, action: :action_sync_admin_query)
|
|
38
|
+
end
|
|
20
39
|
end
|
|
21
40
|
end
|
|
22
41
|
|
|
23
|
-
#
|
|
42
|
+
# Convert path + sync_info to internal `sync_info` format.
|
|
24
43
|
# The resulting sync_info has `args` format only if it contains one of the `sessions` or `instance` keys.
|
|
25
44
|
# It has the `conf` format (default) otherwise.
|
|
26
45
|
# If the `conf` format is detected, then both `local` and `remote` keys are set.
|
|
27
|
-
# @param
|
|
46
|
+
# @param path [String,NilClass] local/remote path; when nil, read from CLI
|
|
47
|
+
# @param sync_info [Hash,NilClass] extra sync info; when nil, read from CLI
|
|
48
|
+
# @param direction [Symbol,NilClass] one of DIRECTIONS, or nil for admin commands
|
|
28
49
|
# @return [Hash] sync info
|
|
29
|
-
def async_info_from_args(direction: nil)
|
|
30
|
-
path
|
|
31
|
-
|
|
50
|
+
def async_info_from_args(path: nil, sync_info: nil, direction: nil)
|
|
51
|
+
if path.nil?
|
|
52
|
+
path = options.get_next_argument('path')
|
|
53
|
+
sync_info = options.get_next_argument('sync info', mandatory: false, validation: Hash, default: {}, schema: Schema::Registry::SYNC_CONF)
|
|
54
|
+
end
|
|
55
|
+
sync_info ||= {}
|
|
32
56
|
# is the positional path a remote path ?
|
|
33
57
|
path_is_remote = direction.eql?(:pull)
|
|
34
58
|
if sync_info.key?('sessions') || sync_info.key?('instance')
|
|
@@ -37,11 +61,11 @@ module Aspera
|
|
|
37
61
|
Aspera.assert(sync_info['sessions'].length == 1, 'Only one session is supported')
|
|
38
62
|
session = sync_info['sessions'].first
|
|
39
63
|
dir_key = path_is_remote ? 'remote_dir' : 'local_dir'
|
|
40
|
-
|
|
64
|
+
Aspera.assert(!session.key?(dir_key)){"Parameter #{dir_key} shall not be in sync_info"}
|
|
41
65
|
session[dir_key] = path
|
|
42
66
|
if direction
|
|
43
67
|
dir_key = path_is_remote ? 'local_dir' : 'remote_dir'
|
|
44
|
-
|
|
68
|
+
Aspera.assert(!session.key?(dir_key)){"Parameter #{dir_key} shall not be in sync_info"}
|
|
45
69
|
session[dir_key] = transfer.destination_folder(path_is_remote ? Transfer::Spec::DIRECTION_RECEIVE : Transfer::Spec::DIRECTION_SEND)
|
|
46
70
|
local_remote = %w[local remote].map{ |i| session["#{i}_dir"]}
|
|
47
71
|
end
|
|
@@ -50,12 +74,12 @@ module Aspera
|
|
|
50
74
|
session = sync_info
|
|
51
75
|
dir_key = path_is_remote ? 'remote' : 'local'
|
|
52
76
|
session[dir_key] ||= {}
|
|
53
|
-
|
|
77
|
+
Aspera.assert(!session[dir_key].key?('path')){"Parameter #{dir_key}.path shall not be in sync_info"}
|
|
54
78
|
session[dir_key]['path'] = path
|
|
55
79
|
if direction
|
|
56
80
|
dir_key = path_is_remote ? 'local' : 'remote'
|
|
57
81
|
session[dir_key] ||= {}
|
|
58
|
-
|
|
82
|
+
Aspera.assert(!session[dir_key].key?('path')){"Parameter #{dir_key}.path shall not be in sync_info"}
|
|
59
83
|
session[dir_key]['path'] = transfer.destination_folder(path_is_remote ? Transfer::Spec::DIRECTION_RECEIVE : Transfer::Spec::DIRECTION_SEND)
|
|
60
84
|
local_remote = %w[local remote].map{ |i| session[i]['path']}
|
|
61
85
|
end
|
|
@@ -63,7 +87,7 @@ module Aspera
|
|
|
63
87
|
session['quiet'] = false if !session.key?('quiet') && Environment.terminal?
|
|
64
88
|
end
|
|
65
89
|
if direction
|
|
66
|
-
|
|
90
|
+
Aspera.assert(!session.key?('direction'), type: BadArgument){'direction shall not be in sync_info'}
|
|
67
91
|
session['direction'] = direction.to_s
|
|
68
92
|
# generate name if not provided by user
|
|
69
93
|
if !session.key?('name')
|
|
@@ -80,66 +104,66 @@ module Aspera
|
|
|
80
104
|
sync_info
|
|
81
105
|
end
|
|
82
106
|
|
|
83
|
-
# Provide database object from
|
|
84
|
-
|
|
85
|
-
|
|
107
|
+
# Provide database object from path + sync_info for admin ops
|
|
108
|
+
# @param path [String,NilClass] when nil, read from CLI
|
|
109
|
+
# @param sync_info [Hash,NilClass] when nil, read from CLI
|
|
110
|
+
def db_from_args(path: nil, sync_info: nil)
|
|
111
|
+
sync_info = async_info_from_args(path: path, sync_info: sync_info)
|
|
86
112
|
session = sync_info.key?('sessions') ? sync_info['sessions'].first : sync_info
|
|
87
113
|
# if name not provided, check in db folder if there is only one name
|
|
88
114
|
if !session.key?('name')
|
|
89
115
|
local_db_dir = Sync::Operations.local_db_folder(sync_info)
|
|
90
116
|
dbs = Sync::Operations.list_db_files(local_db_dir)
|
|
91
|
-
|
|
117
|
+
Aspera.assert(dbs.length == 1){"#{dbs.length} session found in #{local_db_dir}, please provide a name"}
|
|
92
118
|
session['name'] = dbs.keys.first
|
|
93
119
|
end
|
|
94
120
|
Sync::Database.new(Sync::Operations.session_db_file(sync_info))
|
|
95
121
|
end
|
|
96
122
|
|
|
97
|
-
def
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
end
|
|
115
|
-
return Result::ObjectList.new(
|
|
116
|
-
result,
|
|
117
|
-
fields: %w[sstate record_id f_meta_path message]
|
|
118
|
-
)
|
|
119
|
-
when :overview
|
|
120
|
-
return Result::ObjectList.new(
|
|
121
|
-
db_from_args.overview,
|
|
122
|
-
fields: %w[table name type]
|
|
123
|
-
)
|
|
124
|
-
when :query
|
|
125
|
-
return Result.auto(db_from_args.execute(options.get_option(:sql, mandatory: true)))
|
|
126
|
-
else Aspera.error_unexpected_value(command2)
|
|
127
|
-
end
|
|
123
|
+
def action_sync_admin_status(path:, sync_info: {}, **)
|
|
124
|
+
Result::SingleObject.new(Sync::Operations.admin_status(async_info_from_args(path: path, sync_info: sync_info)))
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
def action_sync_admin_find(path:, **)
|
|
128
|
+
dbs = Sync::Operations.list_db_files(path)
|
|
129
|
+
Result::ObjectList.new(dbs.keys.map{ |n| {name: n, path: dbs[n]}})
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
def action_sync_admin_meta(path:, sync_info: {}, **)
|
|
133
|
+
require 'aspera/sync/database'
|
|
134
|
+
Result::SingleObject.new(db_from_args(path: path, sync_info: sync_info).meta(options.get_option(:sql)))
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
def action_sync_admin_counters(path:, sync_info: {}, **)
|
|
138
|
+
require 'aspera/sync/database'
|
|
139
|
+
Result::SingleObject.new(db_from_args(path: path, sync_info: sync_info).counters(options.get_option(:sql)))
|
|
128
140
|
end
|
|
129
141
|
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
case command
|
|
136
|
-
when *Sync::Operations::DIRECTIONS
|
|
137
|
-
Sync::Operations.start(async_info_from_args(direction: command), transfer.user_transfer_spec, &block)
|
|
138
|
-
return Result::Success.new
|
|
139
|
-
when :admin
|
|
140
|
-
return execute_sync_admin
|
|
141
|
-
else Aspera.error_unexpected_value(command)
|
|
142
|
+
def action_sync_admin_file_info(path:, sync_info: {}, **)
|
|
143
|
+
require 'aspera/sync/database'
|
|
144
|
+
result = db_from_args(path: path, sync_info: sync_info).file_info(options.get_option(:sql))
|
|
145
|
+
result.each do |r|
|
|
146
|
+
r['sstate'] = SyncActions::STATE_STR[r['state']] if r['state']
|
|
142
147
|
end
|
|
148
|
+
Result::ObjectList.new(result, fields: %w[sstate record_id f_meta_path message])
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
def action_sync_admin_overview(path:, sync_info: {}, **)
|
|
152
|
+
require 'aspera/sync/database'
|
|
153
|
+
Result::ObjectList.new(db_from_args(path: path, sync_info: sync_info).overview, fields: %w[table name type])
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
def action_sync_admin_query(path:, sync_info: {}, **)
|
|
157
|
+
require 'aspera/sync/database'
|
|
158
|
+
Result.auto(db_from_args(path: path, sync_info: sync_info).execute(options.get_option(:sql, mandatory: true)))
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
# Execute a sync transfer for a given direction.
|
|
162
|
+
# @param direction [Symbol] one of Sync::Operations::DIRECTIONS (:push, :pull, :bidi)
|
|
163
|
+
# @param block [Proc, nil] block to generate transfer spec; receives (direction, local_dir, remote_dir)
|
|
164
|
+
def run_sync_transfer(direction, &block)
|
|
165
|
+
Sync::Operations.start(async_info_from_args(direction: direction), transfer.user_transfer_spec, &block)
|
|
166
|
+
Result::Success.new
|
|
143
167
|
end
|
|
144
168
|
end
|
|
145
169
|
end
|
|
@@ -14,7 +14,7 @@ module Aspera
|
|
|
14
14
|
HINT = 'HINT:'.bg_green.gray.blink.freeze
|
|
15
15
|
include FormatterInterface
|
|
16
16
|
|
|
17
|
-
# Format boolean with colored symbol (
|
|
17
|
+
# Format boolean with colored symbol (+/- or Y/ )
|
|
18
18
|
def tick(yes)
|
|
19
19
|
result =
|
|
20
20
|
if Environment.terminal_supports_unicode?
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'aspera/cli/async_transfer_store'
|
|
4
|
+
require 'aspera/agent/factory'
|
|
5
|
+
require 'aspera/log'
|
|
6
|
+
require 'aspera/assert'
|
|
7
|
+
|
|
8
|
+
module Aspera
|
|
9
|
+
module Cli
|
|
10
|
+
# Mixin for Config plugin: async transfer management actions.
|
|
11
|
+
# Exposes three sub-commands under `config transfer`:
|
|
12
|
+
# status --id=<job_id> Re-query a running/completed transfer
|
|
13
|
+
# list List all persisted async transfer entries
|
|
14
|
+
# cleanup Remove completed/failed/cancelled entries
|
|
15
|
+
#
|
|
16
|
+
# Architecture:
|
|
17
|
+
# - remote-daemon agents (desktop, node, connect, transferd): transfer_id + agent_params
|
|
18
|
+
# are persisted in AsyncTransferStore; status is re-queried via Agent::Xxx.transfer_status
|
|
19
|
+
# - direct agent: transfers live in Ruby threads - store is the agent's @sessions;
|
|
20
|
+
# status is not persistable across process restarts (returns an informational message)
|
|
21
|
+
module TransferActions
|
|
22
|
+
# Re-query the status of a single async transfer job.
|
|
23
|
+
# @param job_id [String] UUID returned at submission time
|
|
24
|
+
def action_transfer_status(job_id:, **)
|
|
25
|
+
store = async_transfer_store
|
|
26
|
+
entry = store.read(job_id)
|
|
27
|
+
Aspera.assert(!entry.nil?, type: Cli::BadArgument){"Unknown job_id: #{job_id}"}
|
|
28
|
+
live = query_live_status(entry)
|
|
29
|
+
if live
|
|
30
|
+
entry.merge!(live)
|
|
31
|
+
store.write(job_id, entry)
|
|
32
|
+
end
|
|
33
|
+
Result::SingleObject.new(entry)
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# List all persisted async transfer entries.
|
|
37
|
+
def action_transfer_list(**)
|
|
38
|
+
rows = async_transfer_store.list
|
|
39
|
+
Result::ObjectList.new(rows, fields: %w[job_id agent_type status started_at ended_at bytes_transferred transfer_id])
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
TERMINAL_STATUSES = %w[completed failed cancelled].freeze
|
|
43
|
+
private_constant :TERMINAL_STATUSES
|
|
44
|
+
|
|
45
|
+
# Delete completed, failed, and cancelled entries from the store.
|
|
46
|
+
def action_transfer_cleanup(**)
|
|
47
|
+
store = async_transfer_store
|
|
48
|
+
deleted = store.list
|
|
49
|
+
.select{ |e| TERMINAL_STATUSES.include?(e['status'])}
|
|
50
|
+
.map do |e|
|
|
51
|
+
store.delete(e['job_id'])
|
|
52
|
+
e['job_id']
|
|
53
|
+
end
|
|
54
|
+
Result::Status.new("Deleted #{deleted.size} completed transfer(s)#{": #{deleted.join(', ')}" unless deleted.empty?}")
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
private
|
|
58
|
+
|
|
59
|
+
# Lazy accessor - requires context.persistency (available in all Base sub-classes).
|
|
60
|
+
def async_transfer_store
|
|
61
|
+
@async_transfer_store ||= AsyncTransferStore.new(persistency)
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
# Delegate to the appropriate agent class method.
|
|
65
|
+
# Returns nil for direct transfers (in-process only, not re-queryable across processes).
|
|
66
|
+
def query_live_status(entry)
|
|
67
|
+
agent_type = entry['agent_type']
|
|
68
|
+
transfer_id = entry['transfer_id']
|
|
69
|
+
agent_params = entry['agent_params'] || {}
|
|
70
|
+
|
|
71
|
+
# direct transfers live in Ruby threads - not persistable across process restarts
|
|
72
|
+
return if agent_type.eql?('direct')
|
|
73
|
+
|
|
74
|
+
require "aspera/agent/#{agent_type}"
|
|
75
|
+
agent_class = Aspera::Agent.const_get(agent_type.capitalize)
|
|
76
|
+
agent_class.transfer_status(transfer_id, agent_params)
|
|
77
|
+
rescue StandardError => e
|
|
78
|
+
Log.log.warn{"Could not re-query #{entry['agent_type']} agent for job #{entry['job_id']}: #{e}"}
|
|
79
|
+
nil
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
end
|
|
@@ -1,11 +1,14 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require 'aspera/agent/factory'
|
|
4
|
+
require 'aspera/schema/registry'
|
|
5
|
+
require 'aspera/transfer/result'
|
|
4
6
|
require 'aspera/transfer/spec'
|
|
5
7
|
require 'aspera/cli/info'
|
|
8
|
+
require 'aspera/cli/async_transfer_store'
|
|
6
9
|
require 'aspera/log'
|
|
7
10
|
require 'aspera/assert'
|
|
8
|
-
require '
|
|
11
|
+
require 'securerandom'
|
|
9
12
|
|
|
10
13
|
module Aspera
|
|
11
14
|
module Cli
|
|
@@ -28,53 +31,50 @@ module Aspera
|
|
|
28
31
|
<%=ts.to_yaml%>
|
|
29
32
|
END_OF_TEMPLATE
|
|
30
33
|
CP4I_REMOTE_HOST_LB = 'N/A'
|
|
31
|
-
private_constant :
|
|
32
|
-
:FILE_LIST_FROM_TRANSFER_SPEC,
|
|
34
|
+
private_constant :FILE_LIST_FROM_TRANSFER_SPEC,
|
|
33
35
|
:FILE_LIST_OPTIONS,
|
|
34
36
|
:DEFAULT_TRANSFER_NOTIFY_TEMPLATE
|
|
35
37
|
|
|
36
38
|
class << self
|
|
37
|
-
#
|
|
38
|
-
#
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
39
|
+
# Declare all transfer CLI options (metadata only - no handler binding yet).
|
|
40
|
+
# @param options [Aspera::Cli::Parser]
|
|
41
|
+
def declare_options(options)
|
|
42
|
+
options.declare(:ts, description: 'Override transfer spec values', schema: Schema::Registry::TRANSFER_SPEC)
|
|
43
|
+
options.declare(:to_folder, description: 'Destination folder for transferred files')
|
|
44
|
+
options.declare(:sources, description: "How list of transferred files is provided (#{FILE_LIST_OPTIONS.join(',')})", default: FILE_LIST_FROM_ARGS)
|
|
45
|
+
options.declare(:src_type, description: 'Type of file list', allowed: %i[list pair], default: :list)
|
|
46
|
+
options.declare(:transfer, description: 'Transfer agent type, or agent parameters with optional agent key', allowed: [Hash, String], schema: Schema::Registry::TRANSFER_AGENT_OPTIONS)
|
|
47
|
+
options.declare(:transfer_info, description: 'Parameters for transfer agent', allowed: Hash, deprecation: 'use --transfer instead', schema: Schema::Registry::TRANSFER_AGENT_OPTIONS)
|
|
46
48
|
end
|
|
47
49
|
end
|
|
48
50
|
|
|
49
|
-
# @param
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
@
|
|
51
|
+
# @param context [Context] Application context
|
|
52
|
+
def initialize(context)
|
|
53
|
+
Aspera.assert_type(context, Context){'context'}
|
|
54
|
+
Aspera.assert_type(context.options, Parser){'context.options'}
|
|
55
|
+
@context = context
|
|
54
56
|
# Command line can override transfer spec
|
|
55
57
|
@user_transfer_spec = {
|
|
56
58
|
'create_dir' => true,
|
|
57
59
|
'resume_policy' => 'sparse_csum'
|
|
58
60
|
}
|
|
59
|
-
# options for transfer agent
|
|
60
|
-
@
|
|
61
|
+
# options for transfer agent (agent type + agent-specific parameters)
|
|
62
|
+
@transfer_options = {}
|
|
61
63
|
# the currently selected transfer agent
|
|
62
64
|
@agent = nil
|
|
63
65
|
# source/destination pair, like "paths" of transfer spec
|
|
64
66
|
@transfer_paths = nil
|
|
65
67
|
# HTTPGW URL provided by webapp
|
|
66
68
|
@httpgw_url_lambda = nil
|
|
67
|
-
@
|
|
68
|
-
@
|
|
69
|
-
@
|
|
70
|
-
@
|
|
71
|
-
@
|
|
72
|
-
@opt_mgr.declare(:transfer_info, 'Parameters for transfer agent', allowed: Hash, handler: {o: self, m: :transfer_info}, schema: Schema::Registry::TRANSFER_INFO)
|
|
73
|
-
@opt_mgr.parse_options!
|
|
69
|
+
self.class.declare_options(@context.options)
|
|
70
|
+
@context.options.set_handler(:ts, object: self, method: :user_transfer_spec)
|
|
71
|
+
@context.options.set_handler(:transfer, object: self, method: :option_transfer)
|
|
72
|
+
@context.options.set_handler(:transfer_info, object: self, method: :transfer_options)
|
|
73
|
+
@context.options.parse_options!
|
|
74
74
|
@notification_cb = nil
|
|
75
|
-
if !@
|
|
75
|
+
if !@context.options.get_option(:notify_to).nil?
|
|
76
76
|
@notification_cb = ->(transfer_spec, global_status) do
|
|
77
|
-
@
|
|
77
|
+
@context.mailer.send_email_template(email_template_default: DEFAULT_TRANSFER_NOTIFY_TEMPLATE, values: {
|
|
78
78
|
subject: "#{Info::CMD_NAME} transfer: #{global_status}",
|
|
79
79
|
status: global_status,
|
|
80
80
|
ts: transfer_spec
|
|
@@ -83,7 +83,23 @@ module Aspera
|
|
|
83
83
|
end
|
|
84
84
|
end
|
|
85
85
|
|
|
86
|
-
attr_accessor :user_transfer_spec, :
|
|
86
|
+
attr_accessor :user_transfer_spec, :transfer_options
|
|
87
|
+
|
|
88
|
+
# Composite option handler for :transfer
|
|
89
|
+
# String value: shorthand for agent type, stored as {'agent' => value}
|
|
90
|
+
# Hash value: merged into @transfer_options (may include 'agent' key)
|
|
91
|
+
def option_transfer(_option_sym, operation, value = nil)
|
|
92
|
+
Aspera.assert_values(operation, %i[set get])
|
|
93
|
+
case operation
|
|
94
|
+
when :set
|
|
95
|
+
value = {'agent' => value} if value.is_a?(String)
|
|
96
|
+
Aspera.assert_type(value, Hash)
|
|
97
|
+
@transfer_options = @transfer_options.deep_merge(value)
|
|
98
|
+
when :get
|
|
99
|
+
return @transfer_options
|
|
100
|
+
end
|
|
101
|
+
nil
|
|
102
|
+
end
|
|
87
103
|
|
|
88
104
|
def agent_instance=(instance)
|
|
89
105
|
@agent = instance
|
|
@@ -92,25 +108,27 @@ module Aspera
|
|
|
92
108
|
# analyze options and create new agent if not already created or set
|
|
93
109
|
def agent_instance
|
|
94
110
|
return @agent unless @agent.nil?
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
agent_options
|
|
111
|
+
# agent type: from composite option 'agent' key, default :direct
|
|
112
|
+
raw_type = @transfer_options['agent'] || :direct
|
|
113
|
+
agent_type = Parser.get_from_list(raw_type.to_s, 'transfer agent', Agent::Factory::ALL.keys)
|
|
114
|
+
# set keys as symbols, strip internal keys not forwarded to the agent constructor
|
|
115
|
+
agent_options = @transfer_options.except('agent', 'asynchronous').symbolize_keys
|
|
116
|
+
agent_options[:progress] = @context.progress_bar
|
|
117
|
+
agent_options[:config_dir] = @context.main_folder
|
|
100
118
|
# special cases
|
|
101
119
|
case agent_type
|
|
102
120
|
when :node
|
|
103
121
|
if !agent_options.key?(:url)
|
|
104
|
-
param_set_name = @
|
|
105
|
-
|
|
106
|
-
agent_options.merge!(@
|
|
122
|
+
param_set_name = @context.presets.plugin_default_name(:node)
|
|
123
|
+
Aspera.assert(!param_set_name.nil?, type: Cli::BadArgument){"No default node configured. Please specify #{Options.option_name_to_line(:transfer)}.url or #{Options.option_name_to_line(:transfer)}"}
|
|
124
|
+
agent_options.merge!(@context.presets.by_name(param_set_name).symbolize_keys)
|
|
107
125
|
end
|
|
108
126
|
when :direct
|
|
109
127
|
# by default do not display ascp native progress bar
|
|
110
128
|
agent_options[:quiet] = true unless agent_options.key?(:quiet)
|
|
111
|
-
agent_options[:check_ignore_cb] = ->(host, port){@
|
|
129
|
+
agent_options[:check_ignore_cb] = ->(host, port){@context.http_config.ignore_cert?(host, port)}
|
|
112
130
|
# JRuby
|
|
113
|
-
agent_options[:trusted_certs] = @
|
|
131
|
+
agent_options[:trusted_certs] = @context.http_config.trusted_cert_locations unless agent_options.key?(:trusted_certs)
|
|
114
132
|
when :httpgw
|
|
115
133
|
unless agent_options.key?(:url) || @httpgw_url_lambda.nil?
|
|
116
134
|
Log.log.debug('retrieving HTTPGW URL from webapp')
|
|
@@ -127,7 +145,7 @@ module Aspera
|
|
|
127
145
|
# @param direction [String] `send`` or `receive``
|
|
128
146
|
# @return [String] Destination folder for transfers (with default based on direction)
|
|
129
147
|
def destination_folder(direction)
|
|
130
|
-
dest_folder = @
|
|
148
|
+
dest_folder = @context.options.get_option(:to_folder)
|
|
131
149
|
# do not expand path, if user wants to expand path: user @path:
|
|
132
150
|
return dest_folder unless dest_folder.nil?
|
|
133
151
|
dest_folder = @user_transfer_spec['destination_root']
|
|
@@ -157,7 +175,7 @@ module Aspera
|
|
|
157
175
|
# Transform the list of paths to a list of hash with source/dest
|
|
158
176
|
# @param file_list [Array<Hash>]
|
|
159
177
|
def list_to_paths(file_list)
|
|
160
|
-
source_type = @
|
|
178
|
+
source_type = @context.options.get_option(:src_type, mandatory: true)
|
|
161
179
|
@transfer_paths =
|
|
162
180
|
case source_type
|
|
163
181
|
when :list
|
|
@@ -181,23 +199,23 @@ module Aspera
|
|
|
181
199
|
# start with lower priority : get paths from transfer spec on command line
|
|
182
200
|
@transfer_paths = @user_transfer_spec['paths'] if @user_transfer_spec.key?('paths')
|
|
183
201
|
# is there a source list option ?
|
|
184
|
-
sources = @
|
|
202
|
+
sources = @context.options.get_option(:sources)
|
|
185
203
|
@transfer_paths =
|
|
186
204
|
case sources
|
|
187
205
|
when FILE_LIST_FROM_ARGS
|
|
188
206
|
Log.log.debug('getting file list as parameters')
|
|
189
207
|
Aspera.assert_type(default, Array, NilClass)
|
|
190
208
|
# get remaining arguments
|
|
191
|
-
list = @
|
|
209
|
+
list = @context.options.get_next_argument('source file list', multiple: true, default: default)
|
|
192
210
|
raise Cli::BadArgument, 'specify at least one file on command line or use ' \
|
|
193
211
|
"--sources=#{FILE_LIST_FROM_TRANSFER_SPEC} to use transfer spec" if !list.is_a?(Array) || list.empty?
|
|
194
212
|
list_to_paths(list)
|
|
195
213
|
when FILE_LIST_FROM_TRANSFER_SPEC
|
|
196
214
|
Log.log.debug('assume list provided in transfer spec')
|
|
197
215
|
special_case_direct_with_list =
|
|
198
|
-
@
|
|
199
|
-
Transfer::Parameters.ascp_args_file_list?(@
|
|
200
|
-
|
|
216
|
+
(@transfer_options['agent'] || :direct).to_sym.eql?(:direct) &&
|
|
217
|
+
Transfer::Parameters.ascp_args_file_list?(@transfer_options['ascp_args'])
|
|
218
|
+
Aspera.assert(!@transfer_paths.nil? || special_case_direct_with_list, type: Cli::BadArgument){'transfer spec on command line must have sources'}
|
|
201
219
|
# can be nil
|
|
202
220
|
@transfer_paths
|
|
203
221
|
when Array
|
|
@@ -211,8 +229,8 @@ module Aspera
|
|
|
211
229
|
end
|
|
212
230
|
|
|
213
231
|
# Start a transfer and wait for completion, plugins shall use this method
|
|
214
|
-
# @param transfer_spec [Hash]
|
|
215
|
-
# @param rest_token [Rest] if oauth token regeneration supported
|
|
232
|
+
# @param transfer_spec [Hash] transfer specification
|
|
233
|
+
# @param rest_token [Rest, nil] if oauth token regeneration supported
|
|
216
234
|
def start(transfer_spec, rest_token: nil)
|
|
217
235
|
# check parameters
|
|
218
236
|
Aspera.assert_type(transfer_spec, Hash){'transfer_spec'}
|
|
@@ -240,15 +258,55 @@ module Aspera
|
|
|
240
258
|
@user_transfer_spec['paths'] = transfer_spec['paths'] || ts_source_paths
|
|
241
259
|
# updated transfer spec with command line
|
|
242
260
|
transfer_spec.deep_merge!(@user_transfer_spec)
|
|
261
|
+
# resolve pseudo-parameter: target_rate -> target_rate_kbps (overrides target_rate_kbps if both are present)
|
|
262
|
+
transfer_spec['target_rate_kbps'] = Transfer::Spec.rate_string_to_kbps(transfer_spec.delete('target_rate')) if transfer_spec.key?('target_rate')
|
|
243
263
|
# recursively remove values that are nil (user wants to delete)
|
|
244
264
|
transfer_spec.deep_do{ |hash, key, value, _unused| hash.delete(key) if value.nil?}
|
|
245
265
|
# if TS from app has content_protection (e.g. F5), that means content is protected: ask password if not provided
|
|
246
|
-
transfer_spec['content_protection_password'] = @
|
|
266
|
+
transfer_spec['content_protection_password'] = @context.options.prompt_user_input('content protection password', sensitive: true) if transfer_spec['content_protection'].eql?('decrypt') && !transfer_spec.key?('content_protection_password')
|
|
247
267
|
# create transfer agent
|
|
248
268
|
agent_instance.start_transfer(transfer_spec, token_regenerator: rest_token)
|
|
249
|
-
#
|
|
269
|
+
# --- async mode ---
|
|
270
|
+
if @transfer_options['asynchronous']
|
|
271
|
+
agent_type = (@transfer_options['agent'] || 'direct').to_s
|
|
272
|
+
raise Cli::BadArgument, 'asynchronous mode is not supported for agent httpgw' if agent_type.eql?('httpgw')
|
|
273
|
+
# For direct agent: the transfer lives in this Ruby process (threads).
|
|
274
|
+
# We return the job_id from the agent itself; no persistent store needed.
|
|
275
|
+
if agent_type.eql?('direct')
|
|
276
|
+
job_id = agent_instance.instance_variable_get(:@sessions).last[:job_id]
|
|
277
|
+
Log.log.info{"Async direct transfer started: job_id=#{job_id}"}
|
|
278
|
+
return Transfer::Result.async(job_id: job_id)
|
|
279
|
+
end
|
|
280
|
+
# For remote-daemon agents (desktop, node, connect, transferd): persist transfer_id
|
|
281
|
+
# so the status can be re-queried in a later process invocation.
|
|
282
|
+
agent_params = @transfer_options.except('agent', 'asynchronous')
|
|
283
|
+
case agent_type
|
|
284
|
+
when 'desktop'
|
|
285
|
+
agent_params['application_id'] = agent_instance.application_id
|
|
286
|
+
when 'connect'
|
|
287
|
+
agent_params['app_id'] = agent_instance.app_id
|
|
288
|
+
when 'transferd'
|
|
289
|
+
# store the resolved daemon endpoint (auto-port may have been assigned)
|
|
290
|
+
agent_params['url'] = agent_instance.daemon_endpoint
|
|
291
|
+
end
|
|
292
|
+
job_id = SecureRandom.uuid
|
|
293
|
+
async_store.write(job_id, {
|
|
294
|
+
'job_id' => job_id,
|
|
295
|
+
'agent_type' => agent_type,
|
|
296
|
+
'transfer_id' => agent_instance.instance_variable_get(:@transfer_id).to_s,
|
|
297
|
+
'agent_params' => agent_params,
|
|
298
|
+
'status' => 'running',
|
|
299
|
+
'bytes_transferred' => 0,
|
|
300
|
+
'started_at' => Time.now.utc.iso8601,
|
|
301
|
+
'ended_at' => nil,
|
|
302
|
+
'error' => nil
|
|
303
|
+
})
|
|
304
|
+
Log.log.info{"Async transfer started: job_id=#{job_id}"}
|
|
305
|
+
return Transfer::Result.async(job_id: job_id)
|
|
306
|
+
end
|
|
307
|
+
# --- synchronous mode (default) ---
|
|
250
308
|
result = agent_instance.wait_for_completion
|
|
251
|
-
@notification_cb&.call(transfer_spec,
|
|
309
|
+
@notification_cb&.call(transfer_spec, result)
|
|
252
310
|
return result
|
|
253
311
|
end
|
|
254
312
|
|
|
@@ -256,6 +314,13 @@ module Aspera
|
|
|
256
314
|
def shutdown
|
|
257
315
|
@agent.shutdown if @agent.respond_to?(:shutdown)
|
|
258
316
|
end
|
|
317
|
+
|
|
318
|
+
private
|
|
319
|
+
|
|
320
|
+
# Lazy accessor for the async transfer store (only created when needed)
|
|
321
|
+
def async_store
|
|
322
|
+
@async_store ||= AsyncTransferStore.new(@context.persistency)
|
|
323
|
+
end
|
|
259
324
|
end
|
|
260
325
|
end
|
|
261
326
|
end
|
|
@@ -25,9 +25,9 @@ module Aspera
|
|
|
25
25
|
end
|
|
26
26
|
|
|
27
27
|
# Called by user of progress bar with a status on a transfer session
|
|
28
|
-
# @param session_id the unique identifier of a transfer session
|
|
28
|
+
# @param session_id [String] the unique identifier of a transfer session
|
|
29
29
|
# @param type [Symbol] one of: sessions_init, session_start, session_size, transfer, session_end and end
|
|
30
|
-
# @param info optional specific additional info for the given event type
|
|
30
|
+
# @param info [Object, nil] optional specific additional info for the given event type
|
|
31
31
|
def event(type, session_id: nil, info: nil)
|
|
32
32
|
Log.log.trace1{"progress: #{type} #{session_id} #{info}"}
|
|
33
33
|
return if @completed
|
|
@@ -50,7 +50,7 @@ module Aspera
|
|
|
50
50
|
when :session_start
|
|
51
51
|
Aspera.assert_type(session_id, String)
|
|
52
52
|
Aspera.assert(info.nil?, 'info must be nil for :session_start event')
|
|
53
|
-
|
|
53
|
+
Aspera.assert(!@sessions[session_id]){"Session #{session_id} already started"}
|
|
54
54
|
@sessions[session_id] = {
|
|
55
55
|
job_size: 0, # Total size of transfer (pre-calc)
|
|
56
56
|
current: 0,
|