aspera-cli 4.26.2 → 4.27.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (136) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +281 -21
  4. data/CONTRIBUTING.md +98 -7
  5. data/bin/ascli +3 -3
  6. data/docs/README.md +10757 -0
  7. data/docs/test-mcp-with-ai.md +202 -0
  8. data/lib/aspera/agent/base.rb +20 -6
  9. data/lib/aspera/agent/connect.rb +53 -16
  10. data/lib/aspera/agent/desktop.rb +52 -17
  11. data/lib/aspera/agent/direct.rb +26 -24
  12. data/lib/aspera/agent/httpgw.rb +1 -1
  13. data/lib/aspera/agent/node.rb +32 -0
  14. data/lib/aspera/agent/transferd.rb +39 -3
  15. data/lib/aspera/api/alee.rb +1 -1
  16. data/lib/aspera/api/aoc.rb +14 -16
  17. data/lib/aspera/api/cos_node.rb +1 -1
  18. data/lib/aspera/api/faspex.rb +15 -14
  19. data/lib/aspera/api/httpgw.rb +4 -4
  20. data/lib/aspera/api/node.rb +26 -23
  21. data/lib/aspera/api/queries/bss_subscription_account.graphql +18 -0
  22. data/lib/aspera/api/queries/bss_subscription_usage.graphql +18 -0
  23. data/lib/aspera/ascmd.rb +21 -19
  24. data/lib/aspera/ascp/installation.rb +37 -38
  25. data/lib/aspera/ascp/management.rb +2 -3
  26. data/lib/aspera/cli/ascp_actions.rb +155 -0
  27. data/lib/aspera/cli/async_transfer_store.rb +81 -0
  28. data/lib/aspera/cli/bootstrapper.rb +197 -0
  29. data/lib/aspera/cli/command_registry.rb +145 -0
  30. data/lib/aspera/cli/command_spec.rb +138 -0
  31. data/lib/aspera/cli/context.rb +16 -3
  32. data/lib/aspera/cli/error.rb +16 -0
  33. data/lib/aspera/cli/extended_value.rb +2 -1
  34. data/lib/aspera/cli/formatter.rb +41 -27
  35. data/lib/aspera/cli/gem_checker.rb +65 -0
  36. data/lib/aspera/cli/hints.rb +2 -2
  37. data/lib/aspera/cli/http.rb +70 -19
  38. data/lib/aspera/cli/info.rb +2 -0
  39. data/lib/aspera/cli/mailer.rb +97 -0
  40. data/lib/aspera/cli/mcp_tool.rb +198 -0
  41. data/lib/aspera/cli/options.schema.yaml +364 -7
  42. data/lib/aspera/cli/{manager.rb → parser.rb} +334 -142
  43. data/lib/aspera/cli/plugins/alee.rb +20 -22
  44. data/lib/aspera/cli/plugins/aoc.rb +1166 -853
  45. data/lib/aspera/cli/plugins/ats.rb +200 -161
  46. data/lib/aspera/cli/plugins/base.rb +457 -100
  47. data/lib/aspera/cli/plugins/basic_auth.rb +14 -4
  48. data/lib/aspera/cli/plugins/config.rb +428 -754
  49. data/lib/aspera/cli/plugins/console.rb +106 -64
  50. data/lib/aspera/cli/plugins/cos.rb +44 -32
  51. data/lib/aspera/cli/plugins/factory.rb +4 -4
  52. data/lib/aspera/cli/plugins/faspex.rb +294 -257
  53. data/lib/aspera/cli/plugins/faspex5.rb +585 -357
  54. data/lib/aspera/cli/plugins/faspio.rb +49 -51
  55. data/lib/aspera/cli/plugins/httpgw.rb +18 -25
  56. data/lib/aspera/cli/plugins/mcp.rb +279 -0
  57. data/lib/aspera/cli/plugins/node.rb +998 -794
  58. data/lib/aspera/cli/plugins/oauth.rb +7 -10
  59. data/lib/aspera/cli/plugins/orchestrator.rb +111 -134
  60. data/lib/aspera/cli/plugins/preview.rb +180 -139
  61. data/lib/aspera/cli/plugins/server.rb +124 -79
  62. data/lib/aspera/cli/plugins/shares.rb +301 -107
  63. data/lib/aspera/cli/preset_actions.rb +139 -0
  64. data/lib/aspera/cli/preset_manager.rb +13 -12
  65. data/lib/aspera/cli/result.rb +61 -11
  66. data/lib/aspera/cli/runner.rb +275 -141
  67. data/lib/aspera/cli/secret_finder.rb +40 -0
  68. data/lib/aspera/cli/special_values.rb +1 -0
  69. data/lib/aspera/cli/sync_actions.rb +83 -59
  70. data/lib/aspera/cli/terminal_formatter.rb +1 -1
  71. data/lib/aspera/cli/transfer_actions.rb +83 -0
  72. data/lib/aspera/cli/transfer_agent.rb +116 -51
  73. data/lib/aspera/cli/transfer_progress.rb +3 -3
  74. data/lib/aspera/cli/vault_manager.rb +57 -0
  75. data/lib/aspera/cli/version.rb +1 -1
  76. data/lib/aspera/cli/wizard.rb +21 -20
  77. data/lib/aspera/command_line_builder.rb +22 -22
  78. data/lib/aspera/data_repository.rb +2 -1
  79. data/lib/aspera/dot_container.rb +4 -4
  80. data/lib/aspera/environment.rb +18 -11
  81. data/lib/aspera/exec_spec.rb +13 -0
  82. data/lib/aspera/faspex_gw.rb +1 -1
  83. data/lib/aspera/faspex_postproc.rb +4 -3
  84. data/lib/aspera/graphql.rb +35 -0
  85. data/lib/aspera/hash_ext.rb +6 -0
  86. data/lib/aspera/json_rpc/client.rb +62 -0
  87. data/lib/aspera/json_rpc/version.rb +7 -0
  88. data/lib/aspera/keychain/encrypted_hash.rb +1 -1
  89. data/lib/aspera/keychain/factory.rb +3 -3
  90. data/lib/aspera/keychain/macos_security.rb +3 -3
  91. data/lib/aspera/link_header.rb +82 -0
  92. data/lib/aspera/log.rb +17 -3
  93. data/lib/aspera/markdown.rb +85 -2
  94. data/lib/aspera/node_simulator.rb +3 -1
  95. data/lib/aspera/oauth/base.rb +5 -5
  96. data/lib/aspera/oauth/boot.rb +2 -2
  97. data/lib/aspera/oauth/factory.rb +7 -5
  98. data/lib/aspera/oauth/jwt.rb +4 -4
  99. data/lib/aspera/oauth/web.rb +2 -2
  100. data/lib/aspera/persistency_action_once.rb +8 -8
  101. data/lib/aspera/persistency_folder.rb +10 -3
  102. data/lib/aspera/preview/file_types.rb +3 -3
  103. data/lib/aspera/preview/generator.rb +3 -3
  104. data/lib/aspera/preview/utils.rb +1 -1
  105. data/lib/aspera/products/connect.rb +1 -1
  106. data/lib/aspera/products/other.rb +1 -1
  107. data/lib/aspera/products/transferd.rb +1 -1
  108. data/lib/aspera/proxy_auto_config.rb +7 -6
  109. data/lib/aspera/rest.rb +10 -36
  110. data/lib/aspera/rest_error_analyzer.rb +3 -3
  111. data/lib/aspera/rest_errors_aspera.rb +0 -10
  112. data/lib/aspera/rest_list.rb +3 -3
  113. data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +1853 -510
  114. data/lib/aspera/schema/documentation.rb +57 -26
  115. data/lib/aspera/schema/reader.rb +59 -10
  116. data/lib/aspera/schema/registry.rb +39 -6
  117. data/lib/aspera/secret_hider.rb +7 -2
  118. data/lib/aspera/ssl.rb +4 -4
  119. data/lib/aspera/sync/conf.schema.yaml +2 -2
  120. data/lib/aspera/sync/operations.rb +10 -13
  121. data/lib/aspera/temp_file_manager.rb +1 -1
  122. data/lib/aspera/timer_limiter.rb +1 -1
  123. data/lib/aspera/transfer/faux_file.rb +24 -11
  124. data/lib/aspera/transfer/parameters.rb +23 -21
  125. data/lib/aspera/transfer/result.rb +74 -0
  126. data/lib/aspera/transfer/resumer.rb +6 -6
  127. data/lib/aspera/transfer/spec.rb +18 -1
  128. data/lib/aspera/transfer/spec.schema.yaml +12 -3
  129. data/lib/aspera/uri_reader.rb +2 -2
  130. data/lib/aspera/web_auth.rb +2 -2
  131. data/lib/aspera/web_server_simple.rb +8 -5
  132. data/lib/aspera/yaml.rb +2 -1
  133. data.tar.gz.sig +0 -0
  134. metadata +25 -3
  135. metadata.gz.sig +0 -0
  136. data/lib/aspera/json_rpc.rb +0 -52
@@ -1,5 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require 'aspera/assert'
3
4
  require 'aspera/cli/plugins/basic_auth'
4
5
  require 'aspera/nagios'
5
6
 
@@ -70,11 +71,114 @@ module Aspera
70
71
  super
71
72
  end
72
73
 
74
+ # --- DSL ---
75
+
76
+ command :health, description: 'Check Console API health', setup: :setup_api
77
+ command :transfer, description: 'Manage transfers', setup: :setup_api
78
+
79
+ commands_under(:transfer) do
80
+ command :current, description: 'Manage current transfers'
81
+ command :smart, description: 'Manage smart transfers'
82
+ end
83
+
84
+ commands_under(%i[transfer current]) do
85
+ command :list, description: 'List current transfers'
86
+ command :show, description: 'Show a transfer',
87
+ arguments: [{name: :transfer_id, type: :identifier}],
88
+ action: ->(api_console:, transfer_id:, **){Result::SingleObject.new(api_console.read("transfers/#{transfer_id}"))}
89
+ command :files, description: 'List files in a transfer',
90
+ arguments: [{name: :transfer_id, type: :identifier}]
91
+ command :start, description: 'Start a transfer',
92
+ arguments: [{name: :transfer_id, type: :identifier}]
93
+ command :pause, description: 'Pause a transfer',
94
+ arguments: [{name: :transfer_id, type: :identifier}]
95
+ command :cancel, description: 'Cancel a transfer',
96
+ arguments: [{name: :transfer_id, type: :identifier}]
97
+ command :resume, description: 'Resume a transfer',
98
+ arguments: [{name: :transfer_id, type: :identifier}]
99
+ command :rerun, description: 'Rerun a transfer',
100
+ arguments: [{name: :transfer_id, type: :identifier}]
101
+ command :change_rate, description: 'Change transfer rate',
102
+ arguments: [{name: :transfer_id, type: :identifier}]
103
+ command :change_policy, description: 'Change transfer policy',
104
+ arguments: [{name: :transfer_id, type: :identifier}]
105
+ command :move_forwards, description: 'Move transfer forwards',
106
+ arguments: [{name: :transfer_id, type: :identifier}]
107
+ command :move_back, description: 'Move transfer backwards',
108
+ arguments: [{name: :transfer_id, type: :identifier}]
109
+ end
110
+
111
+ # Generate one handler per transfer/current action.
112
+ # Convention: action_transfer_current_<verb>
113
+ # All share the same REST pattern: PATCH transfers/<id>/<verb>.
114
+ %i[start pause cancel resume rerun change_rate change_policy move_forwards move_back].each do |verb|
115
+ define_action_method([:transfer, :current, verb]) do |api_console:, transfer_id:, **|
116
+ Result::SingleObject.new(api_console.update("transfers/#{transfer_id}/#{verb}", query_read_delete))
117
+ end
118
+ end
119
+
120
+ commands_under(%i[transfer smart]) do
121
+ command :list, description: 'List smart transfers', action: ->(api_console:){Result::ObjectList.new(api_console.read('smart_transfers'))}
122
+ command :submit, description: 'Submit a smart transfer',
123
+ arguments: [{name: :smart_id}, {name: :transfer_params, type: Hash}]
124
+ end
125
+
126
+ # --- setup ---
127
+
128
+ # Build the Console REST API.
129
+ # @return [Hash] ctx with :api_console
130
+ def setup_api(**)
131
+ {api_console: basic_auth_api('api')}
132
+ end
133
+
134
+ # --- health ---
135
+
136
+ def action_health(api_console:)
137
+ nagios = Nagios.new
138
+ begin
139
+ api_console.read('ssh_keys')
140
+ nagios.add_ok('console api', 'accessible')
141
+ rescue StandardError => e
142
+ nagios.add_critical('console api', e.to_s)
143
+ end
144
+ Result::ObjectList.new(nagios.status_list)
145
+ end
146
+
147
+ # --- transfer current ---
148
+
149
+ def action_transfer_current_list(api_console:)
150
+ query = query_read_delete(default: {})
151
+ if query['from'].nil? && query['to'].nil?
152
+ time_now = Time.now
153
+ query['from'] = self.class.time_to_string(time_now - DEFAULT_FILTER_AGE_SECONDS)
154
+ query['to'] = self.class.time_to_string(time_now)
155
+ end
156
+ parse_extended_filter(query.delete('filter'), query) if query['filter']
157
+ Result::ObjectList.new(
158
+ api_console.read('transfers', query),
159
+ fields: %w[id contact name status]
160
+ )
161
+ end
162
+
163
+ def action_transfer_current_files(api_console:, transfer_id:, **)
164
+ query = query_read_delete(default: {})
165
+ query['limit'] ||= 100
166
+ Result::ObjectList.new(api_console.read("transfers/#{transfer_id}/files", query))
167
+ end
168
+
169
+ # --- transfer smart ---
170
+
171
+ def action_transfer_smart_submit(api_console:, smart_id:, transfer_params:, **)
172
+ Result::ObjectList.new(api_console.create("smart_transfers/#{smart_id}", transfer_params))
173
+ end
174
+
175
+ private
176
+
73
177
  def parse_extended_filter(filter, query)
74
- raise BadArgument, "Invalid filter syntax: #{filter}, shall be (field op val)and(field op val)..." unless filter.start_with?('(') && filter.end_with?(')')
178
+ Aspera.assert(filter.start_with?('(') && filter.end_with?(')'), type: BadArgument){"Invalid filter syntax: #{filter}, shall be (field op val)and(field op val)..."}
75
179
  filter[1..-2].split(')and(').each_with_index do |expr, i|
76
180
  m = expr.match(EXPR_RE)
77
- raise BadArgument, "Invalid expression: #{expr}, shall be: <field> <op> <val>" unless m
181
+ Aspera.assert(m, type: BadArgument){"Invalid expression: #{expr}, shall be: <field> <op> <val>"}
78
182
  t = m.captures
79
183
  i += 1
80
184
  query["filter#{i}"] = t[0]
@@ -82,68 +186,6 @@ module Aspera
82
186
  query["val#{i}"] = t[2]
83
187
  end
84
188
  end
85
-
86
- ACTIONS = %i[transfer health].freeze
87
-
88
- def execute_action
89
- api_console = basic_auth_api('api')
90
- command = options.get_next_command(ACTIONS)
91
- case command
92
- when :health
93
- nagios = Nagios.new
94
- begin
95
- api_console.read('ssh_keys')
96
- nagios.add_ok('console api', 'accessible')
97
- rescue StandardError => e
98
- nagios.add_critical('console api', e.to_s)
99
- end
100
- Result::ObjectList.new(nagios.status_list)
101
- when :transfer
102
- command = options.get_next_command(%i[current smart])
103
- case command
104
- when :smart
105
- command = options.get_next_command(%i[list submit])
106
- case command
107
- when :list
108
- return Result::ObjectList.new(api_console.read('smart_transfers'))
109
- when :submit
110
- smart_id = options.get_next_argument('smart_id')
111
- params = options.get_next_argument('transfer parameters', validation: Hash)
112
- return Result::ObjectList.new(api_console.create("smart_transfers/#{smart_id}", params))
113
- end
114
- when :current
115
- command = options.get_next_command(%i[list show files start pause cancel resume rerun change_rate change_policy move_forwards move_back])
116
- case command
117
- when :list
118
- # https://developer.ibm.com/apis/catalog/aspera--aspera-console-rest-api/Developer+Guides#transfer-list
119
- query = query_read_delete(default: {})
120
- if query['from'].nil? && query['to'].nil?
121
- time_now = Time.now
122
- query['from'] = self.class.time_to_string(time_now - DEFAULT_FILTER_AGE_SECONDS)
123
- query['to'] = self.class.time_to_string(time_now)
124
- end
125
- if (filter = query.delete('filter'))
126
- parse_extended_filter(filter, query)
127
- end
128
- return Result::ObjectList.new(
129
- api_console.read('transfers', query),
130
- fields: %w[id contact name status]
131
- )
132
- when :show
133
- transfer_id = options.instance_identifier(description: 'transfer ID')
134
- return Result::SingleObject.new(api_console.read("transfers/#{transfer_id}"))
135
- when :files
136
- transfer_id = options.instance_identifier(description: 'transfer ID')
137
- query = query_read_delete(default: {})
138
- query['limit'] ||= 100
139
- return Result::ObjectList.new(api_console.read("transfers/#{transfer_id}/files", query))
140
- when :start, :pause, :cancel, :resume, :rerun, :change_rate, :change_policy, :move_forwards, :move_back
141
- transfer_id = options.instance_identifier(description: 'transfer ID')
142
- return Result::SingleObject.new(api_console.update("transfers/#{transfer_id}/#{command}", query_read_delete))
143
- end
144
- end
145
- end
146
- end
147
189
  end
148
190
  end
149
191
  end
@@ -9,44 +9,56 @@ module Aspera
9
9
  module Cli
10
10
  module Plugins
11
11
  class Cos < Base
12
+ application_name 'IBM Cloud Object Storage'
13
+
14
+ command :node, description: 'Execute COS node commands', setup: :setup_cos_node
15
+
16
+ commands_under(:node) do
17
+ Node::COMMANDS_COS.each do |cmd|
18
+ command(cmd, description: "Node #{cmd} command")
19
+ end
20
+ end
21
+
22
+ option :bucket, description: 'Bucket name'
23
+ option :endpoint, description: 'Storage endpoint (URL)'
24
+ option :apikey, description: 'Storage API key'
25
+ option :crn, description: 'Resource instance id (CRN)'
26
+ option :service_credentials, description: 'IBM Cloud service credentials', allowed: [Hash, NilClass]
27
+ option :region, description: 'Storage region'
28
+ option :identity, description: "Authentication URL (#{Api::CosNode::IBM_CLOUD_TOKEN_URL})", default: Api::CosNode::IBM_CLOUD_TOKEN_URL
29
+
12
30
  def initialize(**_)
13
31
  super
14
- options.declare(:bucket, 'Bucket name')
15
- options.declare(:endpoint, 'Storage endpoint (URL)')
16
- options.declare(:apikey, 'Storage API key')
17
- options.declare(:crn, 'Resource instance id (CRN)')
18
- options.declare(:service_credentials, 'IBM Cloud service credentials', allowed: [Hash, NilClass])
19
- options.declare(:region, 'Storage region')
20
- options.declare(:identity, "Authentication URL (#{Api::CosNode::IBM_CLOUD_TOKEN_URL})", default: Api::CosNode::IBM_CLOUD_TOKEN_URL)
21
32
  options.parse_options!
22
33
  Node.declare_options(options)
23
34
  end
24
35
 
25
- ACTIONS = %i[node].freeze
26
-
27
- def execute_action
28
- command = options.get_next_command(ACTIONS)
29
- case command
30
- when :node
31
- # get service credentials, Hash, e.g. @json:@file:...
32
- service_credentials = options.get_option(:service_credentials)
33
- cos_node_params = {
34
- auth_url: options.get_option(:identity, mandatory: true),
35
- bucket: options.get_option(:bucket, mandatory: true),
36
- endpoint: options.get_option(:endpoint)
37
- }
38
- if service_credentials.nil?
39
- Aspera.assert(!cos_node_params[:endpoint].nil?, 'endpoint required when service credentials not provided', type: Cli::BadArgument)
40
- cos_node_params[:api_key] = options.get_option(:apikey, mandatory: true)
41
- cos_node_params[:instance_id] = options.get_option(:crn, mandatory: true)
42
- else
43
- Aspera.assert(cos_node_params[:endpoint].nil?, 'endpoint not allowed when service credentials provided', type: Cli::BadArgument)
44
- cos_node_params.merge!(Api::CosNode.parameters_from_svc_credentials(service_credentials, options.get_option(:region, mandatory: true)))
45
- end
46
- api_node = Api::CosNode.new(**cos_node_params)
47
- node_plugin = Node.new(context: context, api: api_node)
48
- command = options.get_next_command(Node::COMMANDS_COS)
49
- return node_plugin.execute_action(command)
36
+ # Build the COS Node API and plugin from CLI options.
37
+ # @return [Hash] context hash containing :node_plugin
38
+ def setup_cos_node(**)
39
+ # get service credentials, Hash, e.g. @json:@file:...
40
+ service_credentials = options.get_option(:service_credentials)
41
+ cos_node_params = {
42
+ auth_url: options.get_option(:identity, mandatory: true),
43
+ bucket: options.get_option(:bucket, mandatory: true),
44
+ endpoint: options.get_option(:endpoint)
45
+ }
46
+ if service_credentials.nil?
47
+ Aspera.assert(!cos_node_params[:endpoint].nil?, 'endpoint required when service credentials not provided', type: Cli::BadArgument)
48
+ cos_node_params[:api_key] = options.get_option(:apikey, mandatory: true)
49
+ cos_node_params[:instance_id] = options.get_option(:crn, mandatory: true)
50
+ else
51
+ Aspera.assert(cos_node_params[:endpoint].nil?, 'endpoint not allowed when service credentials provided', type: Cli::BadArgument)
52
+ cos_node_params.merge!(Api::CosNode.parameters_from_svc_credentials(service_credentials, options.get_option(:region, mandatory: true)))
53
+ end
54
+ api_node = Api::CosNode.new(**cos_node_params)
55
+ {node_plugin: Node.new(context: context, api: api_node)}
56
+ end
57
+
58
+ # One handler per COMMANDS_COS command - delegates to the Node plugin's dispatch_v3_command.
59
+ Node::COMMANDS_COS.each do |cmd|
60
+ define_action_method([:node, cmd]) do |node_plugin:|
61
+ node_plugin.dispatch_v3_command(cmd)
50
62
  end
51
63
  end
52
64
  end
@@ -30,7 +30,7 @@ module Aspera
30
30
 
31
31
  # add a folder to the list of folders to look for plugins
32
32
  def add_lookup_folder(folder)
33
- @lookup_folders.unshift(folder)
33
+ @lookup_folders.unshift(folder) unless @lookup_folders.include?(folder)
34
34
  end
35
35
 
36
36
  # find plugins in defined paths
@@ -45,19 +45,19 @@ module Aspera
45
45
  plugin_symbol = File.basename(path, Environment::RB_EXT).to_sym
46
46
  next if IGNORE_PLUGINS.include?(plugin_symbol)
47
47
  req = path.sub(/#{Environment::RB_EXT}$/o, '')
48
- Aspera.assert(!@plugins.key?(plugin_symbol), type: :warn){"Plugin already registered: #{plugin_symbol}"}
48
+ next if @plugins.key?(plugin_symbol)
49
49
  @plugins[plugin_symbol] = {source: path, require_stanza: req}
50
50
  end
51
51
  end
52
52
  end
53
53
 
54
- # @return path to source file of plugin
54
+ # @return [String] path to source file of plugin
55
55
  def plugin_source(plugin_name_sym)
56
56
  Aspera.assert(@plugins.key?(plugin_name_sym), type: NoSuchElement){"plugin not found: #{plugin_name_sym}"}
57
57
  @plugins[plugin_name_sym][:source]
58
58
  end
59
59
 
60
- # @return Class object for plugin
60
+ # @return [Class] class object for plugin
61
61
  def plugin_class(plugin_name_sym)
62
62
  Aspera.assert(@plugins.key?(plugin_name_sym), type: NoSuchElement){"plugin not found: #{plugin_name_sym}"}
63
63
  require @plugins[plugin_name_sym][:require_stanza]