aspera-cli 4.28.0 → 4.29.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/BUGS.md +5 -10
- data/CHANGELOG.md +217 -2
- data/CODE_OF_CONDUCT.md +18 -0
- data/CONTRIBUTING.md +135 -56
- data/GOVERNANCE.md +30 -0
- data/LICENSE +201 -0
- data/README.md +122 -26
- data/SECURITY.md +84 -0
- data/bin/ascli +8 -5
- data/bin/asession +9 -6
- data/docs/README.md +1895 -632
- data/docs/test-mcp-with-ai.md +8 -8
- data/lib/aspera/agent/base.rb +23 -2
- data/lib/aspera/agent/connect.rb +16 -12
- data/lib/aspera/agent/desktop.rb +10 -7
- data/lib/aspera/agent/direct.rb +20 -19
- data/lib/aspera/agent/factory.rb +6 -3
- data/lib/aspera/agent/httpgw.rb +7 -5
- data/lib/aspera/agent/node.rb +15 -7
- data/lib/aspera/agent/transferd.rb +167 -43
- data/lib/aspera/api/alee.rb +4 -1
- data/lib/aspera/api/aoc.rb +14 -11
- data/lib/aspera/api/ats.rb +4 -2
- data/lib/aspera/api/cos_node.rb +7 -4
- data/lib/aspera/api/faspex.rb +16 -9
- data/lib/aspera/api/httpgw.rb +9 -6
- data/lib/aspera/api/node.rb +29 -22
- data/lib/aspera/ascmd.rb +8 -5
- data/lib/aspera/ascp/installation.rb +95 -67
- data/lib/aspera/ascp/log.schema.yaml +1084 -0
- data/lib/aspera/ascp/log_analyzer/advice.rb +97 -0
- data/lib/aspera/ascp/log_analyzer/diagnostics.rb +344 -0
- data/lib/aspera/ascp/log_analyzer/lines.rb +75 -0
- data/lib/aspera/ascp/log_analyzer/log_file.rb +593 -0
- data/lib/aspera/ascp/log_analyzer/records.rb +379 -0
- data/lib/aspera/ascp/log_analyzer/rules.rb +172 -0
- data/lib/aspera/ascp/log_analyzer/session.rb +782 -0
- data/lib/aspera/ascp/log_analyzer.rb +370 -0
- data/lib/aspera/ascp/log_locations.rb +163 -0
- data/lib/aspera/ascp/management.rb +16 -8
- data/lib/aspera/ascp/session_errors.rb +217 -0
- data/lib/aspera/assert.rb +4 -0
- data/lib/aspera/cli/ascp_actions.rb +12 -19
- data/lib/aspera/cli/ascp_logs_actions.rb +188 -0
- data/lib/aspera/cli/async_transfer_store.rb +7 -4
- data/lib/aspera/cli/bootstrapper.rb +38 -33
- data/lib/aspera/cli/command_line.rb +63 -8
- data/lib/aspera/cli/command_registry.rb +9 -1
- data/lib/aspera/cli/command_spec.rb +47 -7
- data/lib/aspera/cli/completion/ascli.bash +19 -5
- data/lib/aspera/cli/completion/ascli.fish +13 -2
- data/lib/aspera/cli/completion/ascli.ps1 +31 -9
- data/lib/aspera/cli/completion/ascli.zsh +22 -6
- data/lib/aspera/cli/completion_actions.rb +303 -69
- data/lib/aspera/cli/context.rb +9 -4
- data/lib/aspera/cli/deprecation.rb +5 -2
- data/lib/aspera/cli/error.rb +6 -0
- data/lib/aspera/cli/extended_value.rb +66 -31
- data/lib/aspera/cli/formatter.rb +43 -30
- data/lib/aspera/cli/gem_checker.rb +9 -6
- data/lib/aspera/cli/hints.rb +22 -10
- data/lib/aspera/cli/http.rb +38 -10
- data/lib/aspera/cli/info.rb +12 -7
- data/lib/aspera/cli/mailer.rb +6 -3
- data/lib/aspera/cli/mcp/http_servlet.rb +7 -8
- data/lib/aspera/cli/mcp/secrets.rb +8 -5
- data/lib/aspera/cli/mcp/tool.rb +20 -17
- data/lib/aspera/cli/option_declarator.rb +10 -4
- data/lib/aspera/cli/option_registry.rb +6 -3
- data/lib/aspera/cli/option_types.rb +17 -2
- data/lib/aspera/cli/option_value.rb +77 -18
- data/lib/aspera/cli/options.schema.yaml +641 -80
- data/lib/aspera/cli/parser.rb +71 -35
- data/lib/aspera/cli/plugins/alee.rb +7 -4
- data/lib/aspera/cli/plugins/aoc.rb +248 -37
- data/lib/aspera/cli/plugins/ats.rb +10 -9
- data/lib/aspera/cli/plugins/base.rb +50 -18
- data/lib/aspera/cli/plugins/basic_auth.rb +5 -2
- data/lib/aspera/cli/plugins/config.rb +210 -104
- data/lib/aspera/cli/plugins/console.rb +6 -3
- data/lib/aspera/cli/plugins/cos.rb +8 -4
- data/lib/aspera/cli/plugins/factory.rb +21 -7
- data/lib/aspera/cli/plugins/faspex5.rb +31 -26
- data/lib/aspera/cli/plugins/faspio.rb +7 -5
- data/lib/aspera/cli/plugins/httpgw.rb +7 -4
- data/lib/aspera/cli/plugins/mcp.rb +42 -22
- data/lib/aspera/cli/plugins/node.rb +39 -38
- data/lib/aspera/cli/plugins/oauth.rb +23 -6
- data/lib/aspera/cli/plugins/orchestrator.rb +287 -83
- data/lib/aspera/cli/plugins/preview.rb +72 -45
- data/lib/aspera/cli/plugins/server.rb +66 -14
- data/lib/aspera/cli/plugins/shares.rb +87 -18
- data/lib/aspera/cli/plugins/transferd.rb +228 -0
- data/lib/aspera/cli/poll.rb +236 -0
- data/lib/aspera/cli/preset_actions.rb +144 -30
- data/lib/aspera/cli/preset_manager.rb +113 -35
- data/lib/aspera/cli/prompt.rb +157 -9
- data/lib/aspera/cli/result.rb +12 -8
- data/lib/aspera/cli/runner.rb +89 -48
- data/lib/aspera/cli/secret_finder.rb +4 -1
- data/lib/aspera/cli/security.rb +89 -9
- data/lib/aspera/cli/special_values.rb +4 -1
- data/lib/aspera/cli/sync_actions.rb +15 -10
- data/lib/aspera/cli/table_fitter.rb +3 -0
- data/lib/aspera/cli/terminal_formatter.rb +9 -6
- data/lib/aspera/cli/transfer_actions.rb +11 -8
- data/lib/aspera/cli/transfer_agent.rb +43 -20
- data/lib/aspera/cli/transfer_progress.rb +14 -45
- data/lib/aspera/cli/vault_actions.rb +45 -0
- data/lib/aspera/cli/vault_manager.rb +188 -42
- data/lib/aspera/cli/version.rb +4 -1
- data/lib/aspera/cli/web_secret.rb +12 -10
- data/lib/aspera/cli/wizard.rb +42 -39
- data/lib/aspera/command_line_builder.rb +9 -5
- data/lib/aspera/command_line_converter.rb +4 -1
- data/lib/aspera/coverage.rb +103 -33
- data/lib/aspera/data_repository.rb +4 -1
- data/lib/aspera/dot_container.rb +4 -1
- data/lib/aspera/environment.rb +23 -12
- data/lib/aspera/exec_spec.rb +3 -0
- data/lib/aspera/faspex_gw.rb +13 -9
- data/lib/aspera/faspex_postproc.rb +15 -12
- data/lib/aspera/formatter_interface.rb +4 -1
- data/lib/aspera/graphql.rb +5 -1
- data/lib/aspera/hash_ext.rb +3 -0
- data/lib/aspera/id_generator.rb +20 -5
- data/lib/aspera/inflate.rb +43 -0
- data/lib/aspera/json_rpc/client.rb +6 -5
- data/lib/aspera/json_rpc/version.rb +3 -0
- data/lib/aspera/line_logger.rb +4 -1
- data/lib/aspera/link_header.rb +4 -1
- data/lib/aspera/log.rb +18 -5
- data/lib/aspera/markdown.rb +5 -2
- data/lib/aspera/mime.rb +3 -0
- data/lib/aspera/nagios.rb +6 -4
- data/lib/aspera/node_emulator.rb +15 -13
- data/lib/aspera/oauth/client/base.rb +124 -73
- data/lib/aspera/oauth/client/boot.rb +7 -4
- data/lib/aspera/oauth/client/factory.rb +41 -18
- data/lib/aspera/oauth/client/generic.rb +5 -1
- data/lib/aspera/oauth/client/json_credentials.rb +7 -3
- data/lib/aspera/oauth/client/jwt.rb +6 -3
- data/lib/aspera/oauth/client/shares_web.rb +94 -0
- data/lib/aspera/oauth/client/url_json.rb +7 -3
- data/lib/aspera/oauth/client/web.rb +40 -13
- data/lib/aspera/oauth/server/pkce.rb +6 -5
- data/lib/aspera/orchestrator_workflow_checker.rb +129 -2
- data/lib/aspera/persistency_action_once.rb +5 -3
- data/lib/aspera/persistency_folder.rb +6 -3
- data/lib/aspera/preview/file_types.rb +6 -3
- data/lib/aspera/preview/generator.rb +9 -5
- data/lib/aspera/preview/options.schema.yaml +3 -3
- data/lib/aspera/preview/terminal.rb +16 -6
- data/lib/aspera/preview/utils.rb +6 -4
- data/lib/aspera/products/connect.rb +5 -2
- data/lib/aspera/products/desktop.rb +17 -2
- data/lib/aspera/products/other.rb +4 -2
- data/lib/aspera/products/transferd.rb +6 -2
- data/lib/aspera/products/transferd_service.rb +610 -0
- data/lib/aspera/proxy_auto_config.js +2 -2
- data/lib/aspera/proxy_auto_config.rb +5 -2
- data/lib/aspera/rainbow.rb +3 -0
- data/lib/aspera/rest/aspera_errors.rb +4 -1
- data/lib/aspera/rest/call_error.rb +3 -0
- data/lib/aspera/rest/client.rb +45 -34
- data/lib/aspera/rest/error_analyzer.rb +5 -2
- data/lib/aspera/rest/list.rb +8 -4
- data/lib/aspera/rest/parameters.rb +3 -0
- data/lib/aspera/rest/util.rb +8 -5
- data/lib/aspera/rest.rb +9 -6
- data/lib/aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml +9 -9
- data/lib/aspera/schema/IBM Aspera Orchestrator API-v1-enhanced.yaml +118 -30
- data/lib/aspera/schema/IBM Aspera on Cloud API-0.2.6-enhanced.yaml +11 -11
- data/lib/aspera/schema/IBM Aspera on Cloud Automation API-1.0.5-enhanced.yaml +2437 -1437
- data/lib/aspera/schema/async_tables.yaml +11 -10
- data/lib/aspera/schema/documentation.rb +8 -2
- data/lib/aspera/schema/reader.rb +7 -0
- data/lib/aspera/schema/registry.rb +44 -28
- data/lib/aspera/schema/transferd-config.schema.json +333 -0
- data/lib/aspera/schema/validator.rb +4 -1
- data/lib/aspera/sealed_hash.rb +150 -0
- data/lib/aspera/secret/hider.rb +213 -0
- data/lib/aspera/secret/reference.rb +150 -0
- data/lib/aspera/secret/resolver.rb +212 -0
- data/lib/aspera/secret/store/base.rb +153 -0
- data/lib/aspera/secret/store/encrypted_hash.rb +329 -0
- data/lib/aspera/secret/store/factory.rb +103 -0
- data/lib/aspera/secret/store/hashicorp_vault.rb +390 -0
- data/lib/aspera/secret/store/keeper.rb +141 -0
- data/lib/aspera/secret/store/macos_security.rb +284 -0
- data/lib/aspera/secret/store/one_password_api.rb +99 -0
- data/lib/aspera/secret/store/one_password_base.rb +42 -0
- data/lib/aspera/secret/store/one_password_cli.rb +112 -0
- data/lib/aspera/secret/store/secret_service.rb +125 -0
- data/lib/aspera/secret/strength.rb +119 -0
- data/lib/aspera/servlet_helpers.rb +5 -2
- data/lib/aspera/ssh.rb +10 -7
- data/lib/aspera/ssl.rb +5 -2
- data/lib/aspera/string_ext.rb +3 -0
- data/lib/aspera/sync/args.schema.yaml +3 -0
- data/lib/aspera/sync/conf.schema.yaml +9 -4
- data/lib/aspera/sync/database.rb +3 -0
- data/lib/aspera/sync/operations.rb +17 -15
- data/lib/aspera/temp_file_manager.rb +8 -4
- data/lib/aspera/timer_limiter.rb +3 -0
- data/lib/aspera/transfer/error.rb +4 -1
- data/lib/aspera/transfer/faux_file.rb +4 -1
- data/lib/aspera/transfer/parameters.rb +17 -14
- data/lib/aspera/transfer/result.rb +23 -0
- data/lib/aspera/transfer/resumer.rb +6 -4
- data/lib/aspera/transfer/spec.rb +9 -22
- data/lib/aspera/units/decimal.rb +67 -0
- data/lib/aspera/units/duration.rb +109 -0
- data/lib/aspera/uri_reader.rb +64 -9
- data/lib/aspera/web_auth/index.html.erb +6 -2
- data/lib/aspera/web_auth/paste.rb +109 -0
- data/lib/aspera/web_auth/receiver.rb +50 -0
- data/lib/aspera/web_auth/relay.rb +76 -0
- data/lib/aspera/web_auth/reply.html.erb +150 -0
- data/lib/aspera/web_auth/server.rb +47 -10
- data/lib/aspera/web_page.rb +7 -2
- data/lib/aspera/web_server_simple.rb +6 -3
- data/lib/aspera/yaml.rb +5 -1
- data/llms-install.md +88 -0
- data/llms.txt +87 -0
- data/skills/ascli/SKILL.md +94 -0
- data.tar.gz.sig +0 -0
- metadata +77 -20
- metadata.gz.sig +0 -0
- data/lib/aspera/duration.rb +0 -42
- data/lib/aspera/keychain/base.rb +0 -30
- data/lib/aspera/keychain/encrypted_hash.rb +0 -192
- data/lib/aspera/keychain/factory.rb +0 -58
- data/lib/aspera/keychain/hashicorp_vault.rb +0 -77
- data/lib/aspera/keychain/macos_security.rb +0 -277
- data/lib/aspera/keychain/one_password_api.rb +0 -91
- data/lib/aspera/keychain/one_password_base.rb +0 -39
- data/lib/aspera/keychain/one_password_cli.rb +0 -100
- data/lib/aspera/secret_hider.rb +0 -196
- data/lib/aspera/secret_strength.rb +0 -114
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
3
6
|
require 'psych'
|
|
4
7
|
require 'rexml/document'
|
|
5
8
|
require 'stringio'
|
|
@@ -42,6 +45,9 @@ module Aspera
|
|
|
42
45
|
# Element 6 (plugin versions) is informational: Orchestrator 4.1.6 only reads elements 1 to 5
|
|
43
46
|
ELEMENT_COUNTS = [5, 6].freeze
|
|
44
47
|
TRUE_VALUES = %w[true 1 t].freeze
|
|
48
|
+
# SimpleParameterLookup: separator of the columns of the table, input that returns the table indexed by a column
|
|
49
|
+
LOOKUP_DELIMITER = ':::'
|
|
50
|
+
LOOKUP_INDEX_INPUT = 'KeyForTableIndexing'
|
|
45
51
|
UNNAMED = ['', 'undefined', 'null'].freeze
|
|
46
52
|
SUCCESS_STATUSES = %w[complete true].freeze
|
|
47
53
|
BOOLEAN_TEXT = %w[true false].freeze
|
|
@@ -64,9 +70,12 @@ module Aspera
|
|
|
64
70
|
TABLE_SEPARATOR = ':::'
|
|
65
71
|
# Elements of a step that a `Map_to` of type `output` gives as attributes (in lower case)
|
|
66
72
|
STEP_IDENTITY = %w[Step_name Step_type Action_id].freeze
|
|
73
|
+
# Maximum size of the entries of a package once decompressed: a small package can decompress to gigabytes (decompression bomb)
|
|
74
|
+
PACKAGE_MAX_SIZE = 256 * 1024 * 1024
|
|
75
|
+
private_constant :PACKAGE_MAX_SIZE
|
|
67
76
|
private_constant :WORKFLOW_ATTRIBUTES, :WHITEBOARD_ATTRIBUTES, :VALUE_TYPES, :ELEMENT_COUNTS, :TRUE_VALUES, :UNNAMED, :SUCCESS_STATUSES,
|
|
68
77
|
:BOOLEAN_TEXT, :STEP_CHILDREN, :PARAMETER_MAPPINGS, :TEMPLATE_COLUMNS, :FALSE_VALUES, :INTEGER_TYPES, :SPEC_CODE, :PARTLY_DEFINED, :PLACEHOLDER,
|
|
69
|
-
:STEP_IDENTITY, :SINGLE_NAME, :TABLE_SEPARATOR
|
|
78
|
+
:STEP_IDENTITY, :SINGLE_NAME, :TABLE_SEPARATOR, :LOOKUP_DELIMITER, :LOOKUP_INDEX_INPUT
|
|
70
79
|
|
|
71
80
|
class << self
|
|
72
81
|
# @param file_name [String] Name of the file, ending with `yml` or `wkf`
|
|
@@ -116,10 +125,16 @@ module Aspera
|
|
|
116
125
|
# @param content [String] Bytes of a `.wkf` file: gzipped tar with a manifest (list of files, the first one is the main workflow)
|
|
117
126
|
def check_package(content)
|
|
118
127
|
files = {}
|
|
128
|
+
size = 0
|
|
119
129
|
begin
|
|
120
130
|
Zlib::GzipReader.wrap(StringIO.new(content)) do |gzip|
|
|
121
131
|
Gem::Package::TarReader.new(gzip) do |tar|
|
|
122
|
-
tar.each
|
|
132
|
+
tar.each do |entry|
|
|
133
|
+
# Size declared in the header: checked before the entry is decompressed (read, or skipped)
|
|
134
|
+
size += entry.header.size
|
|
135
|
+
return add(ERROR, 'package.too_large', 'file', "package is too large when decompressed: more than #{PACKAGE_MAX_SIZE} bytes") if size > PACKAGE_MAX_SIZE
|
|
136
|
+
files[entry.full_name] = entry.read if entry.file?
|
|
137
|
+
end
|
|
123
138
|
end
|
|
124
139
|
end
|
|
125
140
|
rescue Zlib::Error, Gem::Package::Error => e
|
|
@@ -278,10 +293,49 @@ module Aspera
|
|
|
278
293
|
closing = terminals.values_at('End', 'Fail', 'Error').flatten
|
|
279
294
|
(closing + steps).each { |node| check_node(node, ids) }
|
|
280
295
|
check_branches(closing, steps)
|
|
296
|
+
# starvation is computed on a graph without loop
|
|
297
|
+
check_starvation(terminals['Start'].first&.attributes&.[]('id'), closing, steps) unless check_loops(closing, steps)
|
|
281
298
|
check_parameters(root, steps)
|
|
299
|
+
check_routers(root)
|
|
300
|
+
check_lookups(steps)
|
|
282
301
|
check_step_interfaces(steps) unless @specs.nil?
|
|
283
302
|
end
|
|
284
303
|
|
|
304
|
+
# SimpleParameterLookup returns a row only if an input selects it: an input named as a column of the header of the table.
|
|
305
|
+
# Without one, no row matches and the step ends with Failed (No configuration match), unless it returns the array of rows
|
|
306
|
+
# (SimpleParameterLookup#execute). The header of a table in a file (param_file_name) is not known.
|
|
307
|
+
def check_lookups(steps)
|
|
308
|
+
steps.each do |step|
|
|
309
|
+
next unless text_of(step, 'Step_type').eql?('SimpleParameterLookup')
|
|
310
|
+
values = @template_attributes["SimpleParameterLookup__#{text_of(step, 'Action_id')}"]
|
|
311
|
+
next if values.nil? || blank?(values['parameters_associations']) || !blank?(values['param_file_name'])
|
|
312
|
+
next if TRUE_VALUES.include?(values['return_array_of_rows'].to_s.downcase)
|
|
313
|
+
header = values['parameters_associations'].lines.first.to_s.strip.split(LOOKUP_DELIMITER, -1).map(&:strip)
|
|
314
|
+
# a carried variable is also given to the step under its name
|
|
315
|
+
provided = step.get_elements('Inputs/Input').map { |input| input.attributes['name'] } + step.get_elements('Outputs/Carry_thru').map { |carry| carry.attributes['name'] }
|
|
316
|
+
next if provided.include?(LOOKUP_INDEX_INPUT) || provided.intersect?(header)
|
|
317
|
+
add(
|
|
318
|
+
ERROR, 'step.lookup_without_key', node_path(step),
|
|
319
|
+
"no input is mapped to a column of the table (#{header.join(', ')}): no row matches and the step ends with Failed, unless return_array_of_rows is set"
|
|
320
|
+
)
|
|
321
|
+
end
|
|
322
|
+
end
|
|
323
|
+
|
|
324
|
+
# Line styles of links in the designer: it writes `new <router>()` in the page to draw each link (WhiteboardsHelper#connect_xmlsteps)
|
|
325
|
+
ROUTERS = %w[Manhattan Bezier Null Fan].map { |name| "draw2d.#{name}ConnectionRouter" }.freeze
|
|
326
|
+
|
|
327
|
+
# Links: prerequisites of nodes and links of parameters (Parameter in Start or Step); absent router: Manhattan
|
|
328
|
+
def check_routers(root)
|
|
329
|
+
root.each_element('//Prerequisite[@router] | //Start/Parameters/Parameter[@router] | //Step/Parameters/Parameter[@router]') do |link|
|
|
330
|
+
router = link.attributes['router']
|
|
331
|
+
next if ROUTERS.include?(router)
|
|
332
|
+
add(
|
|
333
|
+
ERROR, 'graph.router', node_path(link.parent.parent),
|
|
334
|
+
"router '#{router}' of the link from '#{link.attributes['id']}' is not one of #{ROUTERS.join(', ')}: the designer cannot draw the workflow"
|
|
335
|
+
)
|
|
336
|
+
end
|
|
337
|
+
end
|
|
338
|
+
|
|
285
339
|
def node_path(node)
|
|
286
340
|
node.name.eql?('Step') ? "graph/Step[#{text_of(node, 'Step_name')}]" : "graph/#{node.name}"
|
|
287
341
|
end
|
|
@@ -293,6 +347,9 @@ module Aspera
|
|
|
293
347
|
add(ERROR, 'graph.synch_factor', path, 'Synch_factor is missing or not an integer')
|
|
294
348
|
elsif factor.to_i.negative?
|
|
295
349
|
add(ERROR, 'graph.synch_factor_negative', path, 'negative Synch_factor is never met')
|
|
350
|
+
elsif factor.to_i.positive? && %w[x y].any? { |name| node.elements['Synch_factor'].attributes[name].to_s.empty? }
|
|
351
|
+
# the designer writes the position of the join symbol, and draws the workflow with it (WhiteboardsHelper#draw_operator)
|
|
352
|
+
add(WARNING, 'graph.synch_factor_position', path, 'Synch_factor is not 0 and has no attributes x and y (position of the join symbol): the designer shows an empty canvas')
|
|
296
353
|
end
|
|
297
354
|
node.get_elements('Prerequisites/Prerequisite').each do |pre|
|
|
298
355
|
add(ERROR, 'graph.prerequisite_unknown', path, "prerequisite refers to unknown node '#{pre.attributes['id']}'") unless ids.include?(pre.attributes['id'])
|
|
@@ -347,6 +404,76 @@ module Aspera
|
|
|
347
404
|
add(ERROR, 'graph.no_termination', 'graph', 'workflow does not terminate: End, Fail and Error have no prerequisite')
|
|
348
405
|
end
|
|
349
406
|
|
|
407
|
+
# Starvation: a node that waits for exclusive conditions never starts, and the work order never ends.
|
|
408
|
+
# Conditions of a node: pairs [step id, status] that must all be true for the node to start.
|
|
409
|
+
# With Synch_factor 0 (AND), those of all its prerequisites; otherwise (OR), those common to all its prerequisites.
|
|
410
|
+
# A prerequisite on a step adds its status to the conditions of that step.
|
|
411
|
+
# A step ends with one status: two statuses of the same step in the conditions of a node are exclusive,
|
|
412
|
+
# for example the Complete and Failed exits of a step, or an if and an else branch joined without OR.
|
|
413
|
+
# Not checked: steps after an ArrayFanout (one execution per element, with different statuses).
|
|
414
|
+
def check_starvation(start_id, closing, steps)
|
|
415
|
+
nodes = (closing + steps).to_h { |node| [node.attributes['id'], node] }
|
|
416
|
+
fanned = steps.select do |step|
|
|
417
|
+
step.get_elements('Prerequisites/Prerequisite').any? { |pre| text_of(nodes[pre.attributes['id']] || step, 'Step_type').eql?('ArrayFanout') }
|
|
418
|
+
end.map { |step| step.attributes['id'] }
|
|
419
|
+
memo = {}
|
|
420
|
+
conditions = lambda do |id, visiting|
|
|
421
|
+
return [] if id.eql?(start_id) || !nodes.key?(id) || visiting.include?(id)
|
|
422
|
+
return memo[id] if memo.key?(id)
|
|
423
|
+
node = nodes[id]
|
|
424
|
+
branches = node.get_elements('Prerequisites/Prerequisite').map do |pre|
|
|
425
|
+
source = pre.attributes['id']
|
|
426
|
+
own = source.eql?(start_id) || fanned.include?(source) ? [] : [[source, pre.attributes['status'].to_s.downcase]]
|
|
427
|
+
(own + conditions.call(source, visiting + [id])).uniq
|
|
428
|
+
end
|
|
429
|
+
memo[id] = if branches.empty? then []
|
|
430
|
+
elsif text_of(node, 'Synch_factor').to_i.zero? then branches.reduce(:|)
|
|
431
|
+
else branches.reduce(:&)
|
|
432
|
+
end
|
|
433
|
+
end
|
|
434
|
+
exclusive = ->(list) { list.group_by(&:first).select { |_, pairs| pairs.size > 1 }.transform_values { |pairs| pairs.map(&:last) } }
|
|
435
|
+
nodes.each do |id, node|
|
|
436
|
+
conflicts = exclusive.call(conditions.call(id, []))
|
|
437
|
+
# report where the conflict starts, not on the nodes that follow
|
|
438
|
+
inherited = node.get_elements('Prerequisites/Prerequisite').flat_map { |pre| exclusive.call(conditions.call(pre.attributes['id'], [])).keys }
|
|
439
|
+
conflicts.except(*inherited).each do |step_id, statuses|
|
|
440
|
+
step_name = text_of(nodes[step_id], 'Step_name')
|
|
441
|
+
add(
|
|
442
|
+
ERROR, 'graph.starvation', node_path(node),
|
|
443
|
+
"never starts: it waits for step '#{step_name}' to end with #{statuses.map(&:capitalize).join(' and ')}, which are exclusive; " \
|
|
444
|
+
'join exclusive branches with an OR (Synch_factor 1), otherwise the work order never ends'
|
|
445
|
+
)
|
|
446
|
+
end
|
|
447
|
+
end
|
|
448
|
+
end
|
|
449
|
+
|
|
450
|
+
# Loop: a link back to a node it comes from. A step runs once in a work order: it does not run again.
|
|
451
|
+
# @return [Boolean] true if a loop was found
|
|
452
|
+
def check_loops(closing, steps)
|
|
453
|
+
found = false
|
|
454
|
+
nodes = (closing + steps).to_h { |node| [node.attributes['id'], node] }
|
|
455
|
+
sources = nodes.transform_values { |node| node.get_elements('Prerequisites/Prerequisite').map { |pre| pre.attributes['id'] }.uniq & nodes.keys }
|
|
456
|
+
state = {}
|
|
457
|
+
visit = lambda do |id|
|
|
458
|
+
state[id] = :visiting
|
|
459
|
+
sources[id].each do |source|
|
|
460
|
+
if state[source].eql?(:visiting)
|
|
461
|
+
found = true
|
|
462
|
+
add(
|
|
463
|
+
ERROR, 'graph.loop', node_path(nodes[source]),
|
|
464
|
+
"link back to '#{text_of(nodes[id], 'Step_name') || nodes[id].name}' (loop): a step runs only once in a work order; " \
|
|
465
|
+
'to repeat, use a sub-workflow with retries, or an ArrayFanout'
|
|
466
|
+
)
|
|
467
|
+
elsif state[source].nil?
|
|
468
|
+
visit.call(source)
|
|
469
|
+
end
|
|
470
|
+
end
|
|
471
|
+
state[id] = :done
|
|
472
|
+
end
|
|
473
|
+
nodes.each_key { |id| visit.call(id) if state[id].nil? }
|
|
474
|
+
found
|
|
475
|
+
end
|
|
476
|
+
|
|
350
477
|
def check_parameters(root, steps)
|
|
351
478
|
linked = (root.get_elements('Start/Parameters/Parameter') + steps.flat_map { |step| step.get_elements('Parameters/Parameter') }).map { |link| link.attributes['id'] }
|
|
352
479
|
defined = {}
|
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require_relative 'assert'
|
|
3
7
|
require 'json'
|
|
4
|
-
require 'aspera/log'
|
|
5
|
-
require 'aspera/assert'
|
|
6
8
|
|
|
7
9
|
module Aspera
|
|
8
10
|
# Persist data on file system
|
|
@@ -11,7 +13,7 @@ module Aspera
|
|
|
11
13
|
PARSE_DEFAULT = lambda { |t| JSON.parse(t) }
|
|
12
14
|
FORMAT_DEFAULT = lambda { |h| JSON.generate(h) }
|
|
13
15
|
MERGE_DEFAULT = lambda { |current, file| current.concat(file).uniq rescue current }
|
|
14
|
-
MANAGER_METHODS = %i[get put delete]
|
|
16
|
+
MANAGER_METHODS = %i[get put delete].freeze
|
|
15
17
|
private_constant :DELETE_DEFAULT, :PARSE_DEFAULT, :FORMAT_DEFAULT, :MERGE_DEFAULT, :MANAGER_METHODS
|
|
16
18
|
|
|
17
19
|
# @param manager [Object] Mandatory Database (must respond to get/put/delete)
|
|
@@ -1,9 +1,12 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require_relative 'log'
|
|
7
|
+
require_relative 'assert'
|
|
8
|
+
require_relative 'environment'
|
|
3
9
|
require 'fileutils'
|
|
4
|
-
require 'aspera/log'
|
|
5
|
-
require 'aspera/assert'
|
|
6
|
-
require 'aspera/environment'
|
|
7
10
|
|
|
8
11
|
# search: persistency_folder PersistencyFolder
|
|
9
12
|
|
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require_relative '../log'
|
|
7
|
+
require_relative '../assert'
|
|
5
8
|
require 'singleton'
|
|
6
9
|
require 'marcel'
|
|
7
10
|
|
|
@@ -96,7 +99,7 @@ module Aspera
|
|
|
96
99
|
private
|
|
97
100
|
|
|
98
101
|
TYPE_NOT_FOUND = 'application/octet-stream'
|
|
99
|
-
ACCEPT_CTRL_CHARS = [9, 10, 13]
|
|
102
|
+
ACCEPT_CTRL_CHARS = [9, 10, 13].freeze
|
|
100
103
|
|
|
101
104
|
# Returns true if the file looks like ASCII text (printable ASCII + \t, \r, \n, space).
|
|
102
105
|
# It reads only a small prefix (default: 64KB) and fails fast on the first bad byte.
|
|
@@ -1,14 +1,18 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
3
6
|
# ffmpeg options:
|
|
4
7
|
# spellchecker:ignore soffice pauseframes libx264 trunc bufsize muxer apng libmp3lame maxrate posterize movflags faststart
|
|
5
8
|
# spellchecker:ignore palettegen paletteuse pointsize bordercolor repage lanczos unoconv optipng reencode conv transframes
|
|
6
9
|
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
10
|
+
require_relative 'utils'
|
|
11
|
+
require_relative 'file_types'
|
|
12
|
+
require_relative '../schema/registry'
|
|
13
|
+
require_relative '../log'
|
|
14
|
+
require_relative '../assert'
|
|
15
|
+
require 'fileutils'
|
|
12
16
|
|
|
13
17
|
module Aspera
|
|
14
18
|
module Preview
|
|
@@ -48,7 +48,7 @@ properties:
|
|
|
48
48
|
type: string
|
|
49
49
|
enum: [reencode, blend, clips]
|
|
50
50
|
default: reencode
|
|
51
|
-
video_png_conv:
|
|
51
|
+
video_png_conv: # cspell:disable-line
|
|
52
52
|
description: >-
|
|
53
53
|
PNG thumbnail of video: generation method.
|
|
54
54
|
`fixed`: single frame,
|
|
@@ -92,12 +92,12 @@ properties:
|
|
|
92
92
|
type: integer
|
|
93
93
|
minimum: 1
|
|
94
94
|
default: 30
|
|
95
|
-
blend_pauseframes:
|
|
95
|
+
blend_pauseframes: # cspell:disable-line
|
|
96
96
|
description: 'MP4 video preview (`blend`): number of pause frames (repetitions of each key frame).'
|
|
97
97
|
type: integer
|
|
98
98
|
minimum: 0
|
|
99
99
|
default: 3
|
|
100
|
-
blend_transframes:
|
|
100
|
+
blend_transframes: # cspell:disable-line
|
|
101
101
|
description: 'MP4 video preview (`blend`): number of transition frames between key frames.'
|
|
102
102
|
type: integer
|
|
103
103
|
minimum: 0
|
|
@@ -1,12 +1,16 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
#
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
4
5
|
|
|
6
|
+
# cspell:words MAGICKCORE ITERM
|
|
7
|
+
|
|
8
|
+
require_relative '../assert'
|
|
9
|
+
require_relative '../log'
|
|
10
|
+
require_relative '../environment'
|
|
11
|
+
require_relative '../rainbow'
|
|
5
12
|
require 'base64'
|
|
6
13
|
require 'io/console'
|
|
7
|
-
require 'aspera/log'
|
|
8
|
-
require 'aspera/environment'
|
|
9
|
-
require 'aspera/rainbow'
|
|
10
14
|
using Rainbow
|
|
11
15
|
|
|
12
16
|
module Aspera
|
|
@@ -14,6 +18,9 @@ module Aspera
|
|
|
14
18
|
module Backend
|
|
15
19
|
# Base decoder that rescales image data to the current terminal geometry.
|
|
16
20
|
class Base
|
|
21
|
+
# Rows and columns used when the size of the terminal is unknown
|
|
22
|
+
DEFAULT_TERMINAL_SIZE = [24, 80].freeze
|
|
23
|
+
private_constant :DEFAULT_TERMINAL_SIZE
|
|
17
24
|
# @param reserve [Integer] number of terminal rows reserved for non-image output
|
|
18
25
|
# @param double [Boolean] when `true`, render two image rows in one terminal row
|
|
19
26
|
# @param font_ratio [Float] terminal font aspect ratio: height divided by width
|
|
@@ -29,7 +36,9 @@ module Aspera
|
|
|
29
36
|
# @param columns [Integer] source image width in pixels
|
|
30
37
|
# @return [Array<Integer>] scaled width and height for terminal rendering
|
|
31
38
|
def terminal_scaling(rows, columns)
|
|
32
|
-
(term_rows, term_columns) = IO.console
|
|
39
|
+
(term_rows, term_columns) = IO.console&.winsize
|
|
40
|
+
# A terminal that does not know its size (e.g. a pseudo-terminal created without size) reports 0 x 0
|
|
41
|
+
(term_rows, term_columns) = DEFAULT_TERMINAL_SIZE unless term_rows&.positive? && term_columns&.positive?
|
|
33
42
|
term_rows = [term_rows - @reserve, 2].max
|
|
34
43
|
fit_term_ratio = [term_rows.to_f * @font_ratio / rows.to_f, term_columns.to_f / columns.to_f].min
|
|
35
44
|
[(columns * fit_term_ratio).to_i, (rows * fit_term_ratio * @height_ratio / @font_ratio).to_i]
|
|
@@ -141,7 +150,8 @@ module Aspera
|
|
|
141
150
|
begin
|
|
142
151
|
Log.log.debug('Trying rmagick')
|
|
143
152
|
Backend::RMagick.new(blob, reserve: reserve, double: double, font_ratio: font_ratio).terminal_pixels
|
|
144
|
-
rescue => e
|
|
153
|
+
rescue StandardError, LoadError => e
|
|
154
|
+
# LoadError: optional gem rmagick not installed
|
|
145
155
|
Log.log.debug(e.message)
|
|
146
156
|
nil
|
|
147
157
|
end
|
data/lib/aspera/preview/utils.rb
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
#
|
|
4
|
-
|
|
5
|
-
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require_relative '../log'
|
|
7
|
+
require_relative '../assert'
|
|
6
8
|
require 'English'
|
|
7
9
|
require 'tmpdir'
|
|
8
10
|
require 'fileutils'
|
|
@@ -105,7 +107,7 @@ module Aspera
|
|
|
105
107
|
:ffprobe,
|
|
106
108
|
'-loglevel', 'error',
|
|
107
109
|
'-show_entries', 'format=duration',
|
|
108
|
-
'-print_format', 'default=noprint_wrappers=1:nokey=1',
|
|
110
|
+
'-print_format', 'default=noprint_wrappers=1:nokey=1',
|
|
109
111
|
input_file,
|
|
110
112
|
mode: :capture
|
|
111
113
|
).first.to_f
|
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require_relative '../environment'
|
|
4
7
|
require 'singleton'
|
|
5
8
|
|
|
6
9
|
module Aspera
|
|
@@ -79,7 +82,7 @@ module Aspera
|
|
|
79
82
|
end
|
|
80
83
|
|
|
81
84
|
CDN_BASE_URL = 'https://d3gcli72yxqn2z.cloudfront.net/connect'
|
|
82
|
-
VERSION_INFO_FILE = 'connectversions.js'
|
|
85
|
+
VERSION_INFO_FILE = 'connectversions.js'
|
|
83
86
|
|
|
84
87
|
private_constant :CDN_BASE_URL, :VERSION_INFO_FILE
|
|
85
88
|
end
|
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require_relative '../environment'
|
|
4
7
|
|
|
5
8
|
module Aspera
|
|
6
9
|
module Products
|
|
@@ -15,12 +18,24 @@ module Aspera
|
|
|
15
18
|
case Aspera::Environment.instance.os
|
|
16
19
|
when Aspera::Environment::OS_MACOS then [{
|
|
17
20
|
app_root: File.join('', 'Applications', 'IBM Aspera.app'),
|
|
18
|
-
log_root:
|
|
21
|
+
log_root: log_folder,
|
|
19
22
|
sub_bin: File.join('Contents', 'Resources', 'transferd', 'bin')
|
|
20
23
|
}]
|
|
21
24
|
else []
|
|
22
25
|
end.map { |i| i.merge({expected: APP_NAME}) }
|
|
23
26
|
end
|
|
27
|
+
|
|
28
|
+
# Folder of the logs of the application, of `transferd` and of `ascp` (`log_directory` and `fasp_runtime.log.dir` of `transferd`)
|
|
29
|
+
# @return [String, nil] nil if unknown
|
|
30
|
+
def log_folder
|
|
31
|
+
case Aspera::Environment.instance.os
|
|
32
|
+
when Aspera::Environment::OS_MACOS then File.join(Dir.home, 'Library', 'Logs', APP_IDENTIFIER)
|
|
33
|
+
when Aspera::Environment::OS_WINDOWS
|
|
34
|
+
appdata = ENV.fetch('APPDATA', nil)
|
|
35
|
+
File.join(appdata, APP_IDENTIFIER, 'logs') unless appdata.nil?
|
|
36
|
+
else File.join(ENV.fetch('XDG_CONFIG_HOME', File.join(Dir.home, '.config')), APP_IDENTIFIER, 'logs')
|
|
37
|
+
end
|
|
38
|
+
end
|
|
24
39
|
end
|
|
25
40
|
end
|
|
26
41
|
end
|
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
#
|
|
4
|
-
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require_relative '../environment'
|
|
5
7
|
|
|
6
8
|
module Aspera
|
|
7
9
|
# Location of Aspera products, for which an Agent is not proposed
|
|
@@ -1,7 +1,11 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
3
|
+
# SPDX-FileCopyrightText: IBM Corp.
|
|
4
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
|
|
6
|
+
require_relative '../environment'
|
|
7
|
+
require_relative '../log'
|
|
8
|
+
require_relative '../assert'
|
|
5
9
|
module Aspera
|
|
6
10
|
module Products
|
|
7
11
|
class Transferd
|