aspera-cli 4.27.5 → 4.28.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +160 -4
- data/CONTRIBUTING.md +4 -2
- data/README.md +1 -1
- data/bin/asession +1 -1
- data/docs/README.md +1086 -331
- data/docs/test-mcp-with-ai.md +5 -3
- data/lib/aspera/agent/direct.rb +15 -4
- data/lib/aspera/agent/factory.rb +2 -1
- data/lib/aspera/agent/node.rb +1 -1
- data/lib/aspera/api/cos_node.rb +1 -1
- data/lib/aspera/api/faspex.rb +1 -1
- data/lib/aspera/api/node.rb +6 -2
- data/lib/aspera/ascp/installation.rb +47 -11
- data/lib/aspera/ascp/management.rb +89 -43
- data/lib/aspera/cli/ascp_actions.rb +5 -19
- data/lib/aspera/cli/bootstrapper.rb +1 -2
- data/lib/aspera/cli/command_registry.rb +43 -3
- data/lib/aspera/cli/command_spec.rb +3 -0
- data/lib/aspera/cli/completion/ascli.bash +39 -5
- data/lib/aspera/cli/completion/ascli.fish +16 -8
- data/lib/aspera/cli/completion/ascli.ps1 +41 -0
- data/lib/aspera/cli/completion/ascli.zsh +20 -4
- data/lib/aspera/cli/completion_actions.rb +360 -0
- data/lib/aspera/cli/error.rb +42 -0
- data/lib/aspera/cli/extended_value.rb +30 -5
- data/lib/aspera/cli/formatter.rb +118 -56
- data/lib/aspera/cli/hints.rb +0 -1
- data/lib/aspera/cli/http.rb +6 -5
- data/lib/aspera/cli/info.rb +10 -4
- data/lib/aspera/cli/mailer.rb +1 -1
- data/lib/aspera/cli/mcp/http_servlet.rb +156 -0
- data/lib/aspera/cli/mcp/secrets.rb +143 -0
- data/lib/aspera/cli/mcp/tool.rb +231 -0
- data/lib/aspera/cli/option_value.rb +9 -1
- data/lib/aspera/cli/options.schema.yaml +390 -39
- data/lib/aspera/cli/parser.rb +14 -5
- data/lib/aspera/cli/plugins/aoc.rb +8 -11
- data/lib/aspera/cli/plugins/base.rb +36 -15
- data/lib/aspera/cli/plugins/config.rb +15 -44
- data/lib/aspera/cli/plugins/console.rb +3 -1
- data/lib/aspera/cli/plugins/factory.rb +1 -1
- data/lib/aspera/cli/plugins/faspex5.rb +9 -19
- data/lib/aspera/cli/plugins/faspio.rb +36 -5
- data/lib/aspera/cli/plugins/httpgw.rb +1 -0
- data/lib/aspera/cli/plugins/mcp.rb +177 -118
- data/lib/aspera/cli/plugins/node.rb +14 -8
- data/lib/aspera/cli/plugins/orchestrator.rb +351 -38
- data/lib/aspera/cli/plugins/server.rb +3 -1
- data/lib/aspera/cli/plugins/shares.rb +205 -35
- data/lib/aspera/cli/preset_actions.rb +15 -7
- data/lib/aspera/cli/preset_manager.rb +24 -2
- data/lib/aspera/cli/prompt.rb +3 -1
- data/lib/aspera/cli/result.rb +17 -4
- data/lib/aspera/cli/runner.rb +133 -43
- data/lib/aspera/cli/security.rb +134 -0
- data/lib/aspera/cli/table_fitter.rb +116 -0
- data/lib/aspera/cli/terminal_formatter.rb +23 -34
- data/lib/aspera/cli/transfer_agent.rb +4 -1
- data/lib/aspera/cli/vault_manager.rb +6 -5
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/cli/web_secret.html.erb +61 -0
- data/lib/aspera/cli/web_secret.rb +220 -0
- data/lib/aspera/cli/wizard.rb +19 -9
- data/lib/aspera/command_line_builder.rb +26 -0
- data/lib/aspera/data_repository.rb +38 -24
- data/lib/aspera/duration.rb +42 -0
- data/lib/aspera/environment.rb +88 -24
- data/lib/aspera/exec_spec.rb +6 -5
- data/lib/aspera/faspex_gw.rb +98 -32
- data/lib/aspera/faspex_postproc.rb +187 -44
- data/lib/aspera/formatter_interface.rb +0 -1
- data/lib/aspera/id_generator.rb +2 -1
- data/lib/aspera/json_rpc/client.rb +51 -32
- data/lib/aspera/json_rpc/version.rb +1 -0
- data/lib/aspera/keychain/base.rb +5 -0
- data/lib/aspera/keychain/encrypted_hash.rb +6 -2
- data/lib/aspera/keychain/factory.rb +2 -3
- data/lib/aspera/keychain/hashicorp_vault.rb +7 -4
- data/lib/aspera/keychain/macos_security.rb +119 -13
- data/lib/aspera/keychain/one_password_api.rb +8 -3
- data/lib/aspera/keychain/one_password_base.rb +14 -9
- data/lib/aspera/keychain/one_password_cli.rb +4 -2
- data/lib/aspera/log.rb +3 -6
- data/lib/aspera/markdown.rb +152 -115
- data/lib/aspera/mime.rb +6 -0
- data/lib/aspera/nagios.rb +3 -1
- data/lib/aspera/node_emulator.rb +98 -101
- data/lib/aspera/oauth/{base.rb → client/base.rb} +5 -4
- data/lib/aspera/oauth/{boot.rb → client/boot.rb} +3 -2
- data/lib/aspera/oauth/{factory.rb → client/factory.rb} +7 -0
- data/lib/aspera/oauth/{generic.rb → client/generic.rb} +1 -1
- data/lib/aspera/oauth/{json_credentials.rb → client/json_credentials.rb} +1 -1
- data/lib/aspera/oauth/{jwt.rb → client/jwt.rb} +1 -1
- data/lib/aspera/oauth/{url_json.rb → client/url_json.rb} +1 -1
- data/lib/aspera/oauth/{web.rb → client/web.rb} +2 -2
- data/lib/aspera/oauth/server/authorize.html.erb +34 -0
- data/lib/aspera/oauth/server/pkce.rb +426 -0
- data/lib/aspera/orchestrator_workflow_checker.rb +561 -0
- data/lib/aspera/preview/terminal.rb +3 -2
- data/lib/aspera/rainbow.rb +27 -1
- data/lib/aspera/rest/aspera_errors.rb +51 -59
- data/lib/aspera/rest/client.rb +21 -14
- data/lib/aspera/rest/error_analyzer.rb +9 -0
- data/lib/aspera/rest/list.rb +6 -6
- data/lib/aspera/rest/util.rb +7 -1
- data/lib/aspera/rest.rb +0 -1
- data/lib/aspera/schema/IBM Aspera Faspex API-5.0-enhanced.yaml +0 -2
- data/lib/aspera/schema/IBM Aspera Orchestrator API-v1-enhanced.yaml +4001 -0
- data/lib/aspera/schema/IBM Aspera faspio Gateway API-1.0.0-enhanced.yaml +1097 -0
- data/lib/aspera/schema/IBM Aspera on Cloud Automation API-1.0.5-enhanced.yaml +35 -35
- data/lib/aspera/schema/IBM_Aspera_Shares.yaml +819 -7
- data/lib/aspera/schema/documentation.rb +50 -78
- data/lib/aspera/schema/reader.rb +39 -11
- data/lib/aspera/schema/registry.rb +8 -2
- data/lib/aspera/secret_hider.rb +94 -15
- data/lib/aspera/secret_strength.rb +114 -0
- data/lib/aspera/servlet_helpers.rb +111 -0
- data/lib/aspera/sync/args.schema.yaml +3 -1
- data/lib/aspera/sync/conf.schema.yaml +16 -6
- data/lib/aspera/sync/operations.rb +21 -1
- data/lib/aspera/temp_file_manager.rb +79 -20
- data/lib/aspera/transfer/parameters.rb +5 -0
- data/lib/aspera/transfer/spec.rb +52 -12
- data/lib/aspera/transfer/spec.schema.yaml +8 -5
- data/lib/aspera/web_auth/index.html.erb +16 -0
- data/lib/aspera/web_auth/server.rb +18 -10
- data/lib/aspera/web_page.html.erb +262 -0
- data/lib/aspera/web_page.rb +61 -0
- data/lib/aspera/web_server_simple.rb +76 -10
- data.tar.gz.sig +0 -0
- metadata +47 -32
- metadata.gz.sig +0 -0
- data/lib/aspera/cli/mcp_tool.rb +0 -189
- data/lib/aspera/oauth.rb +0 -9
- data/lib/aspera/schema/IBM Aspera Orchestrator API-v1.yaml +0 -1784
- data/lib/aspera/schema/IBM Aspera faspio Gateway API-1.0.0.yaml +0 -249
- data/lib/aspera/web_auth/index.html +0 -147
data/lib/aspera/markdown.rb
CHANGED
|
@@ -1,141 +1,178 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require 'aspera/assert'
|
|
4
|
+
|
|
3
5
|
module Aspera
|
|
4
6
|
# Formatting for Markdown
|
|
5
|
-
|
|
7
|
+
module Markdown
|
|
8
|
+
extend self
|
|
9
|
+
|
|
6
10
|
# Matches: **bold**, `code`, or an HTML entity (&, ©, 💩)
|
|
7
|
-
|
|
11
|
+
INLINE_MARKUP_RE = /(?:\*\*(?<bold>[^*]+?)\*\*)|(?:`(?<code>[^`]+)`)|&(?<entity>(?:[A-Za-z][A-Za-z0-9]{1,31}|#\d{1,7}|#x[0-9A-Fa-f]{1,6}));/m
|
|
12
|
+
# HTML line-break tag used in Markdown table cells to separate multi-line values.
|
|
8
13
|
HTML_BREAK = '<br/>'
|
|
14
|
+
# Matches a Markdown heading line (e.g. `## Title`); group 1 = `#` chars, group 2 = text.
|
|
15
|
+
HEADING_RE = /^(\#{1,6})[ \t]+(\S.*)$/
|
|
16
|
+
# Matches the start or end of a fenced code block; group 1 = fence (``` or ~~~).
|
|
17
|
+
FENCE_RE = /^ {0,3}(`{3,}|~{3,})/
|
|
18
|
+
# Maximum column width (in characters) for generated Markdown tables.
|
|
19
|
+
COL_WIDTH = 80
|
|
20
|
+
# Accepted admonition types.
|
|
21
|
+
ADMONITION_TYPES = %w[NOTE CAUTION WARNING IMPORTANT TIP INFO].freeze
|
|
9
22
|
|
|
10
|
-
|
|
11
|
-
COL_WIDTH = 80
|
|
12
|
-
|
|
13
|
-
# Convert a Markdown heading text to a GitHub-flavoured anchor.
|
|
14
|
-
# Rules: downcase, keep letters/digits/spaces/hyphens, replace spaces with hyphens.
|
|
15
|
-
# Duplicate anchors are disambiguated by appending -1, -2, … (pass a seen Hash to track).
|
|
16
|
-
# @param text [String] raw heading text (without leading # and spaces)
|
|
17
|
-
# @param seen [Hash{String=>Integer}, nil] mutable counter; pass the same Hash across a document
|
|
18
|
-
# @return [String] anchor slug (without leading #)
|
|
19
|
-
def heading_to_anchor(text, seen: nil)
|
|
20
|
-
slug = text
|
|
21
|
-
.downcase
|
|
22
|
-
.gsub(/[`*_]/, '') # strip inline code/bold/italic markers
|
|
23
|
-
.gsub(/&[a-z]+;/, '') # strip HTML entities
|
|
24
|
-
.gsub(/[^\w\s-]/, '') # keep word chars, spaces, hyphens
|
|
25
|
-
.gsub(/\s+/, '-') # spaces → hyphens
|
|
26
|
-
.squeeze('-') # collapse consecutive hyphens
|
|
27
|
-
.strip
|
|
28
|
-
if seen
|
|
29
|
-
count = seen[slug].to_i
|
|
30
|
-
seen[slug] = count + 1
|
|
31
|
-
slug = "#{slug}-#{count}" if count > 0
|
|
32
|
-
end
|
|
33
|
-
slug
|
|
34
|
-
end
|
|
23
|
+
# --- Parsing ---
|
|
35
24
|
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
25
|
+
# Convert a Markdown heading text to a GitHub-flavoured anchor.
|
|
26
|
+
# Rules: downcase, keep letters/digits/spaces/hyphens, replace spaces with hyphens.
|
|
27
|
+
# Duplicate anchors are disambiguated by appending -1, -2, … (pass a seen Hash to track).
|
|
28
|
+
# @param text [String] raw heading text (without leading # and spaces)
|
|
29
|
+
# @param seen [Hash{String=>Integer}, nil] mutable counter; pass the same Hash across a document
|
|
30
|
+
# @return [String] anchor slug (without leading #)
|
|
31
|
+
def heading_to_anchor(text, seen: nil)
|
|
32
|
+
slug = text
|
|
33
|
+
.downcase
|
|
34
|
+
.gsub(/[`*_]/, '') # strip inline code/bold/italic markers
|
|
35
|
+
.gsub(/&[a-z]+;/, '') # strip HTML entities
|
|
36
|
+
.gsub(/[^\w\s-]/, '') # keep word chars, spaces, hyphens
|
|
37
|
+
.gsub(/\s+/, '-') # spaces → hyphens
|
|
38
|
+
.squeeze('-') # collapse consecutive hyphens
|
|
39
|
+
.strip
|
|
40
|
+
if seen
|
|
41
|
+
count = seen[slug].to_i
|
|
42
|
+
seen[slug] = count + 1
|
|
43
|
+
slug = "#{slug}-#{count}" if count > 0
|
|
49
44
|
end
|
|
45
|
+
slug
|
|
46
|
+
end
|
|
50
47
|
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
if
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
next unless slug == anchor
|
|
66
|
-
section_level = m[1].length
|
|
67
|
-
elsif m[1].length <= section_level
|
|
68
|
-
# already in section: stop at same/higher level heading
|
|
69
|
-
break
|
|
70
|
-
end
|
|
48
|
+
# Iterate on the lines of a Markdown document, with the heading match of each line.
|
|
49
|
+
# A line in a fenced code block is not a heading (e.g. a shell comment `# ...`).
|
|
50
|
+
# @param content [String] full Markdown source
|
|
51
|
+
# @yieldparam line [String] the line
|
|
52
|
+
# @yieldparam heading [MatchData, nil] match of HEADING_RE, or nil if not a heading
|
|
53
|
+
def each_line_with_heading(content)
|
|
54
|
+
fence = nil
|
|
55
|
+
content.each_line do |line|
|
|
56
|
+
if (f = line.match(FENCE_RE))
|
|
57
|
+
# a code block ends with the same kind of fence, at least as long, without info string
|
|
58
|
+
if fence.nil?
|
|
59
|
+
fence = f[1]
|
|
60
|
+
elsif f[1].start_with?(fence) && line.strip.eql?(f[1])
|
|
61
|
+
fence = nil
|
|
71
62
|
end
|
|
72
|
-
|
|
63
|
+
yield(line, nil)
|
|
64
|
+
else
|
|
65
|
+
yield(line, fence.nil? ? line.match(HEADING_RE) : nil)
|
|
73
66
|
end
|
|
74
|
-
result.empty? ? nil : result.join
|
|
75
67
|
end
|
|
68
|
+
end
|
|
76
69
|
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
lines = table.map { |line| "| #{line.map { |i| i.to_s.gsub('\\', '\\\\').gsub('|', '\|') }.join(' | ')} |\n" }
|
|
91
|
-
lines[1] = lines[1].tr(' ', '-')
|
|
92
|
-
return lines.join.chomp
|
|
70
|
+
# Extract the table of contents from a Markdown document.
|
|
71
|
+
# @param content [String] full Markdown source
|
|
72
|
+
# @param max_level [Integer, nil] maximum level of headings, or nil for all
|
|
73
|
+
# @return [Array<Hash>] array of { level, title, anchor } (String keys, as other results: `--select` applies)
|
|
74
|
+
def toc(content, max_level: nil)
|
|
75
|
+
seen = {}
|
|
76
|
+
result = []
|
|
77
|
+
each_line_with_heading(content) do |_line, m|
|
|
78
|
+
next unless m
|
|
79
|
+
title = m[2].strip
|
|
80
|
+
# anchor of every heading, so that duplicate anchors are numbered as in the whole document
|
|
81
|
+
anchor = heading_to_anchor(title, seen: seen)
|
|
82
|
+
result.push({'level' => m[1].length, 'title' => title, 'anchor' => anchor}) if max_level.nil? || m[1].length <= max_level
|
|
93
83
|
end
|
|
84
|
+
result
|
|
85
|
+
end
|
|
94
86
|
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
87
|
+
# Extract the content of a single section (heading + body until next heading of same/higher level).
|
|
88
|
+
# @param content [String] full Markdown source
|
|
89
|
+
# @param anchor [String] GitHub anchor slug (without #)
|
|
90
|
+
# @return [String, nil] the section content, or nil if not found
|
|
91
|
+
def extract_section(content, anchor)
|
|
92
|
+
seen = {}
|
|
93
|
+
section_level = nil
|
|
94
|
+
result = []
|
|
95
|
+
each_line_with_heading(content) do |line, m|
|
|
96
|
+
if m
|
|
97
|
+
slug = heading_to_anchor(m[2].strip, seen: seen)
|
|
98
|
+
if section_level.nil?
|
|
99
|
+
# not yet found: check if this heading matches
|
|
100
|
+
next unless slug == anchor
|
|
101
|
+
section_level = m[1].length
|
|
102
|
+
elsif m[1].length <= section_level
|
|
103
|
+
# already in section: stop at same/higher level heading
|
|
104
|
+
break
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
result << line if section_level
|
|
100
108
|
end
|
|
109
|
+
result.empty? ? nil : result.join
|
|
110
|
+
end
|
|
101
111
|
|
|
102
|
-
|
|
103
|
-
# @param title [String] heading text
|
|
104
|
-
# @param level [Integer] heading level (1–6)
|
|
105
|
-
# @return [String] markdown heading
|
|
106
|
-
def heading(title, level: 1)
|
|
107
|
-
"#{'#' * level} #{title}\n\n"
|
|
108
|
-
end
|
|
112
|
+
# --- Generation ---
|
|
109
113
|
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
end
|
|
114
|
+
# Wrap text in inline code backticks
|
|
115
|
+
# @param text [String] text to wrap
|
|
116
|
+
# @return [String] inline code span
|
|
117
|
+
def icode(text)
|
|
118
|
+
"`#{text}`"
|
|
119
|
+
end
|
|
117
120
|
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
end
|
|
121
|
+
# Wrap text in a markdown paragraph (trailing blank line)
|
|
122
|
+
# @param text [String] paragraph content
|
|
123
|
+
# @return [String] paragraph with trailing newlines
|
|
124
|
+
def paragraph(text)
|
|
125
|
+
"#{text}\n\n"
|
|
126
|
+
end
|
|
125
127
|
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
128
|
+
# Generate a markdown heading
|
|
129
|
+
# @param title [String] heading text
|
|
130
|
+
# @param level [Integer] heading level (1–6)
|
|
131
|
+
# @return [String] markdown heading
|
|
132
|
+
def heading(title, level: 1)
|
|
133
|
+
paragraph("#{'#' * level} #{title}")
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
# Generate a fenced code block
|
|
137
|
+
# @param lines [Array<String>] lines of code
|
|
138
|
+
# @param type [String] language identifier for syntax highlighting
|
|
139
|
+
# @return [String] markdown fenced code block
|
|
140
|
+
def code(lines, type: 'shell')
|
|
141
|
+
paragraph("```#{type}\n#{lines.join("\n")}\n```")
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
# Generate a GitHub-flavoured admonition block
|
|
145
|
+
# @param lines [Array<String>] lines of the admonition body
|
|
146
|
+
# @param type [String] admonition type, one of {ADMONITION_TYPES}
|
|
147
|
+
# @return [String] markdown admonition block
|
|
148
|
+
def admonition(lines, type: 'INFO')
|
|
149
|
+
Aspera.assert_values(type, ADMONITION_TYPES) { 'admonition type' }
|
|
150
|
+
paragraph((["[!#{type}]"] + lines).map { |l| "> #{l}" }.join("\n"))
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
# Generate markdown list from the provided list
|
|
154
|
+
# @param items [Array<String>] list of items
|
|
155
|
+
# @return [String] markdown unordered list
|
|
156
|
+
def list(items)
|
|
157
|
+
items.map { |i| "- #{i}" }.join("\n")
|
|
158
|
+
end
|
|
132
159
|
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
160
|
+
# Generate markdown from the provided 2D table
|
|
161
|
+
# @param table [Array<Array<String>>] 2D array of strings
|
|
162
|
+
# @return [String] markdown table
|
|
163
|
+
def table(table)
|
|
164
|
+
# get max width of each columns
|
|
165
|
+
col_widths = table.transpose.map do |col|
|
|
166
|
+
[col.flat_map { |c| c.to_s.delete('`').split(HTML_BREAK).map(&:size) }.max, COL_WIDTH].min
|
|
138
167
|
end
|
|
168
|
+
# do not modify caller's table
|
|
169
|
+
table = table.dup
|
|
170
|
+
headings = table.shift
|
|
171
|
+
table.unshift(col_widths.map { |col_width| '-' * col_width })
|
|
172
|
+
table.unshift(headings)
|
|
173
|
+
lines = table.map { |line| "| #{line.map { |i| i.to_s.gsub('\\', '\\\\').gsub('|', '\|') }.join(' | ')} |\n" }
|
|
174
|
+
lines[1] = lines[1].tr(' ', '-')
|
|
175
|
+
return lines.join.chomp
|
|
139
176
|
end
|
|
140
177
|
end
|
|
141
178
|
end
|
data/lib/aspera/mime.rb
CHANGED
|
@@ -9,8 +9,14 @@ module Aspera
|
|
|
9
9
|
WWW = 'application/x-www-form-urlencoded'
|
|
10
10
|
# Plain text body
|
|
11
11
|
TEXT = 'text/plain'
|
|
12
|
+
# XML body
|
|
13
|
+
XML = 'application/xml'
|
|
14
|
+
# Multipart form body (file upload)
|
|
15
|
+
MULTIPART = 'multipart/form-data'
|
|
12
16
|
# JSON:API body (https://jsonapi.org)
|
|
13
17
|
JSON_API = 'application/vnd.api+json'
|
|
18
|
+
# Server-Sent Events stream
|
|
19
|
+
EVENT_STREAM = 'text/event-stream'
|
|
14
20
|
# Check if a media type is JSON: `application/json` or structured syntax suffix `+json` (RFC 6839)
|
|
15
21
|
# @param mime [String] Media type, without parameters (see `Rest.parse_header`)
|
|
16
22
|
# @return [Boolean] `true` if JSON
|
data/lib/aspera/nagios.rb
CHANGED
|
@@ -13,6 +13,8 @@ module Aspera
|
|
|
13
13
|
DATE_WARN_OFFSET = 2
|
|
14
14
|
DATE_CRIT_OFFSET = 5
|
|
15
15
|
private_constant :LEVELS, :ADD_PREFIX, :DATE_WARN_OFFSET, :DATE_CRIT_OFFSET
|
|
16
|
+
# Exit code when the check itself fails
|
|
17
|
+
UNKNOWN = LEVELS.index(:unknown)
|
|
16
18
|
|
|
17
19
|
# add methods to add nagios error levels, each take component name and message
|
|
18
20
|
LEVELS.each_index do |code|
|
|
@@ -32,7 +34,7 @@ module Aspera
|
|
|
32
34
|
# keep only errors in case of problem, other ok are assumed so
|
|
33
35
|
data = res_errors unless res_errors.empty?
|
|
34
36
|
# first is most critical
|
|
35
|
-
data.sort
|
|
37
|
+
data = data.sort { |a, b| LEVELS.index(a['status'].to_sym) <=> LEVELS.index(b['status'].to_sym) }
|
|
36
38
|
# build message: if multiple components: concatenate
|
|
37
39
|
# message = data.map{|i|"#{i['component']}:#{i['message']}"}.join(', ').gsub("\n",' ')
|
|
38
40
|
message = data
|
data/lib/aspera/node_emulator.rb
CHANGED
|
@@ -2,7 +2,6 @@
|
|
|
2
2
|
|
|
3
3
|
# cspell:ignore precalc noxfer euid gettime
|
|
4
4
|
require 'aspera/assert'
|
|
5
|
-
require 'aspera/mime'
|
|
6
5
|
require 'aspera/ascp/installation'
|
|
7
6
|
require 'aspera/agent/transferd'
|
|
8
7
|
require 'aspera/transfer/spec'
|
|
@@ -10,8 +9,8 @@ require 'aspera/transfer/error'
|
|
|
10
9
|
require 'aspera/secret_hider'
|
|
11
10
|
require 'aspera/string_ext'
|
|
12
11
|
require 'aspera/log'
|
|
12
|
+
require 'aspera/servlet_helpers'
|
|
13
13
|
require 'webrick'
|
|
14
|
-
require 'openssl'
|
|
15
14
|
require 'etc'
|
|
16
15
|
require 'securerandom'
|
|
17
16
|
require 'json'
|
|
@@ -319,59 +318,25 @@ module Aspera
|
|
|
319
318
|
Aspera.assert(in_docroot?(File.realpath(folder_path))) { "Path outside docroot: #{path}" }
|
|
320
319
|
Aspera.assert(File.directory?(folder_path)) { "Not a folder: #{path}" }
|
|
321
320
|
folder_virtual = virtual_path(folder_path)
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
'mtime' => folder_stat.mtime.utc.iso8601,
|
|
333
|
-
'permissions' => permissions(folder_path)
|
|
334
|
-
},
|
|
335
|
-
'items' => []
|
|
336
|
-
}
|
|
337
|
-
|
|
338
|
-
# Iterate over folder contents
|
|
339
|
-
Dir.foreach(folder_path) do |entry|
|
|
340
|
-
next if entry == '.' || entry == '..' # Skip current and parent directory
|
|
341
|
-
|
|
342
|
-
item_path = File.join(folder_path, entry)
|
|
343
|
-
item_type = File.ftype(item_path) rescue 'unknown' # Get the type of file
|
|
344
|
-
item_type = FILE_TYPES.fetch(item_type, item_type)
|
|
345
|
-
item_stat = File.lstat(item_path) # Use lstat to handle symbolic links correctly
|
|
346
|
-
|
|
347
|
-
item = {
|
|
348
|
-
'path' => File.join(folder_virtual, entry),
|
|
349
|
-
'basename' => entry,
|
|
350
|
-
'type' => item_type,
|
|
351
|
-
'size' => item_type.eql?('directory') ? 0 : item_stat.size,
|
|
352
|
-
'mtime' => item_stat.mtime.utc.iso8601,
|
|
353
|
-
'permissions' => permissions(item_path)
|
|
354
|
-
}
|
|
355
|
-
|
|
356
|
-
# Add additional details for specific types
|
|
357
|
-
case item_type
|
|
358
|
-
when 'file'
|
|
359
|
-
item['partial_file'] = false
|
|
360
|
-
when 'symbolic_link'
|
|
361
|
-
item['target'] = link_target(item_path)
|
|
362
|
-
when 'unknown'
|
|
363
|
-
item['note'] = 'File type could not be determined'
|
|
321
|
+
# stable order for paging: only the items of the page are read
|
|
322
|
+
names = Dir.children(folder_path).sort
|
|
323
|
+
items = names.drop(skip).first(count || names.length).map do |name|
|
|
324
|
+
item_path = File.join(folder_path, name)
|
|
325
|
+
# a symbolic link is not followed
|
|
326
|
+
item = item_info(item_path, File.join(folder_virtual, name), File.lstat(item_path))
|
|
327
|
+
case item['type']
|
|
328
|
+
when 'file' then item['partial_file'] = false
|
|
329
|
+
when 'symbolic_link' then item['target'] = link_target(item_path)
|
|
330
|
+
when 'unknown' then item['note'] = 'File type could not be determined'
|
|
364
331
|
end
|
|
365
|
-
|
|
366
|
-
structure['items'] << item
|
|
332
|
+
item
|
|
367
333
|
end
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
structure
|
|
334
|
+
{
|
|
335
|
+
'self' => item_info(folder_path, folder_virtual, File.stat(folder_path)),
|
|
336
|
+
'items' => items,
|
|
337
|
+
'item_count' => items.length,
|
|
338
|
+
'total_count' => names.length
|
|
339
|
+
}
|
|
375
340
|
end
|
|
376
341
|
|
|
377
342
|
# Start a transfer, and monitor it in a thread until terminated
|
|
@@ -379,8 +344,8 @@ module Aspera
|
|
|
379
344
|
# @return [String] transfer id
|
|
380
345
|
def start(transfer_spec)
|
|
381
346
|
Transfer::Spec.fix_transferd_resume_policy(transfer_spec)
|
|
382
|
-
#
|
|
383
|
-
start_spec = SecretHider.instance.
|
|
347
|
+
# copy without secrets, returned in transfer `start_spec`
|
|
348
|
+
start_spec = SecretHider.instance.hidden_copy(transfer_spec)
|
|
384
349
|
Log.dump(:start_spec, start_spec)
|
|
385
350
|
request = ::Transferd::Api::TransferRequest.new(
|
|
386
351
|
transferType: ::Transferd::Api::TransferType::FILE_REGULAR,
|
|
@@ -478,10 +443,23 @@ module Aspera
|
|
|
478
443
|
def real_path(path)
|
|
479
444
|
# `./` prevents expansion of `~`
|
|
480
445
|
real = File.expand_path("./#{path.to_s.sub(%r{\A/+}, '')}", @docroot)
|
|
481
|
-
|
|
446
|
+
# symbolic links shall not lead out of the docroot
|
|
447
|
+
resolved = resolved_path(real) if in_docroot?(real)
|
|
448
|
+
Aspera.assert(!resolved.nil? && in_docroot?(resolved)) { "Path outside docroot: #{path}" }
|
|
482
449
|
real
|
|
483
450
|
end
|
|
484
451
|
|
|
452
|
+
# Resolve symbolic links of the existing part of a path: a destination may not exist yet
|
|
453
|
+
# @param real [String] absolute path, without `..`, within the docroot
|
|
454
|
+
# @return [String, nil] real path of the longest existing part of `real`, `nil` if it is a dangling symbolic link
|
|
455
|
+
def resolved_path(real)
|
|
456
|
+
# stops at the docroot, which exists
|
|
457
|
+
real = File.dirname(real) until File.exist?(real) || File.symlink?(real)
|
|
458
|
+
File.realpath(real)
|
|
459
|
+
rescue Errno::ENOENT, Errno::ELOOP
|
|
460
|
+
nil
|
|
461
|
+
end
|
|
462
|
+
|
|
485
463
|
# Convert a real path to a Node API path
|
|
486
464
|
# @param real [String] real path within the docroot
|
|
487
465
|
# @return [String] path relative to the docroot, with leading `/`
|
|
@@ -496,6 +474,23 @@ module Aspera
|
|
|
496
474
|
real.eql?(@docroot) || real.start_with?(@docroot.end_with?('/') ? @docroot : "#{@docroot}/")
|
|
497
475
|
end
|
|
498
476
|
|
|
477
|
+
# Get the Node API fields of a folder or of an item of a folder
|
|
478
|
+
# @param real [String] real path within the docroot
|
|
479
|
+
# @param virtual [String] Node API path
|
|
480
|
+
# @param stat [File::Stat] information on the item
|
|
481
|
+
# @return [Hash] Node API fields common to a folder and its items
|
|
482
|
+
def item_info(real, virtual, stat)
|
|
483
|
+
{
|
|
484
|
+
'path' => virtual,
|
|
485
|
+
'basename' => File.basename(real),
|
|
486
|
+
'type' => FILE_TYPES.fetch(stat.ftype, stat.ftype),
|
|
487
|
+
# Node API reports 0 for folders
|
|
488
|
+
'size' => stat.directory? ? 0 : stat.size,
|
|
489
|
+
'mtime' => stat.mtime.utc.iso8601,
|
|
490
|
+
'permissions' => permissions(real)
|
|
491
|
+
}
|
|
492
|
+
end
|
|
493
|
+
|
|
499
494
|
# Get the Node API permissions of an item
|
|
500
495
|
# @param real [String] real path within the docroot
|
|
501
496
|
# @return [Array<Hash>] Node API permissions, from the access of the emulator on the item
|
|
@@ -581,6 +576,8 @@ module Aspera
|
|
|
581
576
|
# Answers a subset of the Node API, transfers are delegated to a NodeEmulator
|
|
582
577
|
# a new instance is created for each request
|
|
583
578
|
class NodeEmulatorServlet < WEBrick::HTTPServlet::AbstractServlet
|
|
579
|
+
include ServletHelpers
|
|
580
|
+
|
|
584
581
|
PATH_TRANSFERS = '/ops/transfers'
|
|
585
582
|
PATH_ONE_TRANSFER = %r{/ops/transfers/(.+)$}
|
|
586
583
|
PATH_BROWSE = '/files/browse'
|
|
@@ -589,6 +586,8 @@ module Aspera
|
|
|
589
586
|
CANCEL_STATUSES = %w[canceled cancelled stopped].freeze
|
|
590
587
|
# `PUT` fields modified by transferd
|
|
591
588
|
MODIFIABLE = %w[target_rate_kbps min_rate_kbps rate_policy].freeze
|
|
589
|
+
# HTTP methods with a JSON body
|
|
590
|
+
JSON_METHODS = %w[POST PUT].freeze
|
|
592
591
|
# Create the servlet, with optional Basic authentication
|
|
593
592
|
# @param server [WEBrick::HTTPServer]
|
|
594
593
|
# @param config [Hash] `username` and `password` (Basic authentication expected from clients, optional)
|
|
@@ -599,14 +598,12 @@ module Aspera
|
|
|
599
598
|
@expected_auth = "#{config[:username]}:#{config[:password]}" unless config[:username].nil?
|
|
600
599
|
end
|
|
601
600
|
|
|
602
|
-
#
|
|
601
|
+
# Refuse requests of web pages, check authentication, dispatch to `do_<verb>`, and send errors in Node API format
|
|
603
602
|
# @param request [WEBrick::HTTPRequest]
|
|
604
603
|
# @param response [WEBrick::HTTPResponse]
|
|
605
604
|
def service(request, response)
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
return set_error(request, response, 401, 'Invalid or missing credentials')
|
|
609
|
-
end
|
|
605
|
+
check_request(request, json: JSON_METHODS.include?(request.request_method))
|
|
606
|
+
return set_error(request, response, 401, 'Invalid or missing credentials', headers: {'WWW-Authenticate' => %Q(Basic realm="#{REALM}")}) unless authorized?(request)
|
|
610
607
|
super
|
|
611
608
|
rescue WEBrick::HTTPStatus::Error => e
|
|
612
609
|
set_error(request, response, e.code, e.message)
|
|
@@ -623,40 +620,35 @@ module Aspera
|
|
|
623
620
|
# @param request [WEBrick::HTTPRequest]
|
|
624
621
|
# @param response [WEBrick::HTTPResponse]
|
|
625
622
|
def do_POST(request, response)
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
623
|
+
json =
|
|
624
|
+
case request.path
|
|
625
|
+
when PATH_TRANSFERS
|
|
626
|
+
@simulator.transfer(@simulator.start(JSON.parse(request.body.to_s)))
|
|
627
|
+
when PATH_BROWSE
|
|
628
|
+
req = JSON.parse(request.body.to_s)
|
|
629
|
+
@simulator.browse(req['path'] || '/', skip: req['skip'].to_i, count: req['count']&.to_i)
|
|
630
|
+
else raise WEBrick::HTTPStatus::NotFound, "Unknown path: #{request.path}"
|
|
631
|
+
end
|
|
632
|
+
set_json_response(request, response, json)
|
|
635
633
|
end
|
|
636
634
|
|
|
637
635
|
# Get the node information, the list of transfers, or one transfer
|
|
638
636
|
# @param request [WEBrick::HTTPRequest]
|
|
639
637
|
# @param response [WEBrick::HTTPResponse]
|
|
640
638
|
def do_GET(request, response)
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
if transfer.nil?
|
|
653
|
-
set_error(request, response, 404, 'Unknown transfer')
|
|
654
|
-
else
|
|
655
|
-
set_json_response(request, response, transfer)
|
|
639
|
+
json =
|
|
640
|
+
case request.path
|
|
641
|
+
when '/info' then @simulator.info
|
|
642
|
+
when PATH_TRANSFERS
|
|
643
|
+
@simulator.transfers(
|
|
644
|
+
active_only: query_boolean(request, 'active_only'),
|
|
645
|
+
direction: request.query['direction']&.to_s,
|
|
646
|
+
count: query_positive(request, 'count')
|
|
647
|
+
)
|
|
648
|
+
when PATH_ONE_TRANSFER then known_transfer(@simulator.transfer(transfer_id(request)))
|
|
649
|
+
else raise WEBrick::HTTPStatus::NotFound, "Unknown path: #{request.path}"
|
|
656
650
|
end
|
|
657
|
-
|
|
658
|
-
set_error(request, response, 404, "Unknown path: #{request.path}")
|
|
659
|
-
end
|
|
651
|
+
set_json_response(request, response, json)
|
|
660
652
|
end
|
|
661
653
|
|
|
662
654
|
# Modify a transfer, or cancel it with `status`
|
|
@@ -676,7 +668,7 @@ module Aspera
|
|
|
676
668
|
raise WEBrick::HTTPStatus::BadRequest, "Unsupported status: #{status}" unless CANCEL_STATUSES.include?(status)
|
|
677
669
|
found = @simulator.cancel(id)
|
|
678
670
|
end
|
|
679
|
-
|
|
671
|
+
known_transfer(found)
|
|
680
672
|
set_json_response(request, response, @simulator.transfer(id))
|
|
681
673
|
end
|
|
682
674
|
|
|
@@ -684,7 +676,7 @@ module Aspera
|
|
|
684
676
|
# @param request [WEBrick::HTTPRequest]
|
|
685
677
|
# @param response [WEBrick::HTTPResponse]
|
|
686
678
|
def do_CANCEL(request, response)
|
|
687
|
-
|
|
679
|
+
known_transfer(@simulator.cancel(transfer_id(request)))
|
|
688
680
|
response.status = 204
|
|
689
681
|
end
|
|
690
682
|
|
|
@@ -700,6 +692,14 @@ module Aspera
|
|
|
700
692
|
match[1]
|
|
701
693
|
end
|
|
702
694
|
|
|
695
|
+
# @param found [Object] Result of the emulator for a transfer: `nil` or `false` if unknown
|
|
696
|
+
# @return [Object] found
|
|
697
|
+
# @raise [WEBrick::HTTPStatus::NotFound] if the transfer is unknown
|
|
698
|
+
def known_transfer(found)
|
|
699
|
+
raise WEBrick::HTTPStatus::NotFound, 'Unknown transfer' unless found
|
|
700
|
+
found
|
|
701
|
+
end
|
|
702
|
+
|
|
703
703
|
# Read a boolean query parameter
|
|
704
704
|
# @param request [WEBrick::HTTPRequest]
|
|
705
705
|
# @param name [String] name of the query parameter
|
|
@@ -730,18 +730,16 @@ module Aspera
|
|
|
730
730
|
# @param response [WEBrick::HTTPResponse]
|
|
731
731
|
# @param code [Integer] HTTP status code
|
|
732
732
|
# @param message [String] message for the user
|
|
733
|
-
|
|
734
|
-
|
|
733
|
+
# @param headers [Hash{String => String}] Other headers of the response
|
|
734
|
+
def set_error(request, response, code, message, headers: {})
|
|
735
|
+
set_json_response(request, response, {error: {code: code, reason: WEBrick::HTTPStatus.reason_phrase(code), user_message: message}}, code: code, headers: headers)
|
|
735
736
|
end
|
|
736
737
|
|
|
737
738
|
# Check the Basic credentials of the request
|
|
738
739
|
# @param request [WEBrick::HTTPRequest]
|
|
739
740
|
# @return [Boolean] `true` if the request has the expected Basic credentials, or if none are expected
|
|
740
741
|
def authorized?(request)
|
|
741
|
-
|
|
742
|
-
scheme, value = request['Authorization'].to_s.split(' ', 2)
|
|
743
|
-
# constant time comparison
|
|
744
|
-
scheme.to_s.casecmp?('Basic') && OpenSSL.secure_compare(value.to_s.unpack1('m'), @expected_auth)
|
|
742
|
+
@expected_auth.nil? || basic_authorized?(request, @expected_auth)
|
|
745
743
|
end
|
|
746
744
|
|
|
747
745
|
# Set the status and the JSON body of the response
|
|
@@ -749,11 +747,10 @@ module Aspera
|
|
|
749
747
|
# @param response [WEBrick::HTTPResponse]
|
|
750
748
|
# @param json [Hash, Array] body of the response
|
|
751
749
|
# @param code [Integer] HTTP status code
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
response
|
|
755
|
-
response.
|
|
756
|
-
Log.log.trace1 { Log.obj_dump("response for #{request.request_method} #{request.path}", json) }
|
|
750
|
+
# @param headers [Hash{String => String}] Other headers of the response
|
|
751
|
+
def set_json_response(request, response, json, code: 200, headers: {})
|
|
752
|
+
json_response(response, code, json, headers: headers)
|
|
753
|
+
Log.dump("response for #{request.request_method} #{request.path}", json, level: :trace1)
|
|
757
754
|
end
|
|
758
755
|
end
|
|
759
756
|
end
|