aspera-cli 4.10.0 → 4.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/BUGS.md +19 -0
- data/CHANGELOG.md +528 -0
- data/CONTRIBUTING.md +143 -0
- data/README.md +977 -589
- data/bin/ascli +4 -4
- data/bin/asession +12 -12
- data/docs/test_env.conf +29 -19
- data/examples/aoc.rb +6 -6
- data/examples/dascli +18 -16
- data/examples/faspex4.rb +15 -15
- data/examples/node.rb +12 -12
- data/examples/proxy.pac +2 -2
- data/examples/server.rb +12 -12
- data/lib/aspera/aoc.rb +344 -272
- data/lib/aspera/ascmd.rb +56 -54
- data/lib/aspera/ats_api.rb +4 -4
- data/lib/aspera/cli/basic_auth_plugin.rb +15 -12
- data/lib/aspera/cli/extended_value.rb +9 -9
- data/lib/aspera/cli/{formater.rb → formatter.rb} +69 -69
- data/lib/aspera/cli/listener/line_dump.rb +1 -1
- data/lib/aspera/cli/listener/logger.rb +1 -1
- data/lib/aspera/cli/listener/progress.rb +5 -6
- data/lib/aspera/cli/listener/progress_multi.rb +16 -21
- data/lib/aspera/cli/main.rb +72 -73
- data/lib/aspera/cli/manager.rb +112 -112
- data/lib/aspera/cli/plugin.rb +68 -48
- data/lib/aspera/cli/plugins/alee.rb +4 -4
- data/lib/aspera/cli/plugins/aoc.rb +322 -720
- data/lib/aspera/cli/plugins/ats.rb +50 -52
- data/lib/aspera/cli/plugins/bss.rb +10 -10
- data/lib/aspera/cli/plugins/config.rb +514 -410
- data/lib/aspera/cli/plugins/console.rb +12 -12
- data/lib/aspera/cli/plugins/cos.rb +18 -20
- data/lib/aspera/cli/plugins/faspex.rb +134 -136
- data/lib/aspera/cli/plugins/faspex5.rb +235 -70
- data/lib/aspera/cli/plugins/node.rb +378 -309
- data/lib/aspera/cli/plugins/orchestrator.rb +52 -49
- data/lib/aspera/cli/plugins/preview.rb +129 -120
- data/lib/aspera/cli/plugins/server.rb +137 -83
- data/lib/aspera/cli/plugins/shares.rb +77 -52
- data/lib/aspera/cli/plugins/sync.rb +13 -33
- data/lib/aspera/cli/transfer_agent.rb +61 -61
- data/lib/aspera/cli/version.rb +2 -1
- data/lib/aspera/colors.rb +3 -3
- data/lib/aspera/command_line_builder.rb +78 -74
- data/lib/aspera/cos_node.rb +31 -29
- data/lib/aspera/data_repository.rb +1 -1
- data/lib/aspera/environment.rb +30 -28
- data/lib/aspera/fasp/agent_base.rb +17 -15
- data/lib/aspera/fasp/agent_connect.rb +34 -32
- data/lib/aspera/fasp/agent_direct.rb +70 -73
- data/lib/aspera/fasp/agent_httpgw.rb +79 -74
- data/lib/aspera/fasp/agent_node.rb +26 -26
- data/lib/aspera/fasp/agent_trsdk.rb +20 -20
- data/lib/aspera/fasp/error.rb +3 -2
- data/lib/aspera/fasp/error_info.rb +11 -8
- data/lib/aspera/fasp/installation.rb +80 -80
- data/lib/aspera/fasp/listener.rb +2 -2
- data/lib/aspera/fasp/parameters.rb +103 -92
- data/lib/aspera/fasp/parameters.yaml +313 -214
- data/lib/aspera/fasp/resume_policy.rb +10 -10
- data/lib/aspera/fasp/transfer_spec.rb +22 -2
- data/lib/aspera/fasp/uri.rb +7 -7
- data/lib/aspera/faspex_gw.rb +80 -159
- data/lib/aspera/faspex_postproc.rb +77 -0
- data/lib/aspera/hash_ext.rb +3 -3
- data/lib/aspera/id_generator.rb +5 -5
- data/lib/aspera/keychain/encrypted_hash.rb +23 -28
- data/lib/aspera/keychain/macos_security.rb +21 -20
- data/lib/aspera/log.rb +13 -13
- data/lib/aspera/nagios.rb +24 -23
- data/lib/aspera/node.rb +217 -38
- data/lib/aspera/oauth.rb +78 -74
- data/lib/aspera/open_application.rb +19 -11
- data/lib/aspera/persistency_action_once.rb +4 -4
- data/lib/aspera/persistency_folder.rb +13 -13
- data/lib/aspera/preview/file_types.rb +8 -8
- data/lib/aspera/preview/generator.rb +67 -67
- data/lib/aspera/preview/utils.rb +27 -27
- data/lib/aspera/proxy_auto_config.js +63 -63
- data/lib/aspera/proxy_auto_config.rb +19 -19
- data/lib/aspera/rest.rb +65 -67
- data/lib/aspera/rest_call_error.rb +2 -1
- data/lib/aspera/rest_error_analyzer.rb +22 -21
- data/lib/aspera/rest_errors_aspera.rb +16 -16
- data/lib/aspera/secret_hider.rb +17 -14
- data/lib/aspera/ssh.rb +15 -14
- data/lib/aspera/sync.rb +177 -62
- data/lib/aspera/temp_file_manager.rb +2 -2
- data/lib/aspera/uri_reader.rb +4 -4
- data/lib/aspera/web_auth.rb +13 -64
- data/lib/aspera/web_server_simple.rb +76 -0
- data.tar.gz.sig +0 -0
- metadata +11 -6
- metadata.gz.sig +0 -0
data/lib/aspera/ascmd.rb
CHANGED
@@ -20,32 +20,32 @@ module Aspera
|
|
20
20
|
# @param [Symbol] one of OPERATIONS
|
21
21
|
# @param [Array] parameters for "as" command
|
22
22
|
# @return result of command, type depends on command (bool, array, hash)
|
23
|
-
def execute_single(action_sym,args=nil)
|
23
|
+
def execute_single(action_sym, args=nil)
|
24
24
|
# concatenate arguments, enclose in double quotes, protect backslash and double quotes, add "as_" command and as_exit
|
25
25
|
stdin_input = (args || []).map{|v| '"' + v.gsub(/["\\]/n) {|s| '\\' + s } + '"'}.unshift('as_' + action_sym.to_s).join(' ') + "\nas_exit\n"
|
26
26
|
# execute, get binary output
|
27
|
-
bytebuffer = @command_executor.execute('ascmd',stdin_input).unpack('C*')
|
27
|
+
bytebuffer = @command_executor.execute('ascmd', stdin_input).unpack('C*')
|
28
28
|
# get hash or table result
|
29
|
-
result = self.class.parse(bytebuffer
|
29
|
+
result = self.class.parse(bytebuffer, :result)
|
30
30
|
raise 'ERROR: unparsed bytes remaining' unless bytebuffer.empty?
|
31
31
|
# get and delete info,always present in results
|
32
32
|
system_info = result[:info]
|
33
33
|
result.delete(:info)
|
34
34
|
# make single file result like a folder
|
35
|
-
|
35
|
+
result[:dir] = [result.delete(:file)] if result.key?(:file)
|
36
36
|
# add type field for stats
|
37
|
-
if result.
|
37
|
+
if result.key?(:dir)
|
38
38
|
result[:dir].each do |file|
|
39
|
-
if file.
|
39
|
+
if file.key?(:smode)
|
40
40
|
# Converts the first character of the file mode (see 'man ls') into a type.
|
41
|
-
file[:type] = case file[:smode][0,1];when 'd' then:directory;when '-' then:file;when 'l' then:link;else
|
41
|
+
file[:type] = case file[:smode][0, 1]; when 'd' then:directory; when '-' then:file; when 'l' then:link; else; :other; end # rubocop:disable Style/Semicolon
|
42
42
|
end
|
43
43
|
end
|
44
44
|
end
|
45
45
|
# for info, second overrides first, so restore it
|
46
|
-
case result.keys.length;when 0 then result = system_info;when 1 then result = result[result.keys.first];else raise 'error';end
|
46
|
+
case result.keys.length; when 0 then result = system_info; when 1 then result = result[result.keys.first]; else raise 'error'; end
|
47
47
|
# raise error as exception
|
48
|
-
raise Error.new(result[:errno],result[:errstr],action_sym,args) if
|
48
|
+
raise Error.new(result[:errno], result[:errstr], action_sym, args) if
|
49
49
|
result.is_a?(Hash) && (result.keys.sort == TYPES_DESCR[:error][:fields].map{|i|i[:name]}.sort)
|
50
50
|
return result
|
51
51
|
end # execute_single
|
@@ -53,7 +53,8 @@ module Aspera
|
|
53
53
|
# This exception is raised when +ascmd+ returns an error.
|
54
54
|
class Error < StandardError
|
55
55
|
attr_reader :errno, :errstr, :command, :args
|
56
|
-
|
56
|
+
|
57
|
+
def initialize(errno, errstr, cmd, args); super(); @errno = errno; @errstr = errstr; @command = cmd; @args = args; end # rubocop:disable Style/Semicolon
|
57
58
|
|
58
59
|
def message; "ascmd: (#{errno}) #{errstr}"; end
|
59
60
|
|
@@ -64,46 +65,46 @@ module Aspera
|
|
64
65
|
# key = name of type
|
65
66
|
TYPES_DESCR = {
|
66
67
|
result: {decode: :field_list,
|
67
|
-
fields: [{name: :file,is_a: :stat},{name: :dir,is_a: :stat,special: :substruct},{name: :size,is_a: :size},{name: :error,is_a: :error},
|
68
|
-
{name: :info,is_a: :info},{name: :success,is_a: nil,special: :return_true},{name: :exit,is_a: nil},
|
69
|
-
{name: :df,is_a: :mnt,special: :restart_on_first},{name: :md5sum,is_a: :md5sum}]},
|
68
|
+
fields: [{name: :file, is_a: :stat}, {name: :dir, is_a: :stat, special: :substruct}, {name: :size, is_a: :size}, {name: :error, is_a: :error},
|
69
|
+
{name: :info, is_a: :info}, {name: :success, is_a: nil, special: :return_true}, {name: :exit, is_a: nil},
|
70
|
+
{name: :df, is_a: :mnt, special: :restart_on_first}, {name: :md5sum, is_a: :md5sum}]},
|
70
71
|
stat: {decode: :field_list,
|
71
|
-
fields: [{name: :name,is_a: :zstr},{name: :size,is_a: :int64},{name: :mode,is_a: :int32,check: nil},{name: :zmode,is_a: :zstr},
|
72
|
-
{name: :uid,is_a: :int32,check: nil},{name: :zuid,is_a: :zstr},{name: :gid,is_a: :int32,check: nil},{name: :zgid,is_a: :zstr},
|
73
|
-
{name: :ctime,is_a: :epoch},{name: :zctime,is_a: :zstr},{name: :mtime,is_a: :epoch},{name: :zmtime,is_a: :zstr},
|
74
|
-
{name: :atime,is_a: :epoch},{name: :zatime,is_a: :zstr},{name: :symlink,is_a: :zstr},{name: :errno,is_a: :int32},
|
75
|
-
{name: :errstr,is_a: :zstr}]},
|
72
|
+
fields: [{name: :name, is_a: :zstr}, {name: :size, is_a: :int64}, {name: :mode, is_a: :int32, check: nil}, {name: :zmode, is_a: :zstr},
|
73
|
+
{name: :uid, is_a: :int32, check: nil}, {name: :zuid, is_a: :zstr}, {name: :gid, is_a: :int32, check: nil}, {name: :zgid, is_a: :zstr},
|
74
|
+
{name: :ctime, is_a: :epoch}, {name: :zctime, is_a: :zstr}, {name: :mtime, is_a: :epoch}, {name: :zmtime, is_a: :zstr},
|
75
|
+
{name: :atime, is_a: :epoch}, {name: :zatime, is_a: :zstr}, {name: :symlink, is_a: :zstr}, {name: :errno, is_a: :int32},
|
76
|
+
{name: :errstr, is_a: :zstr}]},
|
76
77
|
info: {decode: :field_list,
|
77
|
-
fields: [{name: :platform,is_a: :zstr},{name: :version,is_a: :zstr},{name: :lang,is_a: :zstr},{name: :territory,is_a: :zstr},
|
78
|
-
{name: :codeset,is_a: :zstr},{name: :lc_ctype,is_a: :zstr},{name: :lc_numeric,is_a: :zstr},{name: :lc_time,is_a: :zstr},
|
79
|
-
{name: :lc_all,is_a: :zstr},{name: :dev,is_a: :zstr,special: :multiple},{name: :browse_caps,is_a: :zstr},
|
80
|
-
{name: :protocol,is_a: :zstr}]},
|
78
|
+
fields: [{name: :platform, is_a: :zstr}, {name: :version, is_a: :zstr}, {name: :lang, is_a: :zstr}, {name: :territory, is_a: :zstr},
|
79
|
+
{name: :codeset, is_a: :zstr}, {name: :lc_ctype, is_a: :zstr}, {name: :lc_numeric, is_a: :zstr}, {name: :lc_time, is_a: :zstr},
|
80
|
+
{name: :lc_all, is_a: :zstr}, {name: :dev, is_a: :zstr, special: :multiple}, {name: :browse_caps, is_a: :zstr},
|
81
|
+
{name: :protocol, is_a: :zstr}]},
|
81
82
|
size: {decode: :field_list,
|
82
|
-
fields: [{name: :size,is_a: :int64},{name: :fcount,is_a: :int32},{name: :dcount,is_a: :int32},{name: :failed_fcount,is_a: :int32},
|
83
|
-
{name: :failed_dcount,is_a: :int32}]},
|
83
|
+
fields: [{name: :size, is_a: :int64}, {name: :fcount, is_a: :int32}, {name: :dcount, is_a: :int32}, {name: :failed_fcount, is_a: :int32},
|
84
|
+
{name: :failed_dcount, is_a: :int32}]},
|
84
85
|
error: {decode: :field_list,
|
85
|
-
fields: [{name: :errno,is_a: :int32},{name: :errstr,is_a: :zstr}]},
|
86
|
+
fields: [{name: :errno, is_a: :int32}, {name: :errstr, is_a: :zstr}]},
|
86
87
|
mnt: {decode: :field_list,
|
87
|
-
fields: [{name: :fs,is_a: :zstr},{name: :dir,is_a: :zstr},{name: :is_a,is_a: :zstr},{name: :total,is_a: :int64},
|
88
|
-
{name: :used,is_a: :int64},{name: :free,is_a: :int64},{name: :fcount,is_a: :int64},{name: :errno,is_a: :int32},
|
89
|
-
{name: :errstr,is_a: :zstr}]},
|
90
|
-
md5sum: {decode: :field_list,fields: [{name: :md5sum,is_a: :zstr}]},
|
91
|
-
int8: {decode: :base,unpack: 'C',size: 1},
|
92
|
-
int32: {decode: :base,unpack: 'L>',size: 4},
|
93
|
-
int64: {decode: :base,unpack: 'Q>',size: 8},
|
94
|
-
epoch: {decode: :base,unpack: 'Q>',size: 8},
|
95
|
-
zstr: {decode: :base,unpack: 'Z*'},
|
88
|
+
fields: [{name: :fs, is_a: :zstr}, {name: :dir, is_a: :zstr}, {name: :is_a, is_a: :zstr}, {name: :total, is_a: :int64},
|
89
|
+
{name: :used, is_a: :int64}, {name: :free, is_a: :int64}, {name: :fcount, is_a: :int64}, {name: :errno, is_a: :int32},
|
90
|
+
{name: :errstr, is_a: :zstr}]},
|
91
|
+
md5sum: {decode: :field_list, fields: [{name: :md5sum, is_a: :zstr}]},
|
92
|
+
int8: {decode: :base, unpack: 'C', size: 1},
|
93
|
+
int32: {decode: :base, unpack: 'L>', size: 4},
|
94
|
+
int64: {decode: :base, unpack: 'Q>', size: 8},
|
95
|
+
epoch: {decode: :base, unpack: 'Q>', size: 8},
|
96
|
+
zstr: {decode: :base, unpack: 'Z*'},
|
96
97
|
blist: {decode: :buffer_list}
|
97
98
|
}.freeze
|
98
99
|
|
99
100
|
# protocol enum start at one, but array index start at zero
|
100
101
|
ENUM_START = 1
|
101
102
|
|
102
|
-
private_constant :TYPES_DESCR
|
103
|
+
private_constant :TYPES_DESCR, :ENUM_START
|
103
104
|
|
104
105
|
class << self
|
105
106
|
# get description of structure's field, @param struct_name, @param typed_buffer provides field name
|
106
|
-
def field_description(struct_name,typed_buffer)
|
107
|
+
def field_description(struct_name, typed_buffer)
|
107
108
|
result = TYPES_DESCR[struct_name][:fields][typed_buffer[:btype] - ENUM_START]
|
108
109
|
raise "Unrecognized field for #{struct_name}: #{typed_buffer[:btype]}\n#{typed_buffer[:buffer]}" if result.nil?
|
109
110
|
return result
|
@@ -112,54 +113,55 @@ module Aspera
|
|
112
113
|
# decodes the provided buffer as provided type name
|
113
114
|
# @return a decoded type.
|
114
115
|
# :base : value, :buffer_list : an array of {btype,buffer}, :field_list : a hash, or array
|
115
|
-
def parse(buffer,type_name,indent_level=nil)
|
116
|
+
def parse(buffer, type_name, indent_level=nil)
|
116
117
|
indent_level = (indent_level || -1) + 1
|
117
118
|
type_descr = TYPES_DESCR[type_name]
|
118
119
|
raise "Unexpected type #{type_name}" if type_descr.nil?
|
119
|
-
Log.log.debug
|
120
|
+
Log.log.debug{"#{' .' * indent_level}parse:#{type_name}:#{type_descr[:decode]}:#{buffer[0, 16]}...".red}
|
120
121
|
result = nil
|
121
122
|
case type_descr[:decode]
|
122
123
|
when :base
|
123
124
|
num_bytes = type_name.eql?(:zstr) ? buffer.length : type_descr[:size]
|
124
125
|
raise 'ERROR:not enough bytes' if buffer.length < num_bytes
|
125
|
-
byte_array = buffer.shift(num_bytes)
|
126
|
+
byte_array = buffer.shift(num_bytes)
|
127
|
+
byte_array = [byte_array] unless byte_array.is_a?(Array)
|
126
128
|
result = byte_array.pack('C*').unpack1(type_descr[:unpack])
|
127
|
-
Log.log.debug
|
129
|
+
Log.log.debug{"#{' .' * indent_level}-> base:#{byte_array} -> #{result}"}
|
128
130
|
result = Time.at(result) if type_name.eql?(:epoch)
|
129
131
|
when :buffer_list
|
130
132
|
result = []
|
131
|
-
|
132
|
-
btype = parse(buffer
|
133
|
-
length = parse(buffer
|
133
|
+
until buffer.empty?
|
134
|
+
btype = parse(buffer, :int8, indent_level)
|
135
|
+
length = parse(buffer, :int32, indent_level)
|
134
136
|
raise 'ERROR:not enough bytes' if buffer.length < length
|
135
137
|
value = buffer.shift(length)
|
136
|
-
result.push({btype: btype,buffer: value})
|
137
|
-
Log.log.debug
|
138
|
+
result.push({btype: btype, buffer: value})
|
139
|
+
Log.log.debug{"#{' .' * indent_level}:buffer_list[#{result.length - 1}] #{result.last}"}
|
138
140
|
end
|
139
141
|
when :field_list
|
140
142
|
# by default the result is one struct
|
141
143
|
result = {}
|
142
144
|
# get individual binary fields
|
143
|
-
parse(buffer
|
145
|
+
parse(buffer, :blist, indent_level).each do |typed_buffer|
|
144
146
|
# what type of field is it ?
|
145
|
-
field_info = field_description(type_name,typed_buffer)
|
146
|
-
Log.log.debug
|
147
|
+
field_info = field_description(type_name, typed_buffer)
|
148
|
+
Log.log.debug{"#{' .' * indent_level}+ field(special=#{field_info[:special]})=#{field_info[:name]}".green}
|
147
149
|
case field_info[:special]
|
148
150
|
when nil
|
149
|
-
result[field_info[:name]] = parse(typed_buffer[:buffer],field_info[:is_a],indent_level)
|
151
|
+
result[field_info[:name]] = parse(typed_buffer[:buffer], field_info[:is_a], indent_level)
|
150
152
|
when :return_true
|
151
153
|
result[field_info[:name]] = true
|
152
154
|
when :substruct
|
153
|
-
result[field_info[:name]] = parse(typed_buffer[:buffer]
|
155
|
+
result[field_info[:name]] = parse(typed_buffer[:buffer], :blist, indent_level).map{|r|parse(r[:buffer], field_info[:is_a], indent_level)}
|
154
156
|
when :multiple
|
155
157
|
result[field_info[:name]] ||= []
|
156
|
-
result[field_info[:name]].push(parse(typed_buffer[:buffer],field_info[:is_a],indent_level))
|
158
|
+
result[field_info[:name]].push(parse(typed_buffer[:buffer], field_info[:is_a], indent_level))
|
157
159
|
when :restart_on_first
|
158
160
|
fl = result[field_info[:name]] = []
|
159
|
-
parse(typed_buffer[:buffer]
|
161
|
+
parse(typed_buffer[:buffer], :blist, indent_level).map do |tb|
|
160
162
|
fl.push({}) if tb[:btype].eql?(ENUM_START)
|
161
|
-
fi = field_description(field_info[:is_a],tb)
|
162
|
-
fl.last[fi[:name]] = parse(tb[:buffer],fi[:is_a],indent_level)
|
163
|
+
fi = field_description(field_info[:is_a], tb)
|
164
|
+
fl.last[fi[:name]] = parse(tb[:buffer], fi[:is_a], indent_level)
|
163
165
|
end
|
164
166
|
end
|
165
167
|
end
|
data/lib/aspera/ats_api.rb
CHANGED
@@ -4,7 +4,7 @@ require 'aspera/log'
|
|
4
4
|
require 'aspera/rest'
|
5
5
|
|
6
6
|
module Aspera
|
7
|
-
class AtsApi < Rest
|
7
|
+
class AtsApi < Aspera::Rest
|
8
8
|
# currently supported clouds
|
9
9
|
# Note to Aspera: shall be an API call
|
10
10
|
CLOUD_NAME = {
|
@@ -19,7 +19,7 @@ module Aspera
|
|
19
19
|
private_constant :CLOUD_NAME
|
20
20
|
|
21
21
|
class << self
|
22
|
-
def base_url;'https://ats.aspera.io';end
|
22
|
+
def base_url; 'https://ats.aspera.io'; end
|
23
23
|
end
|
24
24
|
|
25
25
|
def initialize
|
@@ -28,14 +28,14 @@ module Aspera
|
|
28
28
|
@all_servers_cache = nil
|
29
29
|
end
|
30
30
|
|
31
|
-
def cloud_names;CLOUD_NAME;end
|
31
|
+
def cloud_names; CLOUD_NAME; end
|
32
32
|
|
33
33
|
# all available ATS servers
|
34
34
|
# NOTE to Aspera: an API shall be created to retrieve all servers at once
|
35
35
|
def all_servers
|
36
36
|
if @all_servers_cache.nil?
|
37
37
|
@all_servers_cache = []
|
38
|
-
CLOUD_NAME.
|
38
|
+
CLOUD_NAME.each_key do |name|
|
39
39
|
read("servers/#{name.to_s.upcase}")[:data].each do |i|
|
40
40
|
@all_servers_cache.push(i)
|
41
41
|
end
|
@@ -6,33 +6,36 @@ require 'aspera/cli/plugin'
|
|
6
6
|
module Aspera
|
7
7
|
module Cli
|
8
8
|
# base class for applications supporting basic authentication
|
9
|
-
class BasicAuthPlugin < Plugin
|
9
|
+
class BasicAuthPlugin < Aspera::Cli::Plugin
|
10
10
|
class << self
|
11
11
|
def register_options(env)
|
12
|
-
env[:options].add_opt_simple(:url,'URL of application, e.g. https://org.asperafiles.com')
|
13
|
-
env[:options].add_opt_simple(:username,'username to log in')
|
14
|
-
env[:options].add_opt_simple(:password,"user's password")
|
12
|
+
env[:options].add_opt_simple(:url, 'URL of application, e.g. https://org.asperafiles.com')
|
13
|
+
env[:options].add_opt_simple(:username, 'username to log in')
|
14
|
+
env[:options].add_opt_simple(:password, "user's password")
|
15
15
|
env[:options].parse_options!
|
16
16
|
end
|
17
17
|
end
|
18
18
|
|
19
19
|
def initialize(env)
|
20
20
|
super(env)
|
21
|
-
|
22
|
-
self.class.register_options(env)
|
21
|
+
self.class.register_options(env) unless env[:skip_basic_auth_options]
|
23
22
|
end
|
24
23
|
|
25
24
|
# returns a Rest object with basic auth
|
26
|
-
def
|
27
|
-
api_url = options.get_option(:url,is_type: :mandatory)
|
25
|
+
def basic_auth_params(subpath=nil)
|
26
|
+
api_url = options.get_option(:url, is_type: :mandatory)
|
28
27
|
api_url = api_url + '/' + subpath unless subpath.nil?
|
29
|
-
return
|
28
|
+
return {
|
30
29
|
base_url: api_url,
|
31
30
|
auth: {
|
32
31
|
type: :basic,
|
33
|
-
username: options.get_option(:username,is_type: :mandatory),
|
34
|
-
password: options.get_option(:password,is_type: :mandatory)
|
35
|
-
}}
|
32
|
+
username: options.get_option(:username, is_type: :mandatory),
|
33
|
+
password: options.get_option(:password, is_type: :mandatory)
|
34
|
+
}}
|
35
|
+
end
|
36
|
+
|
37
|
+
def basic_auth_api(subpath=nil)
|
38
|
+
return Rest.new(basic_auth_params(subpath))
|
36
39
|
end
|
37
40
|
end # BasicAuthPlugin
|
38
41
|
end # Cli
|
@@ -19,7 +19,7 @@ module Aspera
|
|
19
19
|
# decode comma separated table text
|
20
20
|
def decode_csvt(value)
|
21
21
|
col_titles = nil
|
22
|
-
|
22
|
+
hash_array = []
|
23
23
|
CSV.parse(value).each do |values|
|
24
24
|
next if values.empty?
|
25
25
|
if col_titles.nil?
|
@@ -27,11 +27,11 @@ module Aspera
|
|
27
27
|
else
|
28
28
|
entry = {}
|
29
29
|
col_titles.each{|title|entry[title] = values.shift}
|
30
|
-
|
30
|
+
hash_array.push(entry)
|
31
31
|
end
|
32
32
|
end
|
33
|
-
Log.log.warn('Titled CSV file without any line') if
|
34
|
-
return
|
33
|
+
Log.log.warn('Titled CSV file without any line') if hash_array.empty?
|
34
|
+
return hash_array
|
35
35
|
end
|
36
36
|
end
|
37
37
|
|
@@ -54,7 +54,7 @@ module Aspera
|
|
54
54
|
path: lambda{|v|File.expand_path(v)},
|
55
55
|
env: lambda{|v|ENV[v]},
|
56
56
|
uri: lambda{|v|UriReader.read(v)},
|
57
|
-
stdin: lambda{|v|raise 'no value allowed for stdin' unless v.empty
|
57
|
+
stdin: lambda{|v|raise 'no value allowed for stdin' unless v.empty?; $stdin.read} # rubocop:disable Style/Semicolon
|
58
58
|
}
|
59
59
|
# other handlers can be set using set_handler, e.g. preset is reader in config plugin
|
60
60
|
}
|
@@ -62,12 +62,12 @@ module Aspera
|
|
62
62
|
|
63
63
|
public
|
64
64
|
|
65
|
-
def modifiers
|
65
|
+
def modifiers; @handlers.keys.map{|i|@handlers[i].keys}.flatten.map(&:to_s); end
|
66
66
|
|
67
67
|
# add a new :reader or :decoder
|
68
68
|
# decoder can be chained, reader is last one on right
|
69
|
-
def set_handler(name,type,method)
|
70
|
-
Log.log.debug
|
69
|
+
def set_handler(name, type, method)
|
70
|
+
Log.log.debug{"setting #{type} handler for #{name}"}
|
71
71
|
raise 'name must be Symbol' unless name.is_a?(Symbol)
|
72
72
|
raise "type #{type} must be one of #{@handlers.keys}" unless @handlers.key?(type)
|
73
73
|
@handlers[type][name] = method
|
@@ -84,7 +84,7 @@ module Aspera
|
|
84
84
|
value = m[2]
|
85
85
|
end
|
86
86
|
# then read value
|
87
|
-
@handlers[:reader].each do |reader,method|
|
87
|
+
@handlers[:reader].each do |reader, method|
|
88
88
|
if (m = value.match(/^@#{reader}:(.*)/))
|
89
89
|
value = method.call(m[1])
|
90
90
|
break
|
@@ -8,7 +8,7 @@ require 'pp'
|
|
8
8
|
module Aspera
|
9
9
|
module Cli
|
10
10
|
# Take care of output
|
11
|
-
class
|
11
|
+
class Formatter
|
12
12
|
FIELDS_ALL = 'ALL'
|
13
13
|
FIELDS_DEFAULT = 'DEF'
|
14
14
|
CSV_RECORD_SEPARATOR = "\n"
|
@@ -17,16 +17,16 @@ module Aspera
|
|
17
17
|
DISPLAY_FORMATS = %i[text nagios ruby json jsonpp yaml table csv].freeze
|
18
18
|
# user output levels
|
19
19
|
DISPLAY_LEVELS = %i[info data error].freeze
|
20
|
-
CONF_OVERVIEW_KEYS
|
20
|
+
CONF_OVERVIEW_KEYS = %w[config parameter value].freeze
|
21
21
|
|
22
|
-
private_constant :FIELDS_ALL
|
22
|
+
private_constant :FIELDS_ALL, :FIELDS_DEFAULT, :DISPLAY_FORMATS, :DISPLAY_LEVELS, :CSV_RECORD_SEPARATOR, :CSV_FIELD_SEPARATOR,
|
23
23
|
:CONF_OVERVIEW_KEYS
|
24
24
|
|
25
25
|
class << self
|
26
26
|
# special for Aspera on Cloud display node
|
27
27
|
# {"param" => [{"name"=>"foo","value"=>"bar"}]} will be expanded to {"param.foo" : "bar"}
|
28
28
|
def flatten_name_value_list(hash)
|
29
|
-
hash.keys.each do |k|
|
29
|
+
hash.keys.each do |k| # rubocop:disable Style/HashEachMethods
|
30
30
|
v = hash[k]
|
31
31
|
next unless v.is_a?(Array) && v.map(&:class).uniq.eql?([Hash]) && v.map(&:keys).flatten.sort.uniq.eql?(%w[name value])
|
32
32
|
v.each do |pair|
|
@@ -36,30 +36,30 @@ module Aspera
|
|
36
36
|
end
|
37
37
|
end
|
38
38
|
|
39
|
-
def flatten_config_overview(
|
39
|
+
def flatten_config_overview(hash_array_conf)
|
40
40
|
r = []
|
41
|
-
|
42
|
-
preset.each do |parameter,value|
|
43
|
-
r.push(CONF_OVERVIEW_KEYS.zip([config,parameter,value]).to_h)
|
41
|
+
hash_array_conf.each do |config, preset|
|
42
|
+
preset.each do |parameter, value|
|
43
|
+
r.push(CONF_OVERVIEW_KEYS.zip([config, parameter, SecretHider.deep_remove_secret(value).to_s]).to_h)
|
44
44
|
end
|
45
45
|
end
|
46
46
|
return r
|
47
47
|
end
|
48
48
|
|
49
49
|
def simple_hash?(h)
|
50
|
-
!(h.values.any?{|v|[Hash,Array].any?{|c|v.is_a?(c)}})
|
50
|
+
!(h.values.any?{|v|[Hash, Array].any?{|c|v.is_a?(c)}})
|
51
51
|
end
|
52
52
|
|
53
53
|
# recursive function to modify a hash
|
54
54
|
# @param source [Hash] to be modified
|
55
|
-
# @param expand_last [
|
55
|
+
# @param expand_last [TrueClass,FalseClass] true if last level is not
|
56
56
|
# @param result [Hash] new hash flattened
|
57
57
|
# @param prefix [String] true if last level is not
|
58
|
-
def flattened_object(source,result: {},prefix: '',expand_last: false)
|
59
|
-
Log.log.debug
|
60
|
-
source.each do |k,v|
|
58
|
+
def flattened_object(source, result: {}, prefix: '', expand_last: false)
|
59
|
+
Log.log.debug{"(#{expand_last})[#{simple_hash?(source)}] -#{source.values}- \n-#{source}-"}
|
60
|
+
source.each do |k, v|
|
61
61
|
if v.is_a?(Hash) && !(expand_last && simple_hash?(v))
|
62
|
-
flattened_object(v,result: result,prefix: prefix + k.to_s + '.',expand_last: expand_last)
|
62
|
+
flattened_object(v, result: result, prefix: prefix + k.to_s + '.', expand_last: expand_last)
|
63
63
|
else
|
64
64
|
result[prefix + k.to_s] = v
|
65
65
|
end
|
@@ -68,8 +68,8 @@ module Aspera
|
|
68
68
|
end
|
69
69
|
end
|
70
70
|
|
71
|
-
attr_accessor :option_flat_hash
|
72
|
-
:option_select
|
71
|
+
attr_accessor :option_flat_hash, :option_transpose_single, :option_format, :option_display, :option_fields, :option_table_style,
|
72
|
+
:option_select, :option_show_secrets
|
73
73
|
|
74
74
|
# adds options but does not parse
|
75
75
|
def initialize(opt_mgr)
|
@@ -81,29 +81,29 @@ module Aspera
|
|
81
81
|
@option_flat_hash = true
|
82
82
|
@option_transpose_single = true
|
83
83
|
@option_show_secrets = false
|
84
|
-
opt_mgr.set_obj_attr(:format,self
|
85
|
-
opt_mgr.set_obj_attr(:display,self
|
86
|
-
opt_mgr.set_obj_attr(:fields,self
|
87
|
-
opt_mgr.set_obj_attr(:select,self
|
88
|
-
opt_mgr.set_obj_attr(:table_style,self
|
89
|
-
opt_mgr.set_obj_attr(:flat_hash,self
|
90
|
-
opt_mgr.set_obj_attr(:transpose_single,self
|
91
|
-
opt_mgr.set_obj_attr(:show_secrets,self
|
92
|
-
opt_mgr.add_opt_list(:format,DISPLAY_FORMATS,'output format')
|
93
|
-
opt_mgr.add_opt_list(:display,DISPLAY_LEVELS,'output only some information')
|
94
|
-
opt_mgr.add_opt_simple(:fields,"comma separated list of fields, or #{FIELDS_ALL}, or #{FIELDS_DEFAULT}")
|
95
|
-
opt_mgr.add_opt_simple(:select,'select only some items in lists, extended value: hash (column, value)')
|
96
|
-
opt_mgr.add_opt_simple(:table_style,'table display style')
|
97
|
-
opt_mgr.add_opt_boolean(:flat_hash,'display hash values as additional keys')
|
98
|
-
opt_mgr.add_opt_boolean(:transpose_single,'single object fields output vertically')
|
99
|
-
opt_mgr.add_opt_boolean(:show_secrets,'show secrets on command output')
|
84
|
+
opt_mgr.set_obj_attr(:format, self, :option_format)
|
85
|
+
opt_mgr.set_obj_attr(:display, self, :option_display)
|
86
|
+
opt_mgr.set_obj_attr(:fields, self, :option_fields)
|
87
|
+
opt_mgr.set_obj_attr(:select, self, :option_select)
|
88
|
+
opt_mgr.set_obj_attr(:table_style, self, :option_table_style)
|
89
|
+
opt_mgr.set_obj_attr(:flat_hash, self, :option_flat_hash)
|
90
|
+
opt_mgr.set_obj_attr(:transpose_single, self, :option_transpose_single)
|
91
|
+
opt_mgr.set_obj_attr(:show_secrets, self, :option_show_secrets)
|
92
|
+
opt_mgr.add_opt_list(:format, DISPLAY_FORMATS, 'output format')
|
93
|
+
opt_mgr.add_opt_list(:display, DISPLAY_LEVELS, 'output only some information')
|
94
|
+
opt_mgr.add_opt_simple(:fields, "comma separated list of fields, or #{FIELDS_ALL}, or #{FIELDS_DEFAULT}")
|
95
|
+
opt_mgr.add_opt_simple(:select, 'select only some items in lists, extended value: hash (column, value)')
|
96
|
+
opt_mgr.add_opt_simple(:table_style, 'table display style')
|
97
|
+
opt_mgr.add_opt_boolean(:flat_hash, 'display hash values as additional keys')
|
98
|
+
opt_mgr.add_opt_boolean(:transpose_single, 'single object fields output vertically')
|
99
|
+
opt_mgr.add_opt_boolean(:show_secrets, 'show secrets on command output')
|
100
100
|
end
|
101
101
|
|
102
102
|
# main output method
|
103
103
|
# data: for requested data, not displayed if level==error
|
104
104
|
# info: additional info, displayed if level==info
|
105
105
|
# error: always displayed on stderr
|
106
|
-
def display_message(message_level,message)
|
106
|
+
def display_message(message_level, message)
|
107
107
|
case message_level
|
108
108
|
when :data then $stdout.puts(message) unless @option_display.eql?(:error)
|
109
109
|
when :info then $stdout.puts(message) if @option_display.eql?(:info)
|
@@ -113,10 +113,10 @@ module Aspera
|
|
113
113
|
end
|
114
114
|
|
115
115
|
def display_status(status)
|
116
|
-
display_message(:info,status)
|
116
|
+
display_message(:info, status)
|
117
117
|
end
|
118
118
|
|
119
|
-
def result_default_fields(results,table_rows_hash_val)
|
119
|
+
def result_default_fields(results, table_rows_hash_val)
|
120
120
|
unless results[:fields].nil?
|
121
121
|
raise "internal error: [fields] must be Array, not #{results[:fields].class}" unless results[:fields].is_a?(Array)
|
122
122
|
if results[:fields].first.eql?(:all_but) && !table_rows_hash_val.empty?
|
@@ -129,37 +129,37 @@ module Aspera
|
|
129
129
|
return table_rows_hash_val.first.keys
|
130
130
|
end
|
131
131
|
|
132
|
-
def result_all_fields(_results,table_rows_hash_val)
|
132
|
+
def result_all_fields(_results, table_rows_hash_val)
|
133
133
|
raise 'internal error: must be array' unless table_rows_hash_val.is_a?(Array)
|
134
|
-
# get the list of all column names used in all lines, not just
|
135
|
-
return table_rows_hash_val.each_with_object({}){|v,m|v.
|
134
|
+
# get the list of all column names used in all lines, not just first one, as all lines may have different columns
|
135
|
+
return table_rows_hash_val.each_with_object({}){|v, m|v.each_key{|c|m[c] = true}; }.keys
|
136
136
|
end
|
137
137
|
|
138
138
|
# this method displays the results, especially the table format
|
139
139
|
def display_results(results)
|
140
140
|
raise "INTERNAL ERROR, result must be Hash (got: #{results.class}: #{results})" unless results.is_a?(Hash)
|
141
|
-
raise 'INTERNAL ERROR, result must have type' unless results.
|
142
|
-
raise 'INTERNAL ERROR, result must have data' unless results.
|
141
|
+
raise 'INTERNAL ERROR, result must have type' unless results.key?(:type)
|
142
|
+
raise 'INTERNAL ERROR, result must have data' unless results.key?(:data) || %i[empty nothing].include?(results[:type])
|
143
143
|
res_data = results[:data]
|
144
|
-
# for config
|
144
|
+
# for config overview, it is name and value
|
145
145
|
is_config_overview = res_data.is_a?(Array) && !res_data.empty? && res_data.first.is_a?(Hash) && res_data.first.keys.sort.eql?(CONF_OVERVIEW_KEYS)
|
146
146
|
SecretHider.deep_remove_secret(res_data, is_name_value: is_config_overview) unless @option_show_secrets || @option_display.eql?(:data)
|
147
147
|
# comma separated list in string format
|
148
148
|
user_asked_fields_list_str = @option_fields
|
149
149
|
case @option_format
|
150
150
|
when :text
|
151
|
-
display_message(:data,res_data.to_s)
|
151
|
+
display_message(:data, res_data.to_s)
|
152
152
|
when :nagios
|
153
153
|
Nagios.process(res_data)
|
154
154
|
when :ruby
|
155
|
-
display_message(:data,PP.pp(res_data
|
155
|
+
display_message(:data, PP.pp(res_data, +''))
|
156
156
|
when :json
|
157
|
-
display_message(:data,JSON.generate(res_data))
|
157
|
+
display_message(:data, JSON.generate(res_data))
|
158
158
|
when :jsonpp
|
159
|
-
display_message(:data,JSON.pretty_generate(res_data))
|
159
|
+
display_message(:data, JSON.pretty_generate(res_data))
|
160
160
|
when :yaml
|
161
|
-
display_message(:data,res_data.to_yaml)
|
162
|
-
when :table
|
161
|
+
display_message(:data, res_data.to_yaml)
|
162
|
+
when :table, :csv
|
163
163
|
if !@option_transpose_single && results[:type].eql?(:single_object)
|
164
164
|
results[:type] = :object_list
|
165
165
|
res_data = [res_data]
|
@@ -171,17 +171,17 @@ module Aspera
|
|
171
171
|
table_rows_hash_val = res_data
|
172
172
|
final_table_columns = nil
|
173
173
|
if @option_flat_hash
|
174
|
-
table_rows_hash_val.map!{|obj|self.class.flattened_object(obj,expand_last: results[:option_expand_last])}
|
174
|
+
table_rows_hash_val.map!{|obj|self.class.flattened_object(obj, expand_last: results[:option_expand_last])}
|
175
175
|
end
|
176
176
|
final_table_columns =
|
177
177
|
case user_asked_fields_list_str
|
178
|
-
when FIELDS_DEFAULT then result_default_fields(results,table_rows_hash_val)
|
179
|
-
when FIELDS_ALL then result_all_fields(results,table_rows_hash_val)
|
178
|
+
when FIELDS_DEFAULT then result_default_fields(results, table_rows_hash_val)
|
179
|
+
when FIELDS_ALL then result_all_fields(results, table_rows_hash_val)
|
180
180
|
else
|
181
181
|
if user_asked_fields_list_str.start_with?('+')
|
182
|
-
result_default_fields(results,table_rows_hash_val).push(*user_asked_fields_list_str.gsub(/^\+/,'').split(','))
|
182
|
+
result_default_fields(results, table_rows_hash_val).push(*user_asked_fields_list_str.gsub(/^\+/, '').split(','))
|
183
183
|
elsif user_asked_fields_list_str.start_with?('-')
|
184
|
-
result_default_fields(results,table_rows_hash_val).reject{|i| user_asked_fields_list_str.gsub(/^-/,'').split(',').include?(i)}
|
184
|
+
result_default_fields(results, table_rows_hash_val).reject{|i| user_asked_fields_list_str.gsub(/^-/, '').split(',').include?(i)}
|
185
185
|
else
|
186
186
|
user_asked_fields_list_str.split(',')
|
187
187
|
end
|
@@ -191,7 +191,7 @@ module Aspera
|
|
191
191
|
raise "internal error: expecting Hash: got #{res_data.class}: #{res_data}" unless res_data.is_a?(Hash)
|
192
192
|
final_table_columns = results[:columns] || %w[key value]
|
193
193
|
if @option_flat_hash
|
194
|
-
res_data=self.class.flattened_object(res_data,expand_last: results[:option_expand_last])
|
194
|
+
res_data = self.class.flattened_object(res_data, expand_last: results[:option_expand_last])
|
195
195
|
self.class.flatten_name_value_list(res_data)
|
196
196
|
end
|
197
197
|
asked_fields =
|
@@ -206,22 +206,22 @@ module Aspera
|
|
206
206
|
final_table_columns = [results[:name]]
|
207
207
|
table_rows_hash_val = res_data.map { |i| { results[:name] => i } }
|
208
208
|
when :empty # no table
|
209
|
-
display_message(:info,'empty')
|
209
|
+
display_message(:info, 'empty')
|
210
210
|
return
|
211
211
|
when :nothing # no result expected
|
212
212
|
Log.log.debug('no result expected')
|
213
213
|
return
|
214
214
|
when :status # no table
|
215
215
|
# :status displays a simple message
|
216
|
-
display_message(:info,res_data)
|
216
|
+
display_message(:info, res_data)
|
217
217
|
return
|
218
218
|
when :text # no table
|
219
219
|
# :status displays a simple message
|
220
|
-
display_message(:data,res_data)
|
220
|
+
display_message(:data, res_data)
|
221
221
|
return
|
222
222
|
when :other_struct # no table
|
223
223
|
# :other_struct is any other type of structure
|
224
|
-
display_message(:data,PP.pp(res_data
|
224
|
+
display_message(:data, PP.pp(res_data, +''))
|
225
225
|
return
|
226
226
|
else
|
227
227
|
raise "unknown data type: #{results[:type]}"
|
@@ -229,14 +229,14 @@ module Aspera
|
|
229
229
|
# here we expect: table_rows_hash_val and final_table_columns
|
230
230
|
raise 'no field specified' if final_table_columns.nil?
|
231
231
|
if table_rows_hash_val.empty?
|
232
|
-
display_message(:info,'empty'.gray) unless @option_format.eql?(:csv)
|
232
|
+
display_message(:info, 'empty'.gray) unless @option_format.eql?(:csv)
|
233
233
|
return
|
234
234
|
end
|
235
235
|
# convert to string with special function. here table_rows_hash_val is an array of hash
|
236
|
-
table_rows_hash_val = results[:textify].call(table_rows_hash_val) if results.
|
236
|
+
table_rows_hash_val = results[:textify].call(table_rows_hash_val) if results.key?(:textify)
|
237
237
|
unless @option_select.nil? || (@option_select.respond_to?(:empty?) && @option_select.empty?)
|
238
|
-
raise CliBadArgument,"expecting hash for select, have #{@option_select.class}: #{@option_select}" unless @option_select.is_a?(Hash)
|
239
|
-
@option_select.each{|k,v|table_rows_hash_val.select!{|i|i[k].eql?(v)}}
|
238
|
+
raise CliBadArgument, "expecting hash for select, have #{@option_select.class}: #{@option_select}" unless @option_select.is_a?(Hash)
|
239
|
+
@option_select.each{|k, v|table_rows_hash_val.select!{|i|i[k].eql?(v)}}
|
240
240
|
end
|
241
241
|
# convert data to string, and keep only display fields
|
242
242
|
final_table_rows = table_rows_hash_val.map { |r| final_table_columns.map { |c| r[c].to_s } }
|
@@ -246,20 +246,20 @@ module Aspera
|
|
246
246
|
when :table
|
247
247
|
style = @option_table_style.chars
|
248
248
|
# display the table !
|
249
|
-
#display_message(:data,Text::Table.new(
|
250
|
-
#head: final_table_columns,
|
251
|
-
#rows: final_table_rows,
|
252
|
-
#horizontal_boundary: style[0],
|
253
|
-
#vertical_boundary: style[1],
|
254
|
-
#boundary_intersection: style[2]))
|
255
|
-
display_message(:data,Terminal::Table.new(
|
249
|
+
# display_message(:data,Text::Table.new(
|
250
|
+
# head: final_table_columns,
|
251
|
+
# rows: final_table_rows,
|
252
|
+
# horizontal_boundary: style[0],
|
253
|
+
# vertical_boundary: style[1],
|
254
|
+
# boundary_intersection: style[2]))
|
255
|
+
display_message(:data, Terminal::Table.new(
|
256
256
|
headings: final_table_columns,
|
257
257
|
rows: final_table_rows,
|
258
258
|
border_x: style[0],
|
259
259
|
border_y: style[1],
|
260
260
|
border_i: style[2]))
|
261
261
|
when :csv
|
262
|
-
display_message(:data,final_table_rows.map{|t| t.join(CSV_FIELD_SEPARATOR)}.join(CSV_RECORD_SEPARATOR))
|
262
|
+
display_message(:data, final_table_rows.map{|t| t.join(CSV_FIELD_SEPARATOR)}.join(CSV_RECORD_SEPARATOR))
|
263
263
|
end
|
264
264
|
end
|
265
265
|
end
|