asperalm 0.10.17 → 0.11

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: fc5f8a17f26353f1a02d57d5a725238f9c4b45fbfb099123aa4dd83d59bf856d
4
- data.tar.gz: 4f7ab2fe089523ce1df56ca4920984ee909946309855c0862f02ae5db4d1a58c
3
+ metadata.gz: 1394f7859bed75467ae576ad24b5f4d163e863ea4d0f5479f0205740f4727e88
4
+ data.tar.gz: 75661a6cd9fb1e50ccbf3b8a2695e240c2224804f50fc89d3a19447f70bd2a5f
5
5
  SHA512:
6
- metadata.gz: b49e09f1a2066c7cdfd9fec565626f1fa48ace34e2fa3b8a47bc7874a9b7164bb0a63e46aa01480637aa4d841f7ca582e1f9c82f809ac533790a070eea24ee61
7
- data.tar.gz: '09b70c5d9bf92d99351073e2871498d5e4f230c35f694607f43d0f794f5371bb113774fe0aeda8c7b35ef7744143fc4a7b7970d8cfe9c7649f2ef75dbd3ff119'
6
+ metadata.gz: 9a97c2ec9571ee1bf124883abb9f949f8888166d608df1a4889d80f511b8e25fa7dc136ea35d7fb47024a7996b0a69a0ae42ce50bc6f5fcbad962b4485974b81
7
+ data.tar.gz: 8952b7aae4ea142fdb777954799a831531c03248b43dc0e88bf89f4509e6cfdc62ca9354dc9e7ab7562434b9e3e50b6becacd682e54dc29ac20bd11fbc58724f
data/README.md CHANGED
@@ -1,6 +1,6 @@
1
1
  # Asperalm - A Ruby library for Aspera transfers and "Amelia", the _Multi Layer IBM Aspera_ Command Line Tool
2
2
 
3
- Version : 0.10.17
3
+ Version : 0.11
4
4
 
5
5
 
6
6
  _Laurent/2016-2018_
@@ -39,7 +39,7 @@ Once the gem is installed, `mlia` shall be accessible:
39
39
 
40
40
  ```bash
41
41
  $ mlia --version
42
- 0.10.17
42
+ 0.11
43
43
  ```
44
44
 
45
45
  ## First use
@@ -994,6 +994,9 @@ UNDER CONSTRUCTION<br/>
994
994
  <tr><td>remove_empty_directories</td><td></td><td>boolean</td><td></td><td class="yes">Y</td><td></td><td>?</td><td>Specifies whether to remove empty directories.</td></tr>
995
995
  <tr><td>multi_session</td><td>1</td><td>integer</td><td class="no">N</td><td class="yes">Y</td><td class="no">N</td><td>-C</td><td>Specifies how many parts the transfer is in.</td></tr>
996
996
  <tr><td>multi_session_threshold</td><td>null</td><td>integer</td><td class="no">N</td><td class="yes">Y</td><td class="no">N</td><td>-</td><td>in bytes</td></tr>
997
+ <tr><td>exclude_newer_than</td><td></td><td>integer</td><td class="yes">Y</td><td></td><td></td><td>--exclude-newer-than</td><td>-</td></tr>
998
+ <tr><td>exclude_older_than</td><td></td><td>integer</td><td class="yes">Y</td><td></td><td></td><td>--exclude-older-than</td><td>-</td></tr>
999
+ <tr><td>preserve_acls</td><td></td><td>string</td><td class="yes">Y</td><td></td><td></td><td>--preserve-acls</td><td>-</td></tr>
997
1000
  <tr><td>dgram_size</td><td></td><td>integer</td><td class="yes">Y</td><td></td><td></td><td>-Z</td><td>in bytes</td></tr>
998
1001
  <tr><td>compression</td><td></td><td>integer</td><td></td><td></td><td></td><td></td><td>ascp4 only, 0 / 1?</td></tr>
999
1002
  <tr><td>read_threads</td><td></td><td>integer</td><td></td><td></td><td></td><td>-</td><td>ascp4 only</td></tr>
@@ -1016,6 +1019,12 @@ UNDER CONSTRUCTION<br/>
1016
1019
  <tr><td>EX_http_proxy_url</td><td>-</td><td>string</td><td class="yes">Y</td><td class="no">N</td><td class="no">N</td><td>-x</td><td>Specify the proxy server address used by HTTP Fallback</td></tr>
1017
1020
  <tr><td>EX_ascp_args</td><td>-</td><td>array</td><td class="yes">Y</td><td class="no">N</td><td class="no">N</td><td>same</td><td>Add command line arguments to ascp</td></tr>
1018
1021
  <tr><td>EX_http_transfer_jpeg</td><td>0</td><td>integer</td><td class="yes">Y</td><td class="no">N</td><td class="no">N</td><td>-j</td><td>HTTP transfers as JPEG file</td></tr>
1022
+ <tr><td>EX_license_text</td><td>-</td><td>string</td><td class="yes">Y</td><td class="no">N</td><td class="no">N</td><td>LICENSE</td><td>license file text</td></tr>
1023
+ <tr><td>EX_file_list</td><td>-</td><td>string</td><td class="yes">Y</td><td class="no">N</td><td class="no">N</td><td>--file-list</td><td>source file list</td></tr>
1024
+ <tr><td>EX_file_pair_list</td><td>-</td><td>string</td><td class="yes">Y</td><td class="no">N</td><td class="no">N</td><td>--file-pair-list</td><td>source file pair list</td></tr>
1025
+ <tr><td>EX_multi_session_part</td><td>-</td><td>string</td><td class="yes">Y</td><td class="no">N</td><td class="no">N</td><td>-C</td><td>part for multisession</td></tr>
1026
+ <tr><td>EX_no_read</td><td>-</td><td>-</td><td class="yes">Y</td><td class="no">N</td><td class="no">N</td><td>--no-read</td><td>no read source</td></tr>
1027
+ <tr><td>EX_no_write</td><td>-</td><td>-</td><td class="yes">Y</td><td class="no">N</td><td class="no">N</td><td>--no-write</td><td>no write estination</td></tr>
1019
1028
  </table>
1020
1029
 
1021
1030
 
@@ -1364,7 +1373,7 @@ mlia sync start --parameters=@json:'{"sessions":[{"name":"test","reset":true,"re
1364
1373
  ```bash
1365
1374
  $ mlia -h
1366
1375
  NAME
1367
- mlia -- a command line tool for Aspera Applications (v0.10.17)
1376
+ mlia -- a command line tool for Aspera Applications (v0.11)
1368
1377
 
1369
1378
  SYNOPSIS
1370
1379
  mlia COMMANDS [OPTIONS] [ARGS]
@@ -1632,7 +1641,8 @@ OPTIONS:
1632
1641
  --url=VALUE URL of application, e.g. https://org.asperafiles.com
1633
1642
  --username=VALUE username to log in
1634
1643
  --password=VALUE user's password
1635
- --ssh-keys=VALUE ssh key path list
1644
+ --ssh-keys=VALUE ssh key path list (Array or single)
1645
+ --ssh-options=VALUE ssh options (Hash)
1636
1646
  --cmd-prefix=VALUE prefix to add for as cmd execution, e.g. sudo or /opt/aspera/bin
1637
1647
 
1638
1648
 
@@ -2288,7 +2298,7 @@ This plugin works at FASP level (SSH/ascp/ascmd) and does not use the node API.
2288
2298
 
2289
2299
  Both password and SSH keys auth are supported.
2290
2300
 
2291
- Multiple SSH key paths can be provided. The value of the parameter can be a single value or an array. Each value is a path to a private key and is expanded ("~" is replaced with the user's home folder).
2301
+ Multiple SSH key paths can be provided. The value of the parameter `ssh_keys` can be a single value or an array. Each value is a path to a private key and is expanded ("~" is replaced with the user's home folder).
2292
2302
 
2293
2303
  Examples:
2294
2304
 
@@ -2298,6 +2308,21 @@ $ mlia server --ssh-keys=@list:,~/.ssh/id_rsa
2298
2308
  $ mlia server --ssh-keys=@json:'["~/.ssh/id_rsa"]'
2299
2309
  ```
2300
2310
 
2311
+ The underlying ssh library `net::ssh` provides several options that may be used
2312
+ depending on environment. By default the ssh library expect that an ssh-agent is running, but if you get the error message:
2313
+
2314
+ ```
2315
+ ERROR -- net.ssh.authentication.agent: could not connect to ssh-agent: Agent not configured
2316
+ ```
2317
+
2318
+ This means that you dont have such agent, check env var: `SSH_AGENT_SOCK`, check if the key is protected with a passphrase. [check the manual](https://net-ssh.github.io/ssh/v1/chapter-2.html#s2)
2319
+
2320
+ To diable use of `ssh-agent`, use the option `ssh_option` like this (or set in preset):
2321
+
2322
+ ```
2323
+ $ mlia server --ssh-options=@ruby:'{use_agent: false}' ...
2324
+ ```
2325
+
2301
2326
  ### Example
2302
2327
 
2303
2328
  One can test the "server" application using the well known demo server:
@@ -3091,6 +3116,18 @@ So, it evolved into `mlia`:
3091
3116
 
3092
3117
  # Release Notes
3093
3118
 
3119
+ * 0.11
3120
+
3121
+ * add option to provide file list directly to ascp like this (only for direct transfer agent):
3122
+
3123
+ ```
3124
+ ... --sources=@ts --ts=@json:'{"paths":[],"EX_file_list":"filelist"}'
3125
+ ```
3126
+
3127
+ * 0.10.18
3128
+
3129
+ * new option in. `server` : `ssh_options`
3130
+
3094
3131
  * 0.10.17
3095
3132
 
3096
3133
  * fixed problem on `server` for option `ssh_keys`, now accepts both single value and list.
@@ -3403,19 +3440,6 @@ OpenSSH keys only supported if ED25519 is available
3403
3440
  Which meant that you do not have ruby support for ED25519 SSH keys.
3404
3441
  You may either install the suggested Gems, or remove your ed25519 key from your `.ssh` folder to solve the issue.
3405
3442
 
3406
- ## could not connect to ssh-agent
3407
-
3408
- If you get error message:
3409
-
3410
- ```
3411
- ERROR -- net.ssh.authentication.agent: could not connect to ssh-agent: Agent not configured
3412
- ```
3413
-
3414
- It means that the provided ssh key does not exist or work.
3415
-
3416
- [check the manual](https://net-ssh.github.io/ssh/v1/chapter-2.html#s2)
3417
- , check env var: `SSH_AGENT_SOCK`, check if the key is protected with a passphrase.
3418
-
3419
3443
  # TODO
3420
3444
 
3421
3445
  * remove rest and oauth classes and use ruby standard gems:
@@ -62,6 +62,9 @@ UNDER CONSTRUCTION<br/>
62
62
  <tr><td>remove_empty_directories</td><td></td><td>boolean</td><td></td><td class="yes">Y</td><td></td><td>?</td><td>Specifies whether to remove empty directories.</td></tr>
63
63
  <tr><td>multi_session</td><td>1</td><td>integer</td><td class="no">N</td><td class="yes">Y</td><td class="no">N</td><td>-C</td><td>Specifies how many parts the transfer is in.</td></tr>
64
64
  <tr><td>multi_session_threshold</td><td>null</td><td>integer</td><td class="no">N</td><td class="yes">Y</td><td class="no">N</td><td>-</td><td>in bytes</td></tr>
65
+ <tr><td>exclude_newer_than</td><td></td><td>integer</td><td class="yes">Y</td><td></td><td></td><td>--exclude-newer-than</td><td>-</td></tr>
66
+ <tr><td>exclude_older_than</td><td></td><td>integer</td><td class="yes">Y</td><td></td><td></td><td>--exclude-older-than</td><td>-</td></tr>
67
+ <tr><td>preserve_acls</td><td></td><td>string</td><td class="yes">Y</td><td></td><td></td><td>--preserve-acls</td><td>-</td></tr>
65
68
  <tr><td>dgram_size</td><td></td><td>integer</td><td class="yes">Y</td><td></td><td></td><td>-Z</td><td>in bytes</td></tr>
66
69
  <tr><td>compression</td><td></td><td>integer</td><td></td><td></td><td></td><td></td><td>ascp4 only, 0 / 1?</td></tr>
67
70
  <tr><td>read_threads</td><td></td><td>integer</td><td></td><td></td><td></td><td>-</td><td>ascp4 only</td></tr>
@@ -84,6 +87,12 @@ UNDER CONSTRUCTION<br/>
84
87
  <tr><td>EX_http_proxy_url</td><td>-</td><td>string</td><td class="yes">Y</td><td class="no">N</td><td class="no">N</td><td>-x</td><td>Specify the proxy server address used by HTTP Fallback</td></tr>
85
88
  <tr><td>EX_ascp_args</td><td>-</td><td>array</td><td class="yes">Y</td><td class="no">N</td><td class="no">N</td><td>same</td><td>Add command line arguments to ascp</td></tr>
86
89
  <tr><td>EX_http_transfer_jpeg</td><td>0</td><td>integer</td><td class="yes">Y</td><td class="no">N</td><td class="no">N</td><td>-j</td><td>HTTP transfers as JPEG file</td></tr>
90
+ <tr><td>EX_license_text</td><td>-</td><td>string</td><td class="yes">Y</td><td class="no">N</td><td class="no">N</td><td>LICENSE</td><td>license file text</td></tr>
91
+ <tr><td>EX_file_list</td><td>-</td><td>string</td><td class="yes">Y</td><td class="no">N</td><td class="no">N</td><td>--file-list</td><td>source file list</td></tr>
92
+ <tr><td>EX_file_pair_list</td><td>-</td><td>string</td><td class="yes">Y</td><td class="no">N</td><td class="no">N</td><td>--file-pair-list</td><td>source file pair list</td></tr>
93
+ <tr><td>EX_multi_session_part</td><td>-</td><td>string</td><td class="yes">Y</td><td class="no">N</td><td class="no">N</td><td>-C</td><td>part for multisession</td></tr>
94
+ <tr><td>EX_no_read</td><td>-</td><td>-</td><td class="yes">Y</td><td class="no">N</td><td class="no">N</td><td>--no-read</td><td>no read source</td></tr>
95
+ <tr><td>EX_no_write</td><td>-</td><td>-</td><td class="yes">Y</td><td class="no">N</td><td class="no">N</td><td>--no-write</td><td>no write estination</td></tr>
87
96
  </table>
88
97
  </body>
89
98
  </html>
@@ -1 +1 @@
1
- 0.10.17
1
+ 0.11
@@ -181,7 +181,7 @@ module Asperalm
181
181
  #example: send_result={'items'=>[{'file'=>"filename1","permissions"=>[{'name'=>'read'},{'name'=>'write'}]}]}
182
182
  # if there is no items
183
183
  case send_result['self']['type']
184
- when 'directory'
184
+ when 'directory','container' # directory: node, container: shares
185
185
  result={ :data => send_result['items'] , :type => :object_list, :textify => lambda { |table_data| c_textify_browse(table_data) } }
186
186
  self.format.display_status("Items: #{send_result['item_count']}/#{send_result['total_count']}")
187
187
  else # 'file','symbolic_link'
@@ -17,9 +17,11 @@ module Asperalm
17
17
 
18
18
  def initialize(env)
19
19
  super(env)
20
- self.options.add_opt_simple(:ssh_keys,'ssh key path list')
20
+ self.options.add_opt_simple(:ssh_keys,'ssh key path list (Array or single)')
21
+ self.options.add_opt_simple(:ssh_options,'ssh options (Hash)')
21
22
  self.options.add_opt_simple(:cmd_prefix,'prefix to add for as cmd execution, e.g. sudo or /opt/aspera/bin ')
22
23
  self.options.set_option(:ssh_keys,[])
24
+ self.options.set_option(:ssh_options,{})
23
25
  self.options.parse_options!
24
26
  end
25
27
 
@@ -72,7 +74,8 @@ module Asperalm
72
74
  'remote_host'=>server_uri.hostname,
73
75
  'remote_user'=>self.options.get_option(:username,:mandatory),
74
76
  }
75
- ssh_options={}
77
+ ssh_options=self.options.get_option(:ssh_options,:optional)
78
+ raise 'expecting a Hash for ssh_options' unless ssh_options.is_a?(Hash)
76
79
  if !server_uri.port.nil?
77
80
  ssh_options[:port]=server_uri.port
78
81
  server_transfer_spec['ssh_port']=server_uri.port
@@ -60,12 +60,15 @@ module Asperalm
60
60
  'sshfp' => { :type => :opt_with_arg, :option_switch=>'--check-sshfp',:accepted_types=>String},
61
61
  'symlink_policy' => { :type => :opt_with_arg, :option_switch=>'--symbolic-links',:accepted_types=>String},
62
62
  'overwrite' => { :type => :opt_with_arg, :accepted_types=>String},
63
- 'exclude_newer_than' => { :type => :opt_with_arg, :accepted_types=>Integer}, #todo: doc
64
- 'exclude_older_than' => { :type => :opt_with_arg, :accepted_types=>Integer}, #todo: doc
65
- 'preserve_acls' => { :type => :opt_with_arg, :accepted_types=>String}, #todo: doc
66
- 'move_after_transfer' => { :type => :opt_with_arg, :accepted_types=>String}, #todo: doc
63
+ 'exclude_newer_than' => { :type => :opt_with_arg, :accepted_types=>Integer},
64
+ 'exclude_older_than' => { :type => :opt_with_arg, :accepted_types=>Integer},
65
+ 'preserve_acls' => { :type => :opt_with_arg, :accepted_types=>String},
66
+ 'move_after_transfer' => { :type => :opt_with_arg, :accepted_types=>String},
67
+ 'multi_session_threshold' => { :type => :opt_with_arg, :accepted_types=>String},
67
68
  # non standard parameters
68
69
  'EX_fasp_proxy_url' => { :type => :opt_with_arg, :option_switch=>'--proxy',:accepted_types=>String},
70
+ 'EX_file_list' => { :type => :opt_with_arg, :option_switch=>'--file-list',:accepted_types=>String},
71
+ 'EX_file_pair_list' => { :type => :opt_with_arg, :option_switch=>'--file-pair-list',:accepted_types=>String},
69
72
  'EX_http_proxy_url' => { :type => :opt_with_arg, :option_switch=>'-x',:accepted_types=>String},
70
73
  'EX_ssh_key_paths' => { :type => :opt_with_arg, :option_switch=>'-i',:accepted_types=>Array},
71
74
  'EX_http_transfer_jpeg' => { :type => :opt_with_arg, :option_switch=>'-j',:accepted_types=>Integer},
@@ -129,27 +132,37 @@ module Asperalm
129
132
  @builder.add_command_line_options(['--dest64'])
130
133
  end
131
134
 
132
- src_dst_list=@builder.process_param('paths',:get_value,:accepted_types=>Array,:mandatory=>!@job_spec.has_key?('keepalive'))
133
- unless src_dst_list.nil?
135
+ paths_array=@builder.process_param('paths',:get_value,:accepted_types=>Array,:mandatory=>!@job_spec.has_key?('keepalive'))
136
+ unless paths_array.nil?
134
137
  # use file list if there is storage defined for it.
135
138
  if @@file_list_folder.nil?
136
139
  # not safe for special characters ? (maybe not, depends on OS)
137
140
  Log.log.debug("placing source file list on command line (no file list file)")
138
- @builder.add_command_line_options(src_dst_list.map{|i|i['source']})
141
+ @builder.add_command_line_options(paths_array.map{|i|i['source']})
139
142
  else
140
- # safer option: file list
141
- # if there is destination in paths, then use filepairlist
142
- # TODO: well, we test only the first one, but anyway it shall be consistent
143
- if src_dst_list.first.has_key?('destination')
144
- option='--file-pair-list'
145
- lines=src_dst_list.inject([]){|m,e|m.push(e['source'],e['destination']);m}
146
- else
143
+ file_list_file=@builder.process_param('EX_file_list',:get_value,:accepted_types=>String,:mandatory=>false)
144
+ if !file_list_file.nil?
147
145
  option='--file-list'
148
- lines=src_dst_list.map{|i|i['source']}
146
+ else
147
+ file_list_file=@builder.process_param('EX_file_pair_list',:get_value,:accepted_types=>String,:mandatory=>false)
148
+ if !file_list_file.nil?
149
+ option='--file-pair-list'
150
+ else
151
+ # safer option: file list
152
+ # if there is destination in paths, then use filepairlist
153
+ # TODO: well, we test only the first one, but anyway it shall be consistent
154
+ if paths_array.first.has_key?('destination')
155
+ option='--file-pair-list'
156
+ lines=paths_array.inject([]){|m,e|m.push(e['source'],e['destination']);m}
157
+ else
158
+ option='--file-list'
159
+ lines=paths_array.map{|i|i['source']}
160
+ end
161
+ file_list_file=Asperalm::TempFileManager.instance.new_file_path_in_folder(@@file_list_folder)
162
+ File.open(file_list_file, 'w+'){|f|f.puts(lines)}
163
+ Log.log.debug("#{option}=\n#{File.read(file_list_file)}".red)
164
+ end
149
165
  end
150
- file_list_file=Asperalm::TempFileManager.instance.new_file_path_in_folder(@@file_list_folder)
151
- File.open(file_list_file, 'w+'){|f|f.puts(lines)}
152
- Log.log.debug("#{option}=\n#{File.read(file_list_file)}".red)
153
166
  @builder.add_command_line_options(["#{option}=#{file_list_file}"])
154
167
  end
155
168
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: asperalm
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.10.17
4
+ version: '0.11'
5
5
  platform: ruby
6
6
  authors:
7
7
  - Laurent Martin
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-09-01 00:00:00.000000000 Z
11
+ date: 2020-09-23 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: xml-simple