aspera-cli 4.0.0.pre1 → 4.0.0.pre2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +213 -255
- data/docs/Makefile +19 -13
- data/docs/README.erb.md +109 -76
- data/docs/test_env.conf +37 -14
- data/examples/aoc.rb +2 -2
- data/examples/transfer.rb +23 -14
- data/lib/aspera/{on_cloud.rb → aoc.rb} +24 -15
- data/lib/aspera/cli/main.rb +1 -0
- data/lib/aspera/cli/plugins/alee.rb +2 -2
- data/lib/aspera/cli/plugins/{oncloud.rb → aoc.rb} +34 -34
- data/lib/aspera/cli/plugins/ats.rb +1 -1
- data/lib/aspera/cli/plugins/config.rb +31 -17
- data/lib/aspera/cli/plugins/cos.rb +9 -49
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/cos_node.rb +50 -0
- data/lib/aspera/data/1 +0 -0
- data/lib/aspera/data/2 +0 -0
- data/lib/aspera/data/3 +1 -0
- data/lib/aspera/data/4 +2 -0
- data/lib/aspera/data/5 +1 -0
- data/lib/aspera/data/6 +2 -0
- data/lib/aspera/data/7 +0 -0
- data/lib/aspera/data_repository.rb +13 -0
- data/lib/aspera/environment.rb +55 -0
- data/lib/aspera/fasp/aoc.rb +6 -6
- data/lib/aspera/fasp/installation.rb +112 -56
- data/lib/aspera/fasp/local.rb +2 -3
- data/lib/aspera/faspex_gw.rb +2 -2
- data/lib/aspera/open_application.rb +11 -24
- data/lib/aspera/preview/generator.rb +1 -5
- data/lib/aspera/rest.rb +56 -48
- data/lib/aspera/rest_error_analyzer.rb +5 -4
- data/lib/aspera/temp_file_manager.rb +1 -1
- metadata +52 -26
- data/docs/secrets.make +0 -38
- data/lib/aspera/cli/plugins/xnode.rb +0 -115
data/docs/secrets.make
DELETED
@@ -1,38 +0,0 @@
|
|
1
|
-
# Private data for test systems
|
2
|
-
# Do not share, github contains only empty values
|
3
|
-
# fill this file and place in folder "local"
|
4
|
-
|
5
|
-
|
6
|
-
# Incoming asset processing
|
7
|
-
|
8
|
-
# received package public link : https://sedemo.ibmaspera.com/packages/public/receive?token=_value_here_
|
9
|
-
CF_AOC_PUBLINK_RECV_PACKAGE=_value_here_
|
10
|
-
# shared inbox public link: https://aspera.pub/xxxxxxx
|
11
|
-
CF_AOC_PUBLINK_SEND_DROPBOX=_value_here_
|
12
|
-
# public link to send package to user: https://aspera.pub/xxxxxx
|
13
|
-
CF_AOC_PUBLINK_SEND_USER=_value_here_
|
14
|
-
# public link to send files to user's shared folder: https://aspera.pub/xxxxxx
|
15
|
-
CF_AOC_PUBLINK_FOLDER=_value_here_
|
16
|
-
# workspace with shared inbox
|
17
|
-
CF_AOC_SH_BX_WS=_value_here_
|
18
|
-
# name of shared inbox in this workspace
|
19
|
-
CF_AOC_SH_BX_NAME=_value_here_
|
20
|
-
# name of node to test "node resource"
|
21
|
-
CF_AOC_NODE1_NAME=_value_here_
|
22
|
-
CF_AOC_NODE1_SECRET=_value_here_
|
23
|
-
|
24
|
-
|
25
|
-
CF_AWS_ACCESS_KEY=_value_here_
|
26
|
-
CF_AWS_SECRET_KEY=_value_here_
|
27
|
-
CF_AWS_BUCKET=_value_here_
|
28
|
-
CF_AWS_REGION=_value_here_
|
29
|
-
|
30
|
-
CF_ICOS_AK_ID=_value_here_
|
31
|
-
CF_ICOS_SECRET_AK=_value_here_
|
32
|
-
CF_ICOS_BUCKET=_value_here_
|
33
|
-
CF_ICOS_REGION=_value_here_
|
34
|
-
|
35
|
-
CF_EMAIL_ADDR=_value_here_
|
36
|
-
CF_AOC_EXTERNAL_EMAIL=_value_here_
|
37
|
-
CF_AOC_ORG=_value_here_
|
38
|
-
CF_AOC_USER=_value_here_
|
@@ -1,115 +0,0 @@
|
|
1
|
-
require 'aspera/cli/basic_auth_plugin'
|
2
|
-
require 'aspera/persistency_action_once'
|
3
|
-
require "base64"
|
4
|
-
|
5
|
-
module Aspera
|
6
|
-
module Cli
|
7
|
-
module Plugins
|
8
|
-
# experiments
|
9
|
-
class Xnode < BasicAuthPlugin
|
10
|
-
def initialize(env)
|
11
|
-
super(env)
|
12
|
-
self.options.add_opt_simple(:filter_transfer,"Ruby expression for filter at transfer level (cleanup)")
|
13
|
-
self.options.add_opt_simple(:filter_file,"Ruby expression for filter at file level (cleanup)")
|
14
|
-
self.options.parse_options!
|
15
|
-
end
|
16
|
-
# "transfer_filter"=>"t['status'].eql?('completed') and t['start_spec']['remote_user'].eql?('faspex')", :file_filter=>"f['status'].eql?('completed') and 0 != f['size'] and t['start_spec']['direction'].eql?('send')"
|
17
|
-
|
18
|
-
ACTIONS=[ :postprocess, :cleanup, :forward ]
|
19
|
-
|
20
|
-
# retrieve tranfer list using API and persistency file
|
21
|
-
def self.get_transfers_iteration(api_node,params)
|
22
|
-
# array with one element max
|
23
|
-
iteration_data=[]
|
24
|
-
iteration_persistency=nil
|
25
|
-
if self.options.get_option(:once_only,:mandatory)
|
26
|
-
iteration_persistency=PersistencyActionOnce.new(
|
27
|
-
manager: @agents[:persistency],
|
28
|
-
data: iteration_data,
|
29
|
-
ids: ['xnode',self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory)])
|
30
|
-
end
|
31
|
-
iteration_data[0]=process_file_events(iteration_data[0])
|
32
|
-
# first time run ? or subsequent run ?
|
33
|
-
params[:iteration_token]=iteration_data[0] unless iteration_data[0].nil?
|
34
|
-
resp=api_node.read('ops/transfers',params)
|
35
|
-
transfers=resp[:data]
|
36
|
-
if transfers.is_a?(Array) then
|
37
|
-
# 3.7.2, released API
|
38
|
-
iteration_data[0]=URI.decode_www_form(URI.parse(resp[:http]['Link'].match(/<([^>]+)>/)[1]).query).to_h['iteration_token']
|
39
|
-
else
|
40
|
-
# 3.5.2, deprecated API
|
41
|
-
iteration_data[0]=transfers['iteration_token']
|
42
|
-
transfers=transfers['transfers']
|
43
|
-
end
|
44
|
-
iteration_persistency.save unless iteration_persistency.nil?
|
45
|
-
return transfers
|
46
|
-
end
|
47
|
-
|
48
|
-
def execute_action
|
49
|
-
api_node=Rest.new({
|
50
|
-
:base_url => self.options.get_option(:url,:mandatory),
|
51
|
-
:auth => {
|
52
|
-
:type => :basic,
|
53
|
-
:username => self.options.get_option(:username,:mandatory),
|
54
|
-
:password => self.options.get_option(:password,:mandatory)
|
55
|
-
}})
|
56
|
-
command=self.options.get_next_command(ACTIONS)
|
57
|
-
case command
|
58
|
-
when :cleanup
|
59
|
-
transfers=self.class.get_transfers_iteration(api_node,{:active_only=>false})
|
60
|
-
filter_transfer=self.options.get_option(:filter_transfer,:mandatory)
|
61
|
-
filter_file=self.options.get_option(:filter_file,:mandatory)
|
62
|
-
Log.log.debug("filter_transfer: #{filter_transfer}")
|
63
|
-
Log.log.debug("filter_file: #{filter_file}")
|
64
|
-
# build list of files to delete: non zero files, downloads, for specified user
|
65
|
-
paths_to_delete=[]
|
66
|
-
transfers.each do |t|
|
67
|
-
if eval(filter_transfer)
|
68
|
-
t['files'].each do |f|
|
69
|
-
if eval(filter_file)
|
70
|
-
if !paths_to_delete.include?(f['path'])
|
71
|
-
paths_to_delete.push(f['path'])
|
72
|
-
Log.log.info("to delete: #{f['path']}")
|
73
|
-
end
|
74
|
-
end
|
75
|
-
end
|
76
|
-
end
|
77
|
-
end
|
78
|
-
# delete files, if any
|
79
|
-
if paths_to_delete.length != 0
|
80
|
-
Log.log.info("deletion")
|
81
|
-
return self.delete_files(api_node,paths_to_delete,nil)
|
82
|
-
else
|
83
|
-
Log.log.info("nothing to delete")
|
84
|
-
end
|
85
|
-
return Main.result_nothing
|
86
|
-
when :forward
|
87
|
-
# detect transfer sessions since last call
|
88
|
-
transfers=self.class.get_transfers_iteration(api_node,{:active_only=>false})
|
89
|
-
# build list of all files received in all sessions
|
90
|
-
filelist=[]
|
91
|
-
transfers.select { |t| t['status'].eql?('completed') and t['start_spec']['direction'].eql?('receive') }.each do |t|
|
92
|
-
t['files'].each { |f| filelist.push(f['path']) }
|
93
|
-
end
|
94
|
-
if filelist.empty?
|
95
|
-
Log.log.debug("NO TRANSFER".red)
|
96
|
-
return Main.result_nothing
|
97
|
-
end
|
98
|
-
Log.log.debug("file list=#{filelist}")
|
99
|
-
# get download transfer spec on destination node
|
100
|
-
transfer_params={ :transfer_requests => [ { :transfer_request => { :paths => filelist.map {|i| {:source=>i} } } } ] }
|
101
|
-
send_result=api_node.call({:operation=>'POST',:subpath=>'files/download_setup',:json_params=>transfer_params})
|
102
|
-
# only one request, so only one answer
|
103
|
-
transfer_spec=send_result[:data]['transfer_specs'].first['transfer_spec']
|
104
|
-
# execute transfer
|
105
|
-
return Main.result_transfer(self.transfer.start(transfer_spec,{:src=>:node_gen3}))
|
106
|
-
when :postprocess
|
107
|
-
transfers=self.class.get_transfers_iteration(api_node,{:view=>'summary',:direction=>'receive',:active_only=>false})
|
108
|
-
return { :type=>:object_list,:data => transfers }
|
109
|
-
end # case command
|
110
|
-
raise "ERROR: shall not reach this line"
|
111
|
-
end # execute_action
|
112
|
-
end # Main
|
113
|
-
end # Plugin
|
114
|
-
end # Cli
|
115
|
-
end # Aspera
|