aspera-cli 4.0.0.pre1 → 4.0.0.pre2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +213 -255
- data/docs/Makefile +19 -13
- data/docs/README.erb.md +109 -76
- data/docs/test_env.conf +37 -14
- data/examples/aoc.rb +2 -2
- data/examples/transfer.rb +23 -14
- data/lib/aspera/{on_cloud.rb → aoc.rb} +24 -15
- data/lib/aspera/cli/main.rb +1 -0
- data/lib/aspera/cli/plugins/alee.rb +2 -2
- data/lib/aspera/cli/plugins/{oncloud.rb → aoc.rb} +34 -34
- data/lib/aspera/cli/plugins/ats.rb +1 -1
- data/lib/aspera/cli/plugins/config.rb +31 -17
- data/lib/aspera/cli/plugins/cos.rb +9 -49
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/cos_node.rb +50 -0
- data/lib/aspera/data/1 +0 -0
- data/lib/aspera/data/2 +0 -0
- data/lib/aspera/data/3 +1 -0
- data/lib/aspera/data/4 +2 -0
- data/lib/aspera/data/5 +1 -0
- data/lib/aspera/data/6 +2 -0
- data/lib/aspera/data/7 +0 -0
- data/lib/aspera/data_repository.rb +13 -0
- data/lib/aspera/environment.rb +55 -0
- data/lib/aspera/fasp/aoc.rb +6 -6
- data/lib/aspera/fasp/installation.rb +112 -56
- data/lib/aspera/fasp/local.rb +2 -3
- data/lib/aspera/faspex_gw.rb +2 -2
- data/lib/aspera/open_application.rb +11 -24
- data/lib/aspera/preview/generator.rb +1 -5
- data/lib/aspera/rest.rb +56 -48
- data/lib/aspera/rest_error_analyzer.rb +5 -4
- data/lib/aspera/temp_file_manager.rb +1 -1
- metadata +52 -26
- data/docs/secrets.make +0 -38
- data/lib/aspera/cli/plugins/xnode.rb +0 -115
data/docs/secrets.make
DELETED
@@ -1,38 +0,0 @@
|
|
1
|
-
# Private data for test systems
|
2
|
-
# Do not share, github contains only empty values
|
3
|
-
# fill this file and place in folder "local"
|
4
|
-
|
5
|
-
|
6
|
-
# Incoming asset processing
|
7
|
-
|
8
|
-
# received package public link : https://sedemo.ibmaspera.com/packages/public/receive?token=_value_here_
|
9
|
-
CF_AOC_PUBLINK_RECV_PACKAGE=_value_here_
|
10
|
-
# shared inbox public link: https://aspera.pub/xxxxxxx
|
11
|
-
CF_AOC_PUBLINK_SEND_DROPBOX=_value_here_
|
12
|
-
# public link to send package to user: https://aspera.pub/xxxxxx
|
13
|
-
CF_AOC_PUBLINK_SEND_USER=_value_here_
|
14
|
-
# public link to send files to user's shared folder: https://aspera.pub/xxxxxx
|
15
|
-
CF_AOC_PUBLINK_FOLDER=_value_here_
|
16
|
-
# workspace with shared inbox
|
17
|
-
CF_AOC_SH_BX_WS=_value_here_
|
18
|
-
# name of shared inbox in this workspace
|
19
|
-
CF_AOC_SH_BX_NAME=_value_here_
|
20
|
-
# name of node to test "node resource"
|
21
|
-
CF_AOC_NODE1_NAME=_value_here_
|
22
|
-
CF_AOC_NODE1_SECRET=_value_here_
|
23
|
-
|
24
|
-
|
25
|
-
CF_AWS_ACCESS_KEY=_value_here_
|
26
|
-
CF_AWS_SECRET_KEY=_value_here_
|
27
|
-
CF_AWS_BUCKET=_value_here_
|
28
|
-
CF_AWS_REGION=_value_here_
|
29
|
-
|
30
|
-
CF_ICOS_AK_ID=_value_here_
|
31
|
-
CF_ICOS_SECRET_AK=_value_here_
|
32
|
-
CF_ICOS_BUCKET=_value_here_
|
33
|
-
CF_ICOS_REGION=_value_here_
|
34
|
-
|
35
|
-
CF_EMAIL_ADDR=_value_here_
|
36
|
-
CF_AOC_EXTERNAL_EMAIL=_value_here_
|
37
|
-
CF_AOC_ORG=_value_here_
|
38
|
-
CF_AOC_USER=_value_here_
|
@@ -1,115 +0,0 @@
|
|
1
|
-
require 'aspera/cli/basic_auth_plugin'
|
2
|
-
require 'aspera/persistency_action_once'
|
3
|
-
require "base64"
|
4
|
-
|
5
|
-
module Aspera
|
6
|
-
module Cli
|
7
|
-
module Plugins
|
8
|
-
# experiments
|
9
|
-
class Xnode < BasicAuthPlugin
|
10
|
-
def initialize(env)
|
11
|
-
super(env)
|
12
|
-
self.options.add_opt_simple(:filter_transfer,"Ruby expression for filter at transfer level (cleanup)")
|
13
|
-
self.options.add_opt_simple(:filter_file,"Ruby expression for filter at file level (cleanup)")
|
14
|
-
self.options.parse_options!
|
15
|
-
end
|
16
|
-
# "transfer_filter"=>"t['status'].eql?('completed') and t['start_spec']['remote_user'].eql?('faspex')", :file_filter=>"f['status'].eql?('completed') and 0 != f['size'] and t['start_spec']['direction'].eql?('send')"
|
17
|
-
|
18
|
-
ACTIONS=[ :postprocess, :cleanup, :forward ]
|
19
|
-
|
20
|
-
# retrieve tranfer list using API and persistency file
|
21
|
-
def self.get_transfers_iteration(api_node,params)
|
22
|
-
# array with one element max
|
23
|
-
iteration_data=[]
|
24
|
-
iteration_persistency=nil
|
25
|
-
if self.options.get_option(:once_only,:mandatory)
|
26
|
-
iteration_persistency=PersistencyActionOnce.new(
|
27
|
-
manager: @agents[:persistency],
|
28
|
-
data: iteration_data,
|
29
|
-
ids: ['xnode',self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory)])
|
30
|
-
end
|
31
|
-
iteration_data[0]=process_file_events(iteration_data[0])
|
32
|
-
# first time run ? or subsequent run ?
|
33
|
-
params[:iteration_token]=iteration_data[0] unless iteration_data[0].nil?
|
34
|
-
resp=api_node.read('ops/transfers',params)
|
35
|
-
transfers=resp[:data]
|
36
|
-
if transfers.is_a?(Array) then
|
37
|
-
# 3.7.2, released API
|
38
|
-
iteration_data[0]=URI.decode_www_form(URI.parse(resp[:http]['Link'].match(/<([^>]+)>/)[1]).query).to_h['iteration_token']
|
39
|
-
else
|
40
|
-
# 3.5.2, deprecated API
|
41
|
-
iteration_data[0]=transfers['iteration_token']
|
42
|
-
transfers=transfers['transfers']
|
43
|
-
end
|
44
|
-
iteration_persistency.save unless iteration_persistency.nil?
|
45
|
-
return transfers
|
46
|
-
end
|
47
|
-
|
48
|
-
def execute_action
|
49
|
-
api_node=Rest.new({
|
50
|
-
:base_url => self.options.get_option(:url,:mandatory),
|
51
|
-
:auth => {
|
52
|
-
:type => :basic,
|
53
|
-
:username => self.options.get_option(:username,:mandatory),
|
54
|
-
:password => self.options.get_option(:password,:mandatory)
|
55
|
-
}})
|
56
|
-
command=self.options.get_next_command(ACTIONS)
|
57
|
-
case command
|
58
|
-
when :cleanup
|
59
|
-
transfers=self.class.get_transfers_iteration(api_node,{:active_only=>false})
|
60
|
-
filter_transfer=self.options.get_option(:filter_transfer,:mandatory)
|
61
|
-
filter_file=self.options.get_option(:filter_file,:mandatory)
|
62
|
-
Log.log.debug("filter_transfer: #{filter_transfer}")
|
63
|
-
Log.log.debug("filter_file: #{filter_file}")
|
64
|
-
# build list of files to delete: non zero files, downloads, for specified user
|
65
|
-
paths_to_delete=[]
|
66
|
-
transfers.each do |t|
|
67
|
-
if eval(filter_transfer)
|
68
|
-
t['files'].each do |f|
|
69
|
-
if eval(filter_file)
|
70
|
-
if !paths_to_delete.include?(f['path'])
|
71
|
-
paths_to_delete.push(f['path'])
|
72
|
-
Log.log.info("to delete: #{f['path']}")
|
73
|
-
end
|
74
|
-
end
|
75
|
-
end
|
76
|
-
end
|
77
|
-
end
|
78
|
-
# delete files, if any
|
79
|
-
if paths_to_delete.length != 0
|
80
|
-
Log.log.info("deletion")
|
81
|
-
return self.delete_files(api_node,paths_to_delete,nil)
|
82
|
-
else
|
83
|
-
Log.log.info("nothing to delete")
|
84
|
-
end
|
85
|
-
return Main.result_nothing
|
86
|
-
when :forward
|
87
|
-
# detect transfer sessions since last call
|
88
|
-
transfers=self.class.get_transfers_iteration(api_node,{:active_only=>false})
|
89
|
-
# build list of all files received in all sessions
|
90
|
-
filelist=[]
|
91
|
-
transfers.select { |t| t['status'].eql?('completed') and t['start_spec']['direction'].eql?('receive') }.each do |t|
|
92
|
-
t['files'].each { |f| filelist.push(f['path']) }
|
93
|
-
end
|
94
|
-
if filelist.empty?
|
95
|
-
Log.log.debug("NO TRANSFER".red)
|
96
|
-
return Main.result_nothing
|
97
|
-
end
|
98
|
-
Log.log.debug("file list=#{filelist}")
|
99
|
-
# get download transfer spec on destination node
|
100
|
-
transfer_params={ :transfer_requests => [ { :transfer_request => { :paths => filelist.map {|i| {:source=>i} } } } ] }
|
101
|
-
send_result=api_node.call({:operation=>'POST',:subpath=>'files/download_setup',:json_params=>transfer_params})
|
102
|
-
# only one request, so only one answer
|
103
|
-
transfer_spec=send_result[:data]['transfer_specs'].first['transfer_spec']
|
104
|
-
# execute transfer
|
105
|
-
return Main.result_transfer(self.transfer.start(transfer_spec,{:src=>:node_gen3}))
|
106
|
-
when :postprocess
|
107
|
-
transfers=self.class.get_transfers_iteration(api_node,{:view=>'summary',:direction=>'receive',:active_only=>false})
|
108
|
-
return { :type=>:object_list,:data => transfers }
|
109
|
-
end # case command
|
110
|
-
raise "ERROR: shall not reach this line"
|
111
|
-
end # execute_action
|
112
|
-
end # Main
|
113
|
-
end # Plugin
|
114
|
-
end # Cli
|
115
|
-
end # Aspera
|