rest-ftp-daemon 0.420.2 → 0.421.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +4 -4
- data/lib/rest-ftp-daemon/helpers/views.rb +2 -1
- data/lib/rest-ftp-daemon/job.rb +6 -11
- data/lib/rest-ftp-daemon/jobs/errors.rb +5 -6
- data/lib/rest-ftp-daemon/jobs/transfer.rb +20 -31
- data/lib/rest-ftp-daemon/jobs/video.rb +2 -2
- data/lib/rest-ftp-daemon/location.rb +39 -24
- data/lib/rest-ftp-daemon/remote.rb +2 -3
- data/lib/rest-ftp-daemon/remote_ftp.rb +31 -17
- data/lib/rest-ftp-daemon/remote_s3.rb +17 -8
- data/lib/rest-ftp-daemon/remote_sftp.rb +21 -20
- data/lib/rest-ftp-daemon/views/dashboard_jobs.haml +2 -0
- data/lib/rest-ftp-daemon/views/dashboard_table.haml +4 -6
- data/lib/rest-ftp-daemon/views/dashboard_workers.haml +1 -1
- data/lib/rest-ftp-daemon/workers/reporter.rb +0 -1
- data/rest-ftp-daemon.gemspec +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c257143aa55e12efe93dcd1f9e2829e662a27d33
|
4
|
+
data.tar.gz: 75b635758132ab9af8e381a9edcca9cfa2b057de
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 966ca0d76da55605f512de091d853a5a250b2b85cfff0419dc1fbacb54a0065a83b68a11bdb507c8dfaf25834050346d8ec21816ab2d69b806dd2b57d2813989
|
7
|
+
data.tar.gz: 53d3336b265bb1282c4412d49be9e3a604e8463df9d3fb6d549be7edfe4b9dc010851dac14194bde48d5c45c18126e7e53a55476b9a99618ef1331db42f59460
|
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
rest-ftp-daemon (0.
|
4
|
+
rest-ftp-daemon (0.421.0)
|
5
5
|
activesupport (~> 4.2)
|
6
6
|
api-auth
|
7
7
|
aws-sdk-resources (~> 2)
|
@@ -39,10 +39,10 @@ GEM
|
|
39
39
|
ast (2.3.0)
|
40
40
|
astrolabe (1.3.1)
|
41
41
|
parser (~> 2.2)
|
42
|
-
aws-sdk-core (2.5.
|
42
|
+
aws-sdk-core (2.5.8)
|
43
43
|
jmespath (~> 1.0)
|
44
|
-
aws-sdk-resources (2.5.
|
45
|
-
aws-sdk-core (= 2.5.
|
44
|
+
aws-sdk-resources (2.5.8)
|
45
|
+
aws-sdk-core (= 2.5.8)
|
46
46
|
axiom-types (0.1.1)
|
47
47
|
descendants_tracker (~> 0.0.4)
|
48
48
|
ice_nine (~> 0.11.0)
|
@@ -60,8 +60,9 @@ module RestFtpDaemon
|
|
60
60
|
|
61
61
|
def location_label uri
|
62
62
|
sprintf(
|
63
|
-
'<div class="transfer-type label label-%s">%s</div>',
|
63
|
+
'<div class="transfer-type label label-%s" title="%s">%s</div>',
|
64
64
|
location_style(uri),
|
65
|
+
uri.to_s,
|
65
66
|
uri.class.name.split('::').last
|
66
67
|
)
|
67
68
|
end
|
data/lib/rest-ftp-daemon/job.rb
CHANGED
@@ -69,7 +69,6 @@ module RestFtpDaemon
|
|
69
69
|
# Logger
|
70
70
|
@logger = BmcDaemonLib::LoggerPool.instance.get :transfer
|
71
71
|
|
72
|
-
|
73
72
|
# Import query params
|
74
73
|
FIELDS.each do |name|
|
75
74
|
instance_variable_set "@#{name}", params[name]
|
@@ -140,7 +139,6 @@ module RestFtpDaemon
|
|
140
139
|
do_after
|
141
140
|
|
142
141
|
rescue StandardError => exception
|
143
|
-
log_debug "Job.process caught [#{exception.class}] #{exception.message}"
|
144
142
|
return oops current_signal, exception
|
145
143
|
|
146
144
|
else
|
@@ -251,15 +249,12 @@ module RestFtpDaemon
|
|
251
249
|
|
252
250
|
def set_info_location prefix, location
|
253
251
|
return unless location.is_a? Location
|
254
|
-
|
255
|
-
|
256
|
-
|
257
|
-
|
258
|
-
|
259
|
-
|
260
|
-
set_info prefix, :aws_region, location.aws_region
|
261
|
-
set_info prefix, :aws_bucket, location.aws_bucket
|
262
|
-
set_info prefix, :aws_id, location.aws_id
|
252
|
+
fields = [:uri, :scheme, :user, :host, :port, :dir, :name, :path, :aws_region, :aws_bucket, :aws_id]
|
253
|
+
|
254
|
+
# Add each field to @infos
|
255
|
+
fields.each do |what|
|
256
|
+
set_info prefix, what, location.send(what)
|
257
|
+
end
|
263
258
|
end
|
264
259
|
|
265
260
|
private
|
@@ -1,8 +1,7 @@
|
|
1
1
|
require "securerandom"
|
2
2
|
require "double_bag_ftps"
|
3
|
-
require "net/ssh"
|
4
|
-
require "net/ftp"
|
5
3
|
require "net/sftp"
|
4
|
+
require "net/ftp"
|
6
5
|
require 'streamio-ffmpeg'
|
7
6
|
|
8
7
|
module RestFtpDaemon
|
@@ -42,10 +41,10 @@ module RestFtpDaemon
|
|
42
41
|
|
43
42
|
ffmpeg_error: FFMPEG::Error,
|
44
43
|
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
sftp_openssl_error:
|
44
|
+
sftp_exception: Net::SFTP::StatusException,
|
45
|
+
sftp_key_mismatch: Net::SSH::HostKeyMismatch,
|
46
|
+
sftp_auth_failed: Net::SSH::AuthenticationFailed,
|
47
|
+
sftp_openssl_error: OpenSSL::SSL::SSLError,
|
49
48
|
# rescue Encoding::UndefinedConversionError => exception
|
50
49
|
# return oops :ended, exception, "encoding_error", true
|
51
50
|
}
|
@@ -16,13 +16,13 @@ module RestFtpDaemon
|
|
16
16
|
set_info :source, :processed, 0
|
17
17
|
|
18
18
|
# Ensure source is FILE
|
19
|
-
raise RestFtpDaemon::SourceNotSupported, @source_loc.scheme unless
|
19
|
+
raise RestFtpDaemon::SourceNotSupported, @source_loc.scheme unless @source_loc.is? URI::FILE
|
20
20
|
|
21
21
|
# Prepare remote object
|
22
22
|
case target_uri
|
23
23
|
when URI::FTP
|
24
24
|
log_info "JobTransfer.before target_method FTP"
|
25
|
-
@remote = RemoteFTP.new @target_loc, log_prefix, @config[:
|
25
|
+
@remote = RemoteFTP.new @target_loc, log_prefix, @config[:debug_ftp]
|
26
26
|
when URI::FTPES, URI::FTPS
|
27
27
|
log_info "JobTransfer.before target_method FTPES/FTPS"
|
28
28
|
@remote = RemoteFTP.new @target_loc, log_prefix, @config[:debug_ftps], :ftpes
|
@@ -48,10 +48,9 @@ module RestFtpDaemon
|
|
48
48
|
# Scan local source files from disk
|
49
49
|
set_status JOB_STATUS_CHECKING_SRC
|
50
50
|
sources = @source_loc.scan_files
|
51
|
-
log_debug "JobTransfer.work sources result #{sources.inspect}"
|
52
51
|
set_info :source, :count, sources.size
|
53
52
|
set_info :source, :files, sources.collect(&:name)
|
54
|
-
log_info "JobTransfer.work sources
|
53
|
+
log_info "JobTransfer.work sources #{sources.collect(&:name)}"
|
55
54
|
raise RestFtpDaemon::SourceNotFound if sources.empty?
|
56
55
|
|
57
56
|
# Guess target file name, and fail if present while we matched multiple sources
|
@@ -77,19 +76,15 @@ module RestFtpDaemon
|
|
77
76
|
source_processed = 0
|
78
77
|
targets = []
|
79
78
|
sources.each do |source|
|
80
|
-
#
|
81
|
-
|
82
|
-
|
83
|
-
# Add the source file name if none found in the target path
|
84
|
-
unless target_final.name
|
85
|
-
target_final.name = source.name
|
86
|
-
end
|
79
|
+
# Build final target, add the source file name if noneh
|
80
|
+
target = @target_loc.clone
|
81
|
+
target.name = source.name unless target.name
|
87
82
|
|
88
83
|
# Do the transfer, for each file
|
89
|
-
|
84
|
+
remote_upload source, target
|
90
85
|
|
91
86
|
# Add it to transferred target names
|
92
|
-
targets <<
|
87
|
+
targets << target.name
|
93
88
|
set_info :target, :files, targets
|
94
89
|
|
95
90
|
# Update counters
|
@@ -99,7 +94,6 @@ module RestFtpDaemon
|
|
99
94
|
|
100
95
|
def do_after
|
101
96
|
# Close FTP connexion and free up memory
|
102
|
-
log_info "JobTransfer.after"
|
103
97
|
@remote.close
|
104
98
|
|
105
99
|
# Free-up remote object
|
@@ -114,28 +108,21 @@ module RestFtpDaemon
|
|
114
108
|
RestFtpDaemon::Counters.instance.add :data, :transferred, @transfer_total
|
115
109
|
end
|
116
110
|
|
117
|
-
def
|
111
|
+
def remote_upload source, target
|
118
112
|
# Method assertions
|
119
|
-
raise RestFtpDaemon::AssertionFailed, "
|
120
|
-
raise RestFtpDaemon::AssertionFailed, "
|
121
|
-
raise RestFtpDaemon::AssertionFailed, "
|
113
|
+
raise RestFtpDaemon::AssertionFailed, "remote_upload/remote" if @remote.nil?
|
114
|
+
raise RestFtpDaemon::AssertionFailed, "remote_upload/source" if source.nil?
|
115
|
+
raise RestFtpDaemon::AssertionFailed, "remote_upload/target" if target.nil?
|
122
116
|
|
123
117
|
# Use source filename if target path provided none (typically with multiple sources)
|
124
|
-
log_info "JobTransfer.
|
118
|
+
log_info "JobTransfer.remote_upload [#{source.name}]: [#{source.path}] > [#{target.path}]"
|
125
119
|
set_info :source, :current, source.name
|
126
120
|
|
127
|
-
#
|
128
|
-
tempname = nil
|
129
|
-
if @tempfile
|
130
|
-
tempname = "#{target.name}.temp-#{identifier(JOB_TEMPFILE_LEN)}"
|
131
|
-
log_debug "JobTransfer.remote_push tempname [#{tempname}]"
|
132
|
-
end
|
133
|
-
|
134
|
-
# Remove any existing version if expected, or test its presence
|
121
|
+
# Remove any existing version if present, or check if it's there
|
135
122
|
if @overwrite
|
136
123
|
@remote.remove! target
|
137
124
|
elsif size = @remote.present?(target)
|
138
|
-
log_debug "JobTransfer.
|
125
|
+
log_debug "JobTransfer.remote_upload existing (#{format_bytes size, 'B'})"
|
139
126
|
raise RestFtpDaemon::TargetFileExists
|
140
127
|
end
|
141
128
|
|
@@ -146,9 +133,11 @@ module RestFtpDaemon
|
|
146
133
|
|
147
134
|
# Start the transfer, update job status after each block transfer
|
148
135
|
set_status JOB_STATUS_UPLOADING
|
149
|
-
|
136
|
+
log_debug "JobTransfer.remote_upload source[#{source.path}] temp[#{@tempfile}]"
|
137
|
+
@remote.upload source, target, @tempfile do |transferred, name|
|
138
|
+
|
150
139
|
# Update transfer statistics
|
151
|
-
|
140
|
+
update_progress transferred, name
|
152
141
|
|
153
142
|
# Touch my worker status
|
154
143
|
touch_job
|
@@ -162,7 +151,7 @@ module RestFtpDaemon
|
|
162
151
|
set_info :source, :current, nil
|
163
152
|
end
|
164
153
|
|
165
|
-
def
|
154
|
+
def update_progress transferred, name = ""
|
166
155
|
# What's current time ?
|
167
156
|
now = Time.now
|
168
157
|
notify_after = @config[:notify_after]
|
@@ -22,8 +22,8 @@ module RestFtpDaemon
|
|
22
22
|
end
|
23
23
|
|
24
24
|
# Ensure source and target are FILE
|
25
|
-
raise RestFtpDaemon::SourceNotSupported, @source_loc.scheme unless
|
26
|
-
raise RestFtpDaemon::TargetNotSupported, @
|
25
|
+
raise RestFtpDaemon::SourceNotSupported, @source_loc.scheme unless @source_loc.is? URI::FILE
|
26
|
+
raise RestFtpDaemon::TargetNotSupported, @target_loc.scheme unless @target_loc.is? URI::FILE
|
27
27
|
end
|
28
28
|
|
29
29
|
def do_work
|
@@ -3,6 +3,8 @@ require 'active_support/core_ext/module/delegation'
|
|
3
3
|
|
4
4
|
module RestFtpDaemon
|
5
5
|
class Location
|
6
|
+
include CommonHelpers
|
7
|
+
|
6
8
|
# Accessors
|
7
9
|
attr_accessor :name
|
8
10
|
|
@@ -10,7 +12,6 @@ module RestFtpDaemon
|
|
10
12
|
attr_reader :scheme
|
11
13
|
attr_reader :dir
|
12
14
|
|
13
|
-
|
14
15
|
attr_reader :aws_region
|
15
16
|
attr_reader :aws_bucket
|
16
17
|
attr_reader :aws_id
|
@@ -32,28 +33,31 @@ module RestFtpDaemon
|
|
32
33
|
resolve_tokens! location_uri
|
33
34
|
fix_scheme! location_uri
|
34
35
|
|
35
|
-
# Parse URL
|
36
|
-
parse_url location_uri
|
37
|
-
|
38
|
-
# Match AWS URL with BUCKET.s3.amazonaws.com
|
39
|
-
init_aws if @uri.is_a? URI::S3
|
40
|
-
|
41
|
-
# Set default user if not provided
|
42
|
-
init_username
|
43
|
-
|
44
36
|
# Ensure result does not contain tokens after replacement
|
45
37
|
detected_tokens = detect_tokens(location_uri)
|
46
38
|
unless detected_tokens.empty?
|
47
39
|
raise RestFtpDaemon::UnresolvedTokens, detected_tokens.join(' ')
|
48
40
|
end
|
49
41
|
|
42
|
+
# Parse URL and do specific initializations
|
43
|
+
parse_url location_uri
|
44
|
+
case @uri
|
45
|
+
when URI::FILE then init_file
|
46
|
+
when URI::S3 then init_aws # Match AWS URL with BUCKET.s3.amazonaws.com
|
47
|
+
end
|
48
|
+
|
50
49
|
# Check that scheme is supported
|
51
50
|
unless @uri.scheme
|
52
51
|
raise RestFtpDaemon::UnsupportedScheme, url
|
53
52
|
end
|
54
53
|
end
|
55
54
|
|
55
|
+
def is? kind
|
56
|
+
@uri.is_a? kind
|
57
|
+
end
|
58
|
+
|
56
59
|
def path
|
60
|
+
return @name if @dir.nil?
|
57
61
|
File.join(@dir.to_s, @name.to_s)
|
58
62
|
end
|
59
63
|
|
@@ -73,6 +77,10 @@ module RestFtpDaemon
|
|
73
77
|
return File.size local_fil_path
|
74
78
|
end
|
75
79
|
|
80
|
+
def generate_temp_name!
|
81
|
+
@name = "#{@name}.temp-#{identifier(JOB_TEMPFILE_LEN)}"
|
82
|
+
end
|
83
|
+
|
76
84
|
private
|
77
85
|
|
78
86
|
def tokenize item
|
@@ -101,24 +109,32 @@ module RestFtpDaemon
|
|
101
109
|
path.gsub! /^\/(.*)/, 'file:/\1'
|
102
110
|
end
|
103
111
|
|
104
|
-
def remove_multiple_slashes path
|
105
|
-
path.gsub! /([^:])\/\//, '\1/'
|
106
|
-
end
|
107
|
-
|
108
112
|
def parse_url path
|
109
113
|
# Parse that URL
|
110
114
|
@uri = URI.parse path # rescue nil
|
111
115
|
raise RestFtpDaemon::LocationParseError, location_path unless @uri
|
112
116
|
|
117
|
+
# Sanitize path
|
118
|
+
cleaned = @uri.path.clone
|
119
|
+
|
120
|
+
# remove_leading_slashes
|
121
|
+
cleaned.gsub! /^\//, ''
|
122
|
+
|
123
|
+
# remove_multiple_slashes
|
124
|
+
cleaned.gsub! /([^:])\/\//, '\1/'
|
125
|
+
|
113
126
|
# Store URL parts
|
114
|
-
@
|
115
|
-
@
|
116
|
-
@dir = extract_dirname uri.path
|
117
|
-
@name = extract_filename uri.path
|
127
|
+
@dir = extract_dirname cleaned
|
128
|
+
@name = extract_filename cleaned
|
118
129
|
end
|
119
130
|
|
120
|
-
def init_username
|
121
|
-
|
131
|
+
# def init_username
|
132
|
+
# @uri.user ||= "anonymous"
|
133
|
+
# end
|
134
|
+
|
135
|
+
def init_file
|
136
|
+
# Dir is absolute
|
137
|
+
@dir = File.join('/', @dir.to_s)
|
122
138
|
end
|
123
139
|
|
124
140
|
def init_aws
|
@@ -135,6 +151,9 @@ module RestFtpDaemon
|
|
135
151
|
# Credentials from config
|
136
152
|
@aws_id = Conf.at(:credentials, @uri.host, :id)
|
137
153
|
@aws_secret = Conf.at(:credentials, @uri.host, :secret)
|
154
|
+
|
155
|
+
# Clear @dir
|
156
|
+
@dir = nil
|
138
157
|
end
|
139
158
|
|
140
159
|
def extract_filename path
|
@@ -149,10 +168,6 @@ module RestFtpDaemon
|
|
149
168
|
return m[1].to_s unless m.nil?
|
150
169
|
end
|
151
170
|
|
152
|
-
def strip_leading_slash_from_dir!
|
153
|
-
@dir.to_s.gsub!(/^\//, '')
|
154
|
-
end
|
155
|
-
|
156
171
|
def detect_tokens item
|
157
172
|
item.scan /\[[^\[\]]*\]/
|
158
173
|
end
|
@@ -16,7 +16,7 @@ module RestFtpDaemon
|
|
16
16
|
# Init
|
17
17
|
@target = target
|
18
18
|
@ftpes = ftpes
|
19
|
-
@debug = debug
|
19
|
+
@debug = !!debug
|
20
20
|
|
21
21
|
# Build and empty job to protect set_info delegation
|
22
22
|
@job = Job.new(nil, {})
|
@@ -26,8 +26,7 @@ module RestFtpDaemon
|
|
26
26
|
@logger = BmcDaemonLib::LoggerPool.instance.get :transfer
|
27
27
|
|
28
28
|
# Annnounce object
|
29
|
-
log_info "Remote.initialize [#{target.path}]"
|
30
|
-
log_debug "Remote.initialize target[#{@target.inspect}]"
|
29
|
+
log_info "Remote.initialize debug[#{debug}] target[#{target.path}] "
|
31
30
|
|
32
31
|
# Prepare real object
|
33
32
|
prepare
|
@@ -26,8 +26,9 @@ module RestFtpDaemon
|
|
26
26
|
end
|
27
27
|
|
28
28
|
def connect
|
29
|
-
# Connect remote server
|
30
29
|
super
|
30
|
+
|
31
|
+
# Connect remote server
|
31
32
|
@ftp.connect @target.host, @target.port
|
32
33
|
@ftp.login @target.user, @target.password
|
33
34
|
end
|
@@ -43,26 +44,28 @@ module RestFtpDaemon
|
|
43
44
|
end
|
44
45
|
|
45
46
|
def remove! target
|
46
|
-
log_debug "RemoteFTP.remove! [#{target.name}]"
|
47
47
|
@ftp.delete target.path
|
48
48
|
rescue Net::FTPPermError
|
49
|
-
log_debug "
|
49
|
+
log_debug "RemoteFTP.remove! [#{target.name}] not found"
|
50
50
|
else
|
51
|
-
log_debug "
|
51
|
+
log_debug "RemoteFTP.remove! [#{target.name}] removed"
|
52
52
|
end
|
53
53
|
|
54
54
|
def mkdir directory
|
55
55
|
log_debug "RemoteFTP.mkdir [#{directory}]"
|
56
56
|
@ftp.mkdir directory
|
57
57
|
|
58
|
-
|
59
|
-
|
58
|
+
rescue StandardError => ex
|
59
|
+
raise TargetPermissionError, ex.message
|
60
60
|
end
|
61
61
|
|
62
62
|
def chdir_or_create directory, mkdir = false
|
63
63
|
# Init, extract my parent name and my own name
|
64
64
|
log_debug "RemoteFTP.chdir_or_create mkdir[#{mkdir}] dir[#{directory}]"
|
65
|
-
parent,
|
65
|
+
parent, current = extract_parent(directory)
|
66
|
+
|
67
|
+
#dirname, _current = extract_parent(directory)
|
68
|
+
|
66
69
|
|
67
70
|
# Access this directory
|
68
71
|
begin
|
@@ -74,7 +77,7 @@ module RestFtpDaemon
|
|
74
77
|
chdir_or_create parent, mkdir
|
75
78
|
|
76
79
|
# Now I was able to chdir into my parent, create the current directory
|
77
|
-
mkdir
|
80
|
+
mkdir current
|
78
81
|
|
79
82
|
# Finally retry the chdir
|
80
83
|
retry
|
@@ -83,20 +86,31 @@ module RestFtpDaemon
|
|
83
86
|
end
|
84
87
|
end
|
85
88
|
|
86
|
-
def
|
89
|
+
def upload source, target, use_temp_name = false, &callback
|
87
90
|
# Push init
|
88
|
-
raise RestFtpDaemon::AssertionFailed, "
|
91
|
+
raise RestFtpDaemon::AssertionFailed, "upload/ftp" if @ftp.nil?
|
89
92
|
|
90
|
-
# Temp file if
|
91
|
-
|
92
|
-
|
93
|
+
# Temp file if needed
|
94
|
+
dest = target.clone
|
95
|
+
if use_temp_name
|
96
|
+
dest.generate_temp_name!
|
97
|
+
end
|
93
98
|
|
94
|
-
#
|
95
|
-
log_debug "RemoteFTP.
|
99
|
+
# Move to the directory
|
100
|
+
log_debug "RemoteFTP.upload chdir [#{dest.dir}]"
|
101
|
+
@ftp.chdir "/#{dest.dir}"
|
96
102
|
|
97
|
-
|
103
|
+
# Do the transfer
|
104
|
+
log_debug "RemoteFTP.upload putbinaryfile [#{dest.name}]"
|
105
|
+
@ftp.putbinaryfile source.path, dest.name, @chunk_size do |data|
|
98
106
|
# Update job status after this block transfer
|
99
|
-
yield data.bytesize,
|
107
|
+
yield data.bytesize, dest.name
|
108
|
+
end
|
109
|
+
|
110
|
+
# Move the file back to its original name
|
111
|
+
if use_temp_name
|
112
|
+
log_debug "RemoteFTP.upload rename [#{dest.name}] > [#{target.name}]"
|
113
|
+
@ftp.rename dest.name, target.name
|
100
114
|
end
|
101
115
|
end
|
102
116
|
|
@@ -13,9 +13,10 @@ module RestFtpDaemon
|
|
13
13
|
end
|
14
14
|
|
15
15
|
def connect
|
16
|
-
# Connect init
|
17
16
|
super
|
18
|
-
|
17
|
+
|
18
|
+
# Connect init
|
19
|
+
log_debug "RemoteS3.connect region[#{target.aws_region}] id[#{target.aws_id}]"
|
19
20
|
|
20
21
|
# Debug level
|
21
22
|
verbosity = @debug ? Logger::DEBUG : false
|
@@ -27,19 +28,27 @@ module RestFtpDaemon
|
|
27
28
|
)
|
28
29
|
end
|
29
30
|
|
30
|
-
def
|
31
|
+
def upload source, target, use_temp_name = false, &callback
|
31
32
|
# Push init
|
32
|
-
raise RestFtpDaemon::AssertionFailed, "
|
33
|
-
log_debug "RemoteS3.
|
33
|
+
raise RestFtpDaemon::AssertionFailed, "upload/client" if @client.nil?
|
34
|
+
log_debug "RemoteS3.upload bucket[#{target.aws_bucket}] name[#{target.name}]"
|
35
|
+
|
36
|
+
# Update progress before
|
37
|
+
#yield 0, target.name
|
34
38
|
|
35
39
|
# Do the transfer
|
36
40
|
bucket = @client.bucket(target.aws_bucket)
|
37
41
|
object = bucket.object(target.name)
|
38
|
-
object.
|
42
|
+
object.upload_file source.path do |progress, total|
|
43
|
+
log_debug "- progress[#{progress}] total[#{total}]"
|
44
|
+
end
|
45
|
+
|
46
|
+
# Update progress after
|
47
|
+
#yield target.size, target.name
|
39
48
|
|
40
49
|
# Dump information about this file
|
41
|
-
log_debug "RemoteS3.
|
42
|
-
log_debug "RemoteS3.
|
50
|
+
log_debug "RemoteS3.upload url[#{object.public_url}]"
|
51
|
+
log_debug "RemoteS3.upload etag[#{object.etag}]"
|
43
52
|
set_info :target, :aws_public_url, object.public_url
|
44
53
|
set_info :target, :aws_etag, object.etag
|
45
54
|
end
|
@@ -11,8 +11,9 @@ module RestFtpDaemon
|
|
11
11
|
end
|
12
12
|
|
13
13
|
def connect
|
14
|
-
# Connect init
|
15
14
|
super
|
15
|
+
|
16
|
+
# Connect init
|
16
17
|
log_debug "RemoteSFTP.connect [#{@target.user}]@[#{@target.host}]:[#{@target.port}]"
|
17
18
|
|
18
19
|
# Debug level
|
@@ -52,8 +53,8 @@ module RestFtpDaemon
|
|
52
53
|
log_debug "RemoteSFTP.mkdir [#{directory}]"
|
53
54
|
@sftp.mkdir! directory
|
54
55
|
|
55
|
-
rescue
|
56
|
-
raise TargetPermissionError
|
56
|
+
rescue StandardError => ex
|
57
|
+
raise TargetPermissionError, ex.message
|
57
58
|
end
|
58
59
|
|
59
60
|
def chdir_or_create directory, mkdir = false
|
@@ -63,8 +64,8 @@ module RestFtpDaemon
|
|
63
64
|
|
64
65
|
# Access this directory
|
65
66
|
begin
|
66
|
-
log_debug "chdir [/#{directory}]"
|
67
|
-
@sftp.opendir!
|
67
|
+
#log_debug "chdir [/#{directory}]"
|
68
|
+
@sftp.opendir! directory
|
68
69
|
|
69
70
|
rescue Net::SFTP::StatusException => _e
|
70
71
|
# If not allowed to create path, that's over, we're stuck
|
@@ -86,17 +87,19 @@ module RestFtpDaemon
|
|
86
87
|
raise JobTargetShouldBeDirectory
|
87
88
|
end
|
88
89
|
|
89
|
-
def
|
90
|
+
def upload source, target, use_temp_name = false, &callback
|
90
91
|
# Push init
|
91
|
-
raise RestFtpDaemon::AssertionFailed, "
|
92
|
+
raise RestFtpDaemon::AssertionFailed, "upload/sftp" if @sftp.nil?
|
92
93
|
|
93
|
-
# Temp file if
|
94
|
-
|
95
|
-
|
94
|
+
# Temp file if needed
|
95
|
+
dest = target.clone
|
96
|
+
if use_temp_name
|
97
|
+
dest.generate_temp_name!
|
98
|
+
end
|
96
99
|
|
97
100
|
# Do the transfer
|
98
|
-
log_debug "RemoteSFTP.
|
99
|
-
@sftp.upload! source.path,
|
101
|
+
log_debug "RemoteSFTP.upload temp[#{use_temp_name}] name[#{dest.name}]"
|
102
|
+
@sftp.upload! source.path, dest.path do |event, _uploader, *args|
|
100
103
|
case event
|
101
104
|
when :open then
|
102
105
|
# args[0] : file metadata
|
@@ -107,7 +110,7 @@ module RestFtpDaemon
|
|
107
110
|
# puts "writing #{args[2].length} bytes to #{args[0].remote} starting at #{args[1]}"
|
108
111
|
|
109
112
|
# Update job status after this block transfer
|
110
|
-
yield args[2].length,
|
113
|
+
yield args[2].length, dest.name
|
111
114
|
|
112
115
|
when :close then
|
113
116
|
# args[0] : file metadata
|
@@ -118,13 +121,11 @@ module RestFtpDaemon
|
|
118
121
|
|
119
122
|
end
|
120
123
|
|
121
|
-
#
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
log_debug "RemoteSFTP.push rename to\t[#{target.name}]"
|
127
|
-
@sftp.rename! destination.path, target.path, flags
|
124
|
+
# Move the file back to its original name
|
125
|
+
if use_temp_name
|
126
|
+
flags = 0x00000001
|
127
|
+
log_debug "RemoteSFTP.upload rename [#{dest.name}] > [#{target.name}]"
|
128
|
+
@sftp.rename! dest.path, target.path, flags
|
128
129
|
end
|
129
130
|
|
130
131
|
# progress:
|
@@ -27,13 +27,11 @@
|
|
27
27
|
|
28
28
|
%td= job_type job
|
29
29
|
|
30
|
-
%td
|
31
|
-
|
32
|
-
= token_highlight job.source
|
30
|
+
%td.text-right= location_label job.source_uri
|
31
|
+
%td= token_highlight job.source
|
33
32
|
|
34
|
-
%td
|
35
|
-
|
36
|
-
= token_highlight job.target
|
33
|
+
%td.text-right= location_label job.target_uri
|
34
|
+
%td= token_highlight job.target
|
37
35
|
|
38
36
|
%td= datetime_short(job.queued_at)
|
39
37
|
|
data/rest-ftp-daemon.gemspec
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rest-ftp-daemon
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.421.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Bruno MEDICI
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-09-
|
11
|
+
date: 2016-09-02 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|