rest-ftp-daemon 0.420.2 → 0.421.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +4 -4
- data/lib/rest-ftp-daemon/helpers/views.rb +2 -1
- data/lib/rest-ftp-daemon/job.rb +6 -11
- data/lib/rest-ftp-daemon/jobs/errors.rb +5 -6
- data/lib/rest-ftp-daemon/jobs/transfer.rb +20 -31
- data/lib/rest-ftp-daemon/jobs/video.rb +2 -2
- data/lib/rest-ftp-daemon/location.rb +39 -24
- data/lib/rest-ftp-daemon/remote.rb +2 -3
- data/lib/rest-ftp-daemon/remote_ftp.rb +31 -17
- data/lib/rest-ftp-daemon/remote_s3.rb +17 -8
- data/lib/rest-ftp-daemon/remote_sftp.rb +21 -20
- data/lib/rest-ftp-daemon/views/dashboard_jobs.haml +2 -0
- data/lib/rest-ftp-daemon/views/dashboard_table.haml +4 -6
- data/lib/rest-ftp-daemon/views/dashboard_workers.haml +1 -1
- data/lib/rest-ftp-daemon/workers/reporter.rb +0 -1
- data/rest-ftp-daemon.gemspec +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c257143aa55e12efe93dcd1f9e2829e662a27d33
|
4
|
+
data.tar.gz: 75b635758132ab9af8e381a9edcca9cfa2b057de
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 966ca0d76da55605f512de091d853a5a250b2b85cfff0419dc1fbacb54a0065a83b68a11bdb507c8dfaf25834050346d8ec21816ab2d69b806dd2b57d2813989
|
7
|
+
data.tar.gz: 53d3336b265bb1282c4412d49be9e3a604e8463df9d3fb6d549be7edfe4b9dc010851dac14194bde48d5c45c18126e7e53a55476b9a99618ef1331db42f59460
|
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
rest-ftp-daemon (0.
|
4
|
+
rest-ftp-daemon (0.421.0)
|
5
5
|
activesupport (~> 4.2)
|
6
6
|
api-auth
|
7
7
|
aws-sdk-resources (~> 2)
|
@@ -39,10 +39,10 @@ GEM
|
|
39
39
|
ast (2.3.0)
|
40
40
|
astrolabe (1.3.1)
|
41
41
|
parser (~> 2.2)
|
42
|
-
aws-sdk-core (2.5.
|
42
|
+
aws-sdk-core (2.5.8)
|
43
43
|
jmespath (~> 1.0)
|
44
|
-
aws-sdk-resources (2.5.
|
45
|
-
aws-sdk-core (= 2.5.
|
44
|
+
aws-sdk-resources (2.5.8)
|
45
|
+
aws-sdk-core (= 2.5.8)
|
46
46
|
axiom-types (0.1.1)
|
47
47
|
descendants_tracker (~> 0.0.4)
|
48
48
|
ice_nine (~> 0.11.0)
|
@@ -60,8 +60,9 @@ module RestFtpDaemon
|
|
60
60
|
|
61
61
|
def location_label uri
|
62
62
|
sprintf(
|
63
|
-
'<div class="transfer-type label label-%s">%s</div>',
|
63
|
+
'<div class="transfer-type label label-%s" title="%s">%s</div>',
|
64
64
|
location_style(uri),
|
65
|
+
uri.to_s,
|
65
66
|
uri.class.name.split('::').last
|
66
67
|
)
|
67
68
|
end
|
data/lib/rest-ftp-daemon/job.rb
CHANGED
@@ -69,7 +69,6 @@ module RestFtpDaemon
|
|
69
69
|
# Logger
|
70
70
|
@logger = BmcDaemonLib::LoggerPool.instance.get :transfer
|
71
71
|
|
72
|
-
|
73
72
|
# Import query params
|
74
73
|
FIELDS.each do |name|
|
75
74
|
instance_variable_set "@#{name}", params[name]
|
@@ -140,7 +139,6 @@ module RestFtpDaemon
|
|
140
139
|
do_after
|
141
140
|
|
142
141
|
rescue StandardError => exception
|
143
|
-
log_debug "Job.process caught [#{exception.class}] #{exception.message}"
|
144
142
|
return oops current_signal, exception
|
145
143
|
|
146
144
|
else
|
@@ -251,15 +249,12 @@ module RestFtpDaemon
|
|
251
249
|
|
252
250
|
def set_info_location prefix, location
|
253
251
|
return unless location.is_a? Location
|
254
|
-
|
255
|
-
|
256
|
-
|
257
|
-
|
258
|
-
|
259
|
-
|
260
|
-
set_info prefix, :aws_region, location.aws_region
|
261
|
-
set_info prefix, :aws_bucket, location.aws_bucket
|
262
|
-
set_info prefix, :aws_id, location.aws_id
|
252
|
+
fields = [:uri, :scheme, :user, :host, :port, :dir, :name, :path, :aws_region, :aws_bucket, :aws_id]
|
253
|
+
|
254
|
+
# Add each field to @infos
|
255
|
+
fields.each do |what|
|
256
|
+
set_info prefix, what, location.send(what)
|
257
|
+
end
|
263
258
|
end
|
264
259
|
|
265
260
|
private
|
@@ -1,8 +1,7 @@
|
|
1
1
|
require "securerandom"
|
2
2
|
require "double_bag_ftps"
|
3
|
-
require "net/ssh"
|
4
|
-
require "net/ftp"
|
5
3
|
require "net/sftp"
|
4
|
+
require "net/ftp"
|
6
5
|
require 'streamio-ffmpeg'
|
7
6
|
|
8
7
|
module RestFtpDaemon
|
@@ -42,10 +41,10 @@ module RestFtpDaemon
|
|
42
41
|
|
43
42
|
ffmpeg_error: FFMPEG::Error,
|
44
43
|
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
sftp_openssl_error:
|
44
|
+
sftp_exception: Net::SFTP::StatusException,
|
45
|
+
sftp_key_mismatch: Net::SSH::HostKeyMismatch,
|
46
|
+
sftp_auth_failed: Net::SSH::AuthenticationFailed,
|
47
|
+
sftp_openssl_error: OpenSSL::SSL::SSLError,
|
49
48
|
# rescue Encoding::UndefinedConversionError => exception
|
50
49
|
# return oops :ended, exception, "encoding_error", true
|
51
50
|
}
|
@@ -16,13 +16,13 @@ module RestFtpDaemon
|
|
16
16
|
set_info :source, :processed, 0
|
17
17
|
|
18
18
|
# Ensure source is FILE
|
19
|
-
raise RestFtpDaemon::SourceNotSupported, @source_loc.scheme unless
|
19
|
+
raise RestFtpDaemon::SourceNotSupported, @source_loc.scheme unless @source_loc.is? URI::FILE
|
20
20
|
|
21
21
|
# Prepare remote object
|
22
22
|
case target_uri
|
23
23
|
when URI::FTP
|
24
24
|
log_info "JobTransfer.before target_method FTP"
|
25
|
-
@remote = RemoteFTP.new @target_loc, log_prefix, @config[:
|
25
|
+
@remote = RemoteFTP.new @target_loc, log_prefix, @config[:debug_ftp]
|
26
26
|
when URI::FTPES, URI::FTPS
|
27
27
|
log_info "JobTransfer.before target_method FTPES/FTPS"
|
28
28
|
@remote = RemoteFTP.new @target_loc, log_prefix, @config[:debug_ftps], :ftpes
|
@@ -48,10 +48,9 @@ module RestFtpDaemon
|
|
48
48
|
# Scan local source files from disk
|
49
49
|
set_status JOB_STATUS_CHECKING_SRC
|
50
50
|
sources = @source_loc.scan_files
|
51
|
-
log_debug "JobTransfer.work sources result #{sources.inspect}"
|
52
51
|
set_info :source, :count, sources.size
|
53
52
|
set_info :source, :files, sources.collect(&:name)
|
54
|
-
log_info "JobTransfer.work sources
|
53
|
+
log_info "JobTransfer.work sources #{sources.collect(&:name)}"
|
55
54
|
raise RestFtpDaemon::SourceNotFound if sources.empty?
|
56
55
|
|
57
56
|
# Guess target file name, and fail if present while we matched multiple sources
|
@@ -77,19 +76,15 @@ module RestFtpDaemon
|
|
77
76
|
source_processed = 0
|
78
77
|
targets = []
|
79
78
|
sources.each do |source|
|
80
|
-
#
|
81
|
-
|
82
|
-
|
83
|
-
# Add the source file name if none found in the target path
|
84
|
-
unless target_final.name
|
85
|
-
target_final.name = source.name
|
86
|
-
end
|
79
|
+
# Build final target, add the source file name if noneh
|
80
|
+
target = @target_loc.clone
|
81
|
+
target.name = source.name unless target.name
|
87
82
|
|
88
83
|
# Do the transfer, for each file
|
89
|
-
|
84
|
+
remote_upload source, target
|
90
85
|
|
91
86
|
# Add it to transferred target names
|
92
|
-
targets <<
|
87
|
+
targets << target.name
|
93
88
|
set_info :target, :files, targets
|
94
89
|
|
95
90
|
# Update counters
|
@@ -99,7 +94,6 @@ module RestFtpDaemon
|
|
99
94
|
|
100
95
|
def do_after
|
101
96
|
# Close FTP connexion and free up memory
|
102
|
-
log_info "JobTransfer.after"
|
103
97
|
@remote.close
|
104
98
|
|
105
99
|
# Free-up remote object
|
@@ -114,28 +108,21 @@ module RestFtpDaemon
|
|
114
108
|
RestFtpDaemon::Counters.instance.add :data, :transferred, @transfer_total
|
115
109
|
end
|
116
110
|
|
117
|
-
def
|
111
|
+
def remote_upload source, target
|
118
112
|
# Method assertions
|
119
|
-
raise RestFtpDaemon::AssertionFailed, "
|
120
|
-
raise RestFtpDaemon::AssertionFailed, "
|
121
|
-
raise RestFtpDaemon::AssertionFailed, "
|
113
|
+
raise RestFtpDaemon::AssertionFailed, "remote_upload/remote" if @remote.nil?
|
114
|
+
raise RestFtpDaemon::AssertionFailed, "remote_upload/source" if source.nil?
|
115
|
+
raise RestFtpDaemon::AssertionFailed, "remote_upload/target" if target.nil?
|
122
116
|
|
123
117
|
# Use source filename if target path provided none (typically with multiple sources)
|
124
|
-
log_info "JobTransfer.
|
118
|
+
log_info "JobTransfer.remote_upload [#{source.name}]: [#{source.path}] > [#{target.path}]"
|
125
119
|
set_info :source, :current, source.name
|
126
120
|
|
127
|
-
#
|
128
|
-
tempname = nil
|
129
|
-
if @tempfile
|
130
|
-
tempname = "#{target.name}.temp-#{identifier(JOB_TEMPFILE_LEN)}"
|
131
|
-
log_debug "JobTransfer.remote_push tempname [#{tempname}]"
|
132
|
-
end
|
133
|
-
|
134
|
-
# Remove any existing version if expected, or test its presence
|
121
|
+
# Remove any existing version if present, or check if it's there
|
135
122
|
if @overwrite
|
136
123
|
@remote.remove! target
|
137
124
|
elsif size = @remote.present?(target)
|
138
|
-
log_debug "JobTransfer.
|
125
|
+
log_debug "JobTransfer.remote_upload existing (#{format_bytes size, 'B'})"
|
139
126
|
raise RestFtpDaemon::TargetFileExists
|
140
127
|
end
|
141
128
|
|
@@ -146,9 +133,11 @@ module RestFtpDaemon
|
|
146
133
|
|
147
134
|
# Start the transfer, update job status after each block transfer
|
148
135
|
set_status JOB_STATUS_UPLOADING
|
149
|
-
|
136
|
+
log_debug "JobTransfer.remote_upload source[#{source.path}] temp[#{@tempfile}]"
|
137
|
+
@remote.upload source, target, @tempfile do |transferred, name|
|
138
|
+
|
150
139
|
# Update transfer statistics
|
151
|
-
|
140
|
+
update_progress transferred, name
|
152
141
|
|
153
142
|
# Touch my worker status
|
154
143
|
touch_job
|
@@ -162,7 +151,7 @@ module RestFtpDaemon
|
|
162
151
|
set_info :source, :current, nil
|
163
152
|
end
|
164
153
|
|
165
|
-
def
|
154
|
+
def update_progress transferred, name = ""
|
166
155
|
# What's current time ?
|
167
156
|
now = Time.now
|
168
157
|
notify_after = @config[:notify_after]
|
@@ -22,8 +22,8 @@ module RestFtpDaemon
|
|
22
22
|
end
|
23
23
|
|
24
24
|
# Ensure source and target are FILE
|
25
|
-
raise RestFtpDaemon::SourceNotSupported, @source_loc.scheme unless
|
26
|
-
raise RestFtpDaemon::TargetNotSupported, @
|
25
|
+
raise RestFtpDaemon::SourceNotSupported, @source_loc.scheme unless @source_loc.is? URI::FILE
|
26
|
+
raise RestFtpDaemon::TargetNotSupported, @target_loc.scheme unless @target_loc.is? URI::FILE
|
27
27
|
end
|
28
28
|
|
29
29
|
def do_work
|
@@ -3,6 +3,8 @@ require 'active_support/core_ext/module/delegation'
|
|
3
3
|
|
4
4
|
module RestFtpDaemon
|
5
5
|
class Location
|
6
|
+
include CommonHelpers
|
7
|
+
|
6
8
|
# Accessors
|
7
9
|
attr_accessor :name
|
8
10
|
|
@@ -10,7 +12,6 @@ module RestFtpDaemon
|
|
10
12
|
attr_reader :scheme
|
11
13
|
attr_reader :dir
|
12
14
|
|
13
|
-
|
14
15
|
attr_reader :aws_region
|
15
16
|
attr_reader :aws_bucket
|
16
17
|
attr_reader :aws_id
|
@@ -32,28 +33,31 @@ module RestFtpDaemon
|
|
32
33
|
resolve_tokens! location_uri
|
33
34
|
fix_scheme! location_uri
|
34
35
|
|
35
|
-
# Parse URL
|
36
|
-
parse_url location_uri
|
37
|
-
|
38
|
-
# Match AWS URL with BUCKET.s3.amazonaws.com
|
39
|
-
init_aws if @uri.is_a? URI::S3
|
40
|
-
|
41
|
-
# Set default user if not provided
|
42
|
-
init_username
|
43
|
-
|
44
36
|
# Ensure result does not contain tokens after replacement
|
45
37
|
detected_tokens = detect_tokens(location_uri)
|
46
38
|
unless detected_tokens.empty?
|
47
39
|
raise RestFtpDaemon::UnresolvedTokens, detected_tokens.join(' ')
|
48
40
|
end
|
49
41
|
|
42
|
+
# Parse URL and do specific initializations
|
43
|
+
parse_url location_uri
|
44
|
+
case @uri
|
45
|
+
when URI::FILE then init_file
|
46
|
+
when URI::S3 then init_aws # Match AWS URL with BUCKET.s3.amazonaws.com
|
47
|
+
end
|
48
|
+
|
50
49
|
# Check that scheme is supported
|
51
50
|
unless @uri.scheme
|
52
51
|
raise RestFtpDaemon::UnsupportedScheme, url
|
53
52
|
end
|
54
53
|
end
|
55
54
|
|
55
|
+
def is? kind
|
56
|
+
@uri.is_a? kind
|
57
|
+
end
|
58
|
+
|
56
59
|
def path
|
60
|
+
return @name if @dir.nil?
|
57
61
|
File.join(@dir.to_s, @name.to_s)
|
58
62
|
end
|
59
63
|
|
@@ -73,6 +77,10 @@ module RestFtpDaemon
|
|
73
77
|
return File.size local_fil_path
|
74
78
|
end
|
75
79
|
|
80
|
+
def generate_temp_name!
|
81
|
+
@name = "#{@name}.temp-#{identifier(JOB_TEMPFILE_LEN)}"
|
82
|
+
end
|
83
|
+
|
76
84
|
private
|
77
85
|
|
78
86
|
def tokenize item
|
@@ -101,24 +109,32 @@ module RestFtpDaemon
|
|
101
109
|
path.gsub! /^\/(.*)/, 'file:/\1'
|
102
110
|
end
|
103
111
|
|
104
|
-
def remove_multiple_slashes path
|
105
|
-
path.gsub! /([^:])\/\//, '\1/'
|
106
|
-
end
|
107
|
-
|
108
112
|
def parse_url path
|
109
113
|
# Parse that URL
|
110
114
|
@uri = URI.parse path # rescue nil
|
111
115
|
raise RestFtpDaemon::LocationParseError, location_path unless @uri
|
112
116
|
|
117
|
+
# Sanitize path
|
118
|
+
cleaned = @uri.path.clone
|
119
|
+
|
120
|
+
# remove_leading_slashes
|
121
|
+
cleaned.gsub! /^\//, ''
|
122
|
+
|
123
|
+
# remove_multiple_slashes
|
124
|
+
cleaned.gsub! /([^:])\/\//, '\1/'
|
125
|
+
|
113
126
|
# Store URL parts
|
114
|
-
@
|
115
|
-
@
|
116
|
-
@dir = extract_dirname uri.path
|
117
|
-
@name = extract_filename uri.path
|
127
|
+
@dir = extract_dirname cleaned
|
128
|
+
@name = extract_filename cleaned
|
118
129
|
end
|
119
130
|
|
120
|
-
def init_username
|
121
|
-
|
131
|
+
# def init_username
|
132
|
+
# @uri.user ||= "anonymous"
|
133
|
+
# end
|
134
|
+
|
135
|
+
def init_file
|
136
|
+
# Dir is absolute
|
137
|
+
@dir = File.join('/', @dir.to_s)
|
122
138
|
end
|
123
139
|
|
124
140
|
def init_aws
|
@@ -135,6 +151,9 @@ module RestFtpDaemon
|
|
135
151
|
# Credentials from config
|
136
152
|
@aws_id = Conf.at(:credentials, @uri.host, :id)
|
137
153
|
@aws_secret = Conf.at(:credentials, @uri.host, :secret)
|
154
|
+
|
155
|
+
# Clear @dir
|
156
|
+
@dir = nil
|
138
157
|
end
|
139
158
|
|
140
159
|
def extract_filename path
|
@@ -149,10 +168,6 @@ module RestFtpDaemon
|
|
149
168
|
return m[1].to_s unless m.nil?
|
150
169
|
end
|
151
170
|
|
152
|
-
def strip_leading_slash_from_dir!
|
153
|
-
@dir.to_s.gsub!(/^\//, '')
|
154
|
-
end
|
155
|
-
|
156
171
|
def detect_tokens item
|
157
172
|
item.scan /\[[^\[\]]*\]/
|
158
173
|
end
|
@@ -16,7 +16,7 @@ module RestFtpDaemon
|
|
16
16
|
# Init
|
17
17
|
@target = target
|
18
18
|
@ftpes = ftpes
|
19
|
-
@debug = debug
|
19
|
+
@debug = !!debug
|
20
20
|
|
21
21
|
# Build and empty job to protect set_info delegation
|
22
22
|
@job = Job.new(nil, {})
|
@@ -26,8 +26,7 @@ module RestFtpDaemon
|
|
26
26
|
@logger = BmcDaemonLib::LoggerPool.instance.get :transfer
|
27
27
|
|
28
28
|
# Annnounce object
|
29
|
-
log_info "Remote.initialize [#{target.path}]"
|
30
|
-
log_debug "Remote.initialize target[#{@target.inspect}]"
|
29
|
+
log_info "Remote.initialize debug[#{debug}] target[#{target.path}] "
|
31
30
|
|
32
31
|
# Prepare real object
|
33
32
|
prepare
|
@@ -26,8 +26,9 @@ module RestFtpDaemon
|
|
26
26
|
end
|
27
27
|
|
28
28
|
def connect
|
29
|
-
# Connect remote server
|
30
29
|
super
|
30
|
+
|
31
|
+
# Connect remote server
|
31
32
|
@ftp.connect @target.host, @target.port
|
32
33
|
@ftp.login @target.user, @target.password
|
33
34
|
end
|
@@ -43,26 +44,28 @@ module RestFtpDaemon
|
|
43
44
|
end
|
44
45
|
|
45
46
|
def remove! target
|
46
|
-
log_debug "RemoteFTP.remove! [#{target.name}]"
|
47
47
|
@ftp.delete target.path
|
48
48
|
rescue Net::FTPPermError
|
49
|
-
log_debug "
|
49
|
+
log_debug "RemoteFTP.remove! [#{target.name}] not found"
|
50
50
|
else
|
51
|
-
log_debug "
|
51
|
+
log_debug "RemoteFTP.remove! [#{target.name}] removed"
|
52
52
|
end
|
53
53
|
|
54
54
|
def mkdir directory
|
55
55
|
log_debug "RemoteFTP.mkdir [#{directory}]"
|
56
56
|
@ftp.mkdir directory
|
57
57
|
|
58
|
-
|
59
|
-
|
58
|
+
rescue StandardError => ex
|
59
|
+
raise TargetPermissionError, ex.message
|
60
60
|
end
|
61
61
|
|
62
62
|
def chdir_or_create directory, mkdir = false
|
63
63
|
# Init, extract my parent name and my own name
|
64
64
|
log_debug "RemoteFTP.chdir_or_create mkdir[#{mkdir}] dir[#{directory}]"
|
65
|
-
parent,
|
65
|
+
parent, current = extract_parent(directory)
|
66
|
+
|
67
|
+
#dirname, _current = extract_parent(directory)
|
68
|
+
|
66
69
|
|
67
70
|
# Access this directory
|
68
71
|
begin
|
@@ -74,7 +77,7 @@ module RestFtpDaemon
|
|
74
77
|
chdir_or_create parent, mkdir
|
75
78
|
|
76
79
|
# Now I was able to chdir into my parent, create the current directory
|
77
|
-
mkdir
|
80
|
+
mkdir current
|
78
81
|
|
79
82
|
# Finally retry the chdir
|
80
83
|
retry
|
@@ -83,20 +86,31 @@ module RestFtpDaemon
|
|
83
86
|
end
|
84
87
|
end
|
85
88
|
|
86
|
-
def
|
89
|
+
def upload source, target, use_temp_name = false, &callback
|
87
90
|
# Push init
|
88
|
-
raise RestFtpDaemon::AssertionFailed, "
|
91
|
+
raise RestFtpDaemon::AssertionFailed, "upload/ftp" if @ftp.nil?
|
89
92
|
|
90
|
-
# Temp file if
|
91
|
-
|
92
|
-
|
93
|
+
# Temp file if needed
|
94
|
+
dest = target.clone
|
95
|
+
if use_temp_name
|
96
|
+
dest.generate_temp_name!
|
97
|
+
end
|
93
98
|
|
94
|
-
#
|
95
|
-
log_debug "RemoteFTP.
|
99
|
+
# Move to the directory
|
100
|
+
log_debug "RemoteFTP.upload chdir [#{dest.dir}]"
|
101
|
+
@ftp.chdir "/#{dest.dir}"
|
96
102
|
|
97
|
-
|
103
|
+
# Do the transfer
|
104
|
+
log_debug "RemoteFTP.upload putbinaryfile [#{dest.name}]"
|
105
|
+
@ftp.putbinaryfile source.path, dest.name, @chunk_size do |data|
|
98
106
|
# Update job status after this block transfer
|
99
|
-
yield data.bytesize,
|
107
|
+
yield data.bytesize, dest.name
|
108
|
+
end
|
109
|
+
|
110
|
+
# Move the file back to its original name
|
111
|
+
if use_temp_name
|
112
|
+
log_debug "RemoteFTP.upload rename [#{dest.name}] > [#{target.name}]"
|
113
|
+
@ftp.rename dest.name, target.name
|
100
114
|
end
|
101
115
|
end
|
102
116
|
|
@@ -13,9 +13,10 @@ module RestFtpDaemon
|
|
13
13
|
end
|
14
14
|
|
15
15
|
def connect
|
16
|
-
# Connect init
|
17
16
|
super
|
18
|
-
|
17
|
+
|
18
|
+
# Connect init
|
19
|
+
log_debug "RemoteS3.connect region[#{target.aws_region}] id[#{target.aws_id}]"
|
19
20
|
|
20
21
|
# Debug level
|
21
22
|
verbosity = @debug ? Logger::DEBUG : false
|
@@ -27,19 +28,27 @@ module RestFtpDaemon
|
|
27
28
|
)
|
28
29
|
end
|
29
30
|
|
30
|
-
def
|
31
|
+
def upload source, target, use_temp_name = false, &callback
|
31
32
|
# Push init
|
32
|
-
raise RestFtpDaemon::AssertionFailed, "
|
33
|
-
log_debug "RemoteS3.
|
33
|
+
raise RestFtpDaemon::AssertionFailed, "upload/client" if @client.nil?
|
34
|
+
log_debug "RemoteS3.upload bucket[#{target.aws_bucket}] name[#{target.name}]"
|
35
|
+
|
36
|
+
# Update progress before
|
37
|
+
#yield 0, target.name
|
34
38
|
|
35
39
|
# Do the transfer
|
36
40
|
bucket = @client.bucket(target.aws_bucket)
|
37
41
|
object = bucket.object(target.name)
|
38
|
-
object.
|
42
|
+
object.upload_file source.path do |progress, total|
|
43
|
+
log_debug "- progress[#{progress}] total[#{total}]"
|
44
|
+
end
|
45
|
+
|
46
|
+
# Update progress after
|
47
|
+
#yield target.size, target.name
|
39
48
|
|
40
49
|
# Dump information about this file
|
41
|
-
log_debug "RemoteS3.
|
42
|
-
log_debug "RemoteS3.
|
50
|
+
log_debug "RemoteS3.upload url[#{object.public_url}]"
|
51
|
+
log_debug "RemoteS3.upload etag[#{object.etag}]"
|
43
52
|
set_info :target, :aws_public_url, object.public_url
|
44
53
|
set_info :target, :aws_etag, object.etag
|
45
54
|
end
|
@@ -11,8 +11,9 @@ module RestFtpDaemon
|
|
11
11
|
end
|
12
12
|
|
13
13
|
def connect
|
14
|
-
# Connect init
|
15
14
|
super
|
15
|
+
|
16
|
+
# Connect init
|
16
17
|
log_debug "RemoteSFTP.connect [#{@target.user}]@[#{@target.host}]:[#{@target.port}]"
|
17
18
|
|
18
19
|
# Debug level
|
@@ -52,8 +53,8 @@ module RestFtpDaemon
|
|
52
53
|
log_debug "RemoteSFTP.mkdir [#{directory}]"
|
53
54
|
@sftp.mkdir! directory
|
54
55
|
|
55
|
-
rescue
|
56
|
-
raise TargetPermissionError
|
56
|
+
rescue StandardError => ex
|
57
|
+
raise TargetPermissionError, ex.message
|
57
58
|
end
|
58
59
|
|
59
60
|
def chdir_or_create directory, mkdir = false
|
@@ -63,8 +64,8 @@ module RestFtpDaemon
|
|
63
64
|
|
64
65
|
# Access this directory
|
65
66
|
begin
|
66
|
-
log_debug "chdir [/#{directory}]"
|
67
|
-
@sftp.opendir!
|
67
|
+
#log_debug "chdir [/#{directory}]"
|
68
|
+
@sftp.opendir! directory
|
68
69
|
|
69
70
|
rescue Net::SFTP::StatusException => _e
|
70
71
|
# If not allowed to create path, that's over, we're stuck
|
@@ -86,17 +87,19 @@ module RestFtpDaemon
|
|
86
87
|
raise JobTargetShouldBeDirectory
|
87
88
|
end
|
88
89
|
|
89
|
-
def
|
90
|
+
def upload source, target, use_temp_name = false, &callback
|
90
91
|
# Push init
|
91
|
-
raise RestFtpDaemon::AssertionFailed, "
|
92
|
+
raise RestFtpDaemon::AssertionFailed, "upload/sftp" if @sftp.nil?
|
92
93
|
|
93
|
-
# Temp file if
|
94
|
-
|
95
|
-
|
94
|
+
# Temp file if needed
|
95
|
+
dest = target.clone
|
96
|
+
if use_temp_name
|
97
|
+
dest.generate_temp_name!
|
98
|
+
end
|
96
99
|
|
97
100
|
# Do the transfer
|
98
|
-
log_debug "RemoteSFTP.
|
99
|
-
@sftp.upload! source.path,
|
101
|
+
log_debug "RemoteSFTP.upload temp[#{use_temp_name}] name[#{dest.name}]"
|
102
|
+
@sftp.upload! source.path, dest.path do |event, _uploader, *args|
|
100
103
|
case event
|
101
104
|
when :open then
|
102
105
|
# args[0] : file metadata
|
@@ -107,7 +110,7 @@ module RestFtpDaemon
|
|
107
110
|
# puts "writing #{args[2].length} bytes to #{args[0].remote} starting at #{args[1]}"
|
108
111
|
|
109
112
|
# Update job status after this block transfer
|
110
|
-
yield args[2].length,
|
113
|
+
yield args[2].length, dest.name
|
111
114
|
|
112
115
|
when :close then
|
113
116
|
# args[0] : file metadata
|
@@ -118,13 +121,11 @@ module RestFtpDaemon
|
|
118
121
|
|
119
122
|
end
|
120
123
|
|
121
|
-
#
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
log_debug "RemoteSFTP.push rename to\t[#{target.name}]"
|
127
|
-
@sftp.rename! destination.path, target.path, flags
|
124
|
+
# Move the file back to its original name
|
125
|
+
if use_temp_name
|
126
|
+
flags = 0x00000001
|
127
|
+
log_debug "RemoteSFTP.upload rename [#{dest.name}] > [#{target.name}]"
|
128
|
+
@sftp.rename! dest.path, target.path, flags
|
128
129
|
end
|
129
130
|
|
130
131
|
# progress:
|
@@ -27,13 +27,11 @@
|
|
27
27
|
|
28
28
|
%td= job_type job
|
29
29
|
|
30
|
-
%td
|
31
|
-
|
32
|
-
= token_highlight job.source
|
30
|
+
%td.text-right= location_label job.source_uri
|
31
|
+
%td= token_highlight job.source
|
33
32
|
|
34
|
-
%td
|
35
|
-
|
36
|
-
= token_highlight job.target
|
33
|
+
%td.text-right= location_label job.target_uri
|
34
|
+
%td= token_highlight job.target
|
37
35
|
|
38
36
|
%td= datetime_short(job.queued_at)
|
39
37
|
|
data/rest-ftp-daemon.gemspec
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rest-ftp-daemon
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.421.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Bruno MEDICI
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-09-
|
11
|
+
date: 2016-09-02 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|