rest-ftp-daemon 0.222.0 → 0.230.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/CODE_OF_CONDUCT.md +13 -0
- data/Gemfile.lock +47 -20
- data/README.md +160 -94
- data/Rakefile +7 -1
- data/bin/rest-ftp-daemon +22 -3
- data/lib/rest-ftp-daemon.rb +25 -21
- data/lib/rest-ftp-daemon/constants.rb +19 -5
- data/lib/rest-ftp-daemon/exceptions.rb +2 -1
- data/lib/rest-ftp-daemon/helpers.rb +10 -5
- data/lib/rest-ftp-daemon/job.rb +181 -304
- data/lib/rest-ftp-daemon/job_queue.rb +5 -3
- data/lib/rest-ftp-daemon/logger.rb +4 -3
- data/lib/rest-ftp-daemon/logger_helper.rb +14 -10
- data/lib/rest-ftp-daemon/notification.rb +54 -43
- data/lib/rest-ftp-daemon/paginate.rb +2 -2
- data/lib/rest-ftp-daemon/path.rb +43 -0
- data/lib/rest-ftp-daemon/remote.rb +57 -0
- data/lib/rest-ftp-daemon/remote_ftp.rb +141 -0
- data/lib/rest-ftp-daemon/remote_sftp.rb +160 -0
- data/lib/rest-ftp-daemon/uri.rb +11 -4
- data/lib/rest-ftp-daemon/views/dashboard_table.haml +1 -1
- data/lib/rest-ftp-daemon/views/dashboard_workers.haml +1 -1
- data/lib/rest-ftp-daemon/worker.rb +10 -2
- data/lib/rest-ftp-daemon/worker_conchita.rb +12 -6
- data/lib/rest-ftp-daemon/worker_job.rb +8 -11
- data/rest-ftp-daemon.gemspec +6 -1
- data/rest-ftp-daemon.yml.sample +4 -2
- data/spec/rest-ftp-daemon/features/dashboard_spec.rb +8 -4
- data/spec/rest-ftp-daemon/features/jobs_spec.rb +68 -0
- data/spec/rest-ftp-daemon/features/routes_spec.rb +20 -0
- data/spec/rest-ftp-daemon/features/status_spec.rb +19 -0
- data/spec/spec_helper.rb +6 -2
- data/spec/support/config.yml +0 -1
- data/spec/support/request_helpers.rb +22 -0
- metadata +53 -3
- data/.ruby-version +0 -1
@@ -165,14 +165,16 @@ module RestFtpDaemon
|
|
165
165
|
@waiting.size
|
166
166
|
end
|
167
167
|
|
168
|
-
def expire status, maxage
|
168
|
+
def expire status, maxage, verbose = false
|
169
169
|
# FIXME: clean both @jobs and @queue
|
170
170
|
# Init
|
171
171
|
return if status.nil? || maxage <= 0
|
172
172
|
|
173
173
|
# Compute oldest possible birthday
|
174
174
|
before = Time.now - maxage.to_i
|
175
|
-
|
175
|
+
|
176
|
+
# Verbose output ?
|
177
|
+
log_info "JobQueue.expire \t[#{status.to_s}] \tbefore \t[#{before}]" if verbose
|
176
178
|
|
177
179
|
@mutex.synchronize do
|
178
180
|
# Delete jobs from the queue when they match status and age limits
|
@@ -185,7 +187,7 @@ module RestFtpDaemon
|
|
185
187
|
|
186
188
|
# Ok, we have to clean it up ..
|
187
189
|
log_info "expire [#{status.to_s}] [#{maxage}] > [#{job.id}] [#{job.updated_at}]"
|
188
|
-
log_info "
|
190
|
+
log_info "#{LOG_INDENT}unqueued" if @queue.delete(job)
|
189
191
|
|
190
192
|
true
|
191
193
|
end
|
@@ -22,9 +22,9 @@ class Logger
|
|
22
22
|
|
23
23
|
end
|
24
24
|
|
25
|
-
|
26
25
|
# Prepend plain message to output
|
27
|
-
output.unshift (prefix1 + message.strip)
|
26
|
+
#output.unshift (prefix1 + message.strip)
|
27
|
+
output.unshift (prefix1 + message)
|
28
28
|
|
29
29
|
# Send all this to logger
|
30
30
|
add context[:level], output
|
@@ -43,7 +43,8 @@ class Logger
|
|
43
43
|
|
44
44
|
def build_from_array prefix, lines
|
45
45
|
lines.map do |value|
|
46
|
-
text = value.to_s.strip[0..LOG_TRIM_LINE]
|
46
|
+
#text = value.to_s.strip[0..LOG_TRIM_LINE]
|
47
|
+
text = value.to_s[0..LOG_TRIM_LINE]
|
47
48
|
"#{prefix}#{text}"
|
48
49
|
end
|
49
50
|
end
|
@@ -4,23 +4,27 @@ module RestFtpDaemon
|
|
4
4
|
protected
|
5
5
|
|
6
6
|
def log_info message, lines = []
|
7
|
-
|
8
|
-
from: self.class.to_s,
|
9
|
-
lines: lines,
|
10
|
-
level: Logger::INFO
|
11
|
-
})
|
7
|
+
log message, lines, Logger::INFO
|
12
8
|
end
|
13
9
|
|
14
10
|
def log_error message, lines = []
|
15
|
-
|
16
|
-
from: self.class.to_s,
|
17
|
-
lines: lines,
|
18
|
-
level: Logger::ERROR
|
19
|
-
})
|
11
|
+
log message, lines, Logger::ERROR
|
20
12
|
end
|
21
13
|
|
22
14
|
def log_context
|
23
15
|
{}
|
24
16
|
end
|
17
|
+
|
18
|
+
private
|
19
|
+
|
20
|
+
def log message, lines, level
|
21
|
+
context = log_context || {}
|
22
|
+
logger.info_with_id message, context.merge({
|
23
|
+
from: self.class.to_s,
|
24
|
+
lines: lines,
|
25
|
+
level: level
|
26
|
+
})
|
27
|
+
end
|
28
|
+
|
25
29
|
end
|
26
30
|
end
|
@@ -12,6 +12,10 @@ module RestFtpDaemon
|
|
12
12
|
attr_accessor :job
|
13
13
|
|
14
14
|
def initialize url, params
|
15
|
+
# Remember params
|
16
|
+
@url = url
|
17
|
+
@params = params
|
18
|
+
|
15
19
|
# Generate a random key
|
16
20
|
@id = Helpers.identifier(NOTIFY_IDENTIFIER_LEN)
|
17
21
|
@jid = nil
|
@@ -19,64 +23,71 @@ module RestFtpDaemon
|
|
19
23
|
# Logger
|
20
24
|
@logger = RestFtpDaemon::LoggerPool.instance.get :notify
|
21
25
|
|
26
|
+
# Handle the notification
|
27
|
+
log_info "initialized [#{@url}]"
|
28
|
+
process
|
29
|
+
end
|
30
|
+
|
31
|
+
protected
|
32
|
+
|
33
|
+
def process
|
22
34
|
# Check context
|
23
|
-
if url.nil?
|
24
|
-
log_info "skipping (missing url): #{params.inspect}"
|
35
|
+
if @url.nil?
|
36
|
+
log_info "skipping (missing url): #{@params.inspect}"
|
25
37
|
return
|
26
|
-
elsif params[:event].nil?
|
27
|
-
log_info "skipping (missing event): #{params.inspect}"
|
38
|
+
elsif @params[:event].nil?
|
39
|
+
log_info "skipping (missing event): #{@params.inspect}"
|
28
40
|
return
|
29
41
|
end
|
30
42
|
|
31
43
|
# Build body and extract job ID if provided
|
32
|
-
|
33
|
-
id: params[:id].to_s,
|
34
|
-
signal: "#{NOTIFY_PREFIX}.#{params[:event]}",
|
35
|
-
error: params[:error],
|
44
|
+
flags = {
|
45
|
+
id: @params[:id].to_s,
|
46
|
+
signal: "#{NOTIFY_PREFIX}.#{@params[:event]}",
|
47
|
+
error: @params[:error],
|
36
48
|
host: Settings.host.to_s,
|
37
49
|
}
|
38
|
-
|
39
|
-
|
40
|
-
@jid = params[:id]
|
41
|
-
log_info "initialized"
|
50
|
+
flags[:status] = @params[:status] if @params[:status].is_a? Enumerable
|
51
|
+
flags[:message] = @params[:message].to_s unless @params[:message].nil?
|
52
|
+
@jid = @params[:id]
|
42
53
|
|
43
|
-
|
44
|
-
# Send message in a thread
|
54
|
+
# Spawn a dedicated thread
|
45
55
|
Thread.new do
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
"Content-Type" => "application/json",
|
50
|
-
"Accept" => "application/json",
|
51
|
-
"User-Agent" => "#{APP_NAME} - #{APP_VER}"
|
52
|
-
}
|
53
|
-
data = body.to_json
|
54
|
-
log_info "sending #{data}"
|
55
|
-
|
56
|
-
|
57
|
-
# Prepare HTTP client
|
58
|
-
http = Net::HTTP.new uri.host, uri.port
|
59
|
-
# http.initialize_http_header({'User-Agent' => APP_NAME})
|
60
|
-
|
61
|
-
# Post notification
|
62
|
-
response = http.post uri.path, data, headers
|
63
|
-
|
64
|
-
# Handle server response / multi-lines
|
65
|
-
response_lines = response.body.lines
|
66
|
-
|
67
|
-
if response_lines.size > 1
|
68
|
-
human_size = Helpers.format_bytes(response.body.bytesize, "B")
|
69
|
-
#human_size = 0
|
70
|
-
log_info "received [#{response.code}] #{human_size} (#{response_lines.size} lines)", response_lines
|
71
|
-
else
|
72
|
-
log_info "received [#{response.code}] #{response.body.strip}"
|
73
|
-
end
|
56
|
+
send flags
|
57
|
+
end # end Thread
|
58
|
+
end
|
74
59
|
|
60
|
+
def send flags
|
61
|
+
# Prepare query
|
62
|
+
headers = {
|
63
|
+
"Content-Type" => "application/json",
|
64
|
+
"Accept" => "application/json",
|
65
|
+
"User-Agent" => "#{APP_NAME} - #{APP_VER}"
|
66
|
+
}
|
67
|
+
data = flags.to_json
|
68
|
+
|
69
|
+
# Send notification through HTTP
|
70
|
+
uri = URI @url
|
71
|
+
http = Net::HTTP.new uri.host, uri.port
|
72
|
+
|
73
|
+
# Post notification, handle server response / multi-lines
|
74
|
+
log_info "sending #{data}"
|
75
|
+
response = http.post uri.path, data, headers
|
76
|
+
response_lines = response.body.lines
|
77
|
+
|
78
|
+
if response_lines.size > 1
|
79
|
+
human_size = Helpers.format_bytes(response.body.bytesize, "B")
|
80
|
+
log_info "received [#{response.code}] #{human_size} (#{response_lines.size} lines)", response_lines
|
81
|
+
else
|
82
|
+
log_info "received [#{response.code}] #{response.body.strip}"
|
75
83
|
end
|
76
84
|
|
85
|
+
# Handle exceptions
|
86
|
+
rescue StandardError => ex
|
87
|
+
log_error "EXCEPTION: #{ex.inspect}"
|
88
|
+
|
77
89
|
end
|
78
90
|
|
79
|
-
protected
|
80
91
|
|
81
92
|
def log_context
|
82
93
|
{
|
@@ -18,7 +18,7 @@ module RestFtpDaemon
|
|
18
18
|
@total = @data.count
|
19
19
|
|
20
20
|
# Count pages
|
21
|
-
@pages = (@total.to_f /
|
21
|
+
@pages = (@total.to_f / DEFAULT_PAGE_SIZE).ceil
|
22
22
|
@pages = 1 if @pages < 1
|
23
23
|
end
|
24
24
|
|
@@ -39,7 +39,7 @@ module RestFtpDaemon
|
|
39
39
|
end
|
40
40
|
|
41
41
|
def subset
|
42
|
-
size =
|
42
|
+
size = DEFAULT_PAGE_SIZE.to_i
|
43
43
|
offset = (@page-1) * size
|
44
44
|
@data[offset, size]
|
45
45
|
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
module RestFtpDaemon
|
2
|
+
class Path
|
3
|
+
attr_accessor :name
|
4
|
+
attr_accessor :dir
|
5
|
+
|
6
|
+
def initialize full, strip_leading_slash = false
|
7
|
+
# Extract path parts
|
8
|
+
@name = extract_filename full.to_s
|
9
|
+
@dir = extract_dirname full.to_s
|
10
|
+
|
11
|
+
# Remove leading slash if needed
|
12
|
+
strip_leading_slash_from_dir! if strip_leading_slash
|
13
|
+
end
|
14
|
+
|
15
|
+
def full
|
16
|
+
return @name if @dir.nil? || @dir.empty?
|
17
|
+
return File.join @dir, @name
|
18
|
+
end
|
19
|
+
|
20
|
+
def size
|
21
|
+
File.size full if File.exists? full
|
22
|
+
end
|
23
|
+
|
24
|
+
private
|
25
|
+
|
26
|
+
def extract_filename path
|
27
|
+
# match everything that's after a slash at the end of the string
|
28
|
+
m = path.match /\/?([^\/]+)$/
|
29
|
+
return m[1].to_s unless m.nil?
|
30
|
+
end
|
31
|
+
|
32
|
+
def extract_dirname path
|
33
|
+
# match all the beginning of the string up to the last slash
|
34
|
+
m = path.match(/^(.*)\/[^\/]*$/)
|
35
|
+
return m[1].to_s unless m.nil?
|
36
|
+
end
|
37
|
+
|
38
|
+
def strip_leading_slash_from_dir!
|
39
|
+
@dir.to_s.gsub!(/^\//, '')
|
40
|
+
end
|
41
|
+
|
42
|
+
end
|
43
|
+
end
|
@@ -0,0 +1,57 @@
|
|
1
|
+
module RestFtpDaemon
|
2
|
+
class Remote
|
3
|
+
include LoggerHelper
|
4
|
+
attr_reader :logger
|
5
|
+
attr_reader :log_context
|
6
|
+
|
7
|
+
def initialize url, log_context, options = {}
|
8
|
+
# Logger
|
9
|
+
@log_context = log_context || {}
|
10
|
+
@logger = RestFtpDaemon::LoggerPool.instance.get :jobs
|
11
|
+
|
12
|
+
# Extract URL parts
|
13
|
+
@url = url
|
14
|
+
@url.user ||= "anonymous"
|
15
|
+
|
16
|
+
# Annnounce object
|
17
|
+
log_info "Remote.initialize [#{url.to_s}]"
|
18
|
+
end
|
19
|
+
|
20
|
+
def connect
|
21
|
+
# Debug mode ?
|
22
|
+
debug_header if @debug
|
23
|
+
end
|
24
|
+
|
25
|
+
def close
|
26
|
+
# Debug mode ?
|
27
|
+
puts "-------------------- SESSION CLOSING --------------------------" if @debug
|
28
|
+
end
|
29
|
+
|
30
|
+
private
|
31
|
+
|
32
|
+
# def log_context
|
33
|
+
# @log_context
|
34
|
+
# end
|
35
|
+
|
36
|
+
def myname
|
37
|
+
self.class.to_s
|
38
|
+
end
|
39
|
+
|
40
|
+
def debug_header
|
41
|
+
# Output header to STDOUT
|
42
|
+
puts
|
43
|
+
puts "-------------------- SESSION STARTING -------------------------"
|
44
|
+
#puts "job id\t #{@id}"
|
45
|
+
#puts "source\t #{@source}"
|
46
|
+
#puts "target\t #{@target}"
|
47
|
+
puts "class\t #{myname}"
|
48
|
+
#puts "class\t #{myname}"
|
49
|
+
puts "host\t #{@url.host}"
|
50
|
+
puts "user\t #{@url.user}"
|
51
|
+
puts "port\t #{@url.port}"
|
52
|
+
puts "options\t #{@options.inspect}"
|
53
|
+
puts "---------------------------------------------------------------"
|
54
|
+
end
|
55
|
+
|
56
|
+
end
|
57
|
+
end
|
@@ -0,0 +1,141 @@
|
|
1
|
+
module RestFtpDaemon
|
2
|
+
class RemoteFTP < Remote
|
3
|
+
attr_reader :ftp
|
4
|
+
|
5
|
+
def initialize url, log_context, options = {}
|
6
|
+
# Call super
|
7
|
+
super
|
8
|
+
|
9
|
+
# Use debug ?
|
10
|
+
@debug = (Settings.at :debug, :ftp) == true
|
11
|
+
|
12
|
+
# Create FTP object
|
13
|
+
if options[:ftpes]
|
14
|
+
prepare_ftpes
|
15
|
+
else
|
16
|
+
prepare_ftp
|
17
|
+
end
|
18
|
+
@ftp.passive = true
|
19
|
+
@ftp.debug_mode = !!@debug
|
20
|
+
|
21
|
+
# Config
|
22
|
+
@chunk_size = DEFAULT_FTP_CHUNK.to_i * 1024
|
23
|
+
|
24
|
+
# Announce object
|
25
|
+
log_info "RemoteFTP.initialize chunk_size:#{@chunk_size}"
|
26
|
+
end
|
27
|
+
|
28
|
+
def connect
|
29
|
+
# Connect init
|
30
|
+
super
|
31
|
+
|
32
|
+
# Connect remote server
|
33
|
+
@ftp.connect @url.host, @url.port
|
34
|
+
@ftp.login @url.user, @url.password
|
35
|
+
end
|
36
|
+
|
37
|
+
def present? target
|
38
|
+
size = @ftp.size target.full
|
39
|
+
log_info "RemoteFTP.present? [#{target.name}]"
|
40
|
+
|
41
|
+
rescue Net::FTPPermError
|
42
|
+
# log_info "RemoteFTP.present? [#{target.name}] NOT_FOUND"
|
43
|
+
return false
|
44
|
+
else
|
45
|
+
return size
|
46
|
+
end
|
47
|
+
|
48
|
+
def remove! target
|
49
|
+
log_info "RemoteFTP.remove! [#{target.name}]"
|
50
|
+
@ftp.delete target.full
|
51
|
+
rescue Net::FTPPermError
|
52
|
+
log_info "#{LOG_INDENT}[#{target.name}] file not found"
|
53
|
+
else
|
54
|
+
log_info "#{LOG_INDENT}[#{target.name}] removed"
|
55
|
+
end
|
56
|
+
|
57
|
+
def mkdir directory
|
58
|
+
log_info "RemoteFTP.mkdir [#{directory}]"
|
59
|
+
@ftp.mkdir directory
|
60
|
+
|
61
|
+
rescue
|
62
|
+
raise JobTargetPermissionError
|
63
|
+
end
|
64
|
+
|
65
|
+
def chdir_or_create directory, mkdir = false
|
66
|
+
# Init, extract my parent name and my own name
|
67
|
+
log_info "RemoteFTP.chdir_or_create mkdir[#{mkdir}] dir[#{directory}]"
|
68
|
+
parent, current = Helpers.extract_parent(directory)
|
69
|
+
|
70
|
+
fulldir = "/#{directory}"
|
71
|
+
|
72
|
+
# Access this directory
|
73
|
+
begin
|
74
|
+
@ftp.chdir "/#{directory}"
|
75
|
+
|
76
|
+
rescue Net::FTPPermError => e
|
77
|
+
# If not allowed to create path, that's over, we're stuck
|
78
|
+
return false unless mkdir
|
79
|
+
|
80
|
+
#log_info "#{LOG_INDENT}upward [#{parent}]"
|
81
|
+
chdir_or_create parent, mkdir
|
82
|
+
|
83
|
+
# Now I was able to chdir into my parent, create the current directory
|
84
|
+
#log_info "#{LOG_INDENT}mkdir [#{directory}]"
|
85
|
+
mkdir "/#{directory}"
|
86
|
+
|
87
|
+
# Finally retry the chdir
|
88
|
+
retry
|
89
|
+
else
|
90
|
+
return true
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
94
|
+
def push source, target, tempname = nil, &callback
|
95
|
+
# Push init
|
96
|
+
raise RestFtpDaemon::JobAssertionFailed, "push/1" if @ftp.nil?
|
97
|
+
|
98
|
+
# Temp file if provided
|
99
|
+
destination = target.clone
|
100
|
+
destination.name = tempname if tempname
|
101
|
+
|
102
|
+
# Do the transfer
|
103
|
+
log_info "RemoteFTP.push to [#{destination.name}]"
|
104
|
+
|
105
|
+
@ftp.putbinaryfile source.full, target.name, @chunk_size do |data|
|
106
|
+
# Update the worker activity marker
|
107
|
+
#FIXME worker_is_still_active
|
108
|
+
|
109
|
+
# Update job status after this block transfer
|
110
|
+
yield data.bytesize, destination.name
|
111
|
+
end
|
112
|
+
|
113
|
+
end
|
114
|
+
|
115
|
+
def close
|
116
|
+
# Close init
|
117
|
+
super
|
118
|
+
|
119
|
+
# Close FTP connexion and free up memory
|
120
|
+
@ftp.close
|
121
|
+
end
|
122
|
+
|
123
|
+
def connected?
|
124
|
+
!@ftp.welcome.nil?
|
125
|
+
end
|
126
|
+
|
127
|
+
private
|
128
|
+
|
129
|
+
def prepare_ftp
|
130
|
+
@ftp = Net::FTP.new
|
131
|
+
end
|
132
|
+
|
133
|
+
def prepare_ftpes
|
134
|
+
@ftp = DoubleBagFTPS.new
|
135
|
+
@ftp.ssl_context = DoubleBagFTPS.create_ssl_context(verify_mode: OpenSSL::SSL::VERIFY_NONE)
|
136
|
+
@ftp.ftps_mode = DoubleBagFTPS::EXPLICIT
|
137
|
+
end
|
138
|
+
|
139
|
+
|
140
|
+
end
|
141
|
+
end
|