rest-ftp-daemon 0.222.0 → 0.230.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/CODE_OF_CONDUCT.md +13 -0
- data/Gemfile.lock +47 -20
- data/README.md +160 -94
- data/Rakefile +7 -1
- data/bin/rest-ftp-daemon +22 -3
- data/lib/rest-ftp-daemon.rb +25 -21
- data/lib/rest-ftp-daemon/constants.rb +19 -5
- data/lib/rest-ftp-daemon/exceptions.rb +2 -1
- data/lib/rest-ftp-daemon/helpers.rb +10 -5
- data/lib/rest-ftp-daemon/job.rb +181 -304
- data/lib/rest-ftp-daemon/job_queue.rb +5 -3
- data/lib/rest-ftp-daemon/logger.rb +4 -3
- data/lib/rest-ftp-daemon/logger_helper.rb +14 -10
- data/lib/rest-ftp-daemon/notification.rb +54 -43
- data/lib/rest-ftp-daemon/paginate.rb +2 -2
- data/lib/rest-ftp-daemon/path.rb +43 -0
- data/lib/rest-ftp-daemon/remote.rb +57 -0
- data/lib/rest-ftp-daemon/remote_ftp.rb +141 -0
- data/lib/rest-ftp-daemon/remote_sftp.rb +160 -0
- data/lib/rest-ftp-daemon/uri.rb +11 -4
- data/lib/rest-ftp-daemon/views/dashboard_table.haml +1 -1
- data/lib/rest-ftp-daemon/views/dashboard_workers.haml +1 -1
- data/lib/rest-ftp-daemon/worker.rb +10 -2
- data/lib/rest-ftp-daemon/worker_conchita.rb +12 -6
- data/lib/rest-ftp-daemon/worker_job.rb +8 -11
- data/rest-ftp-daemon.gemspec +6 -1
- data/rest-ftp-daemon.yml.sample +4 -2
- data/spec/rest-ftp-daemon/features/dashboard_spec.rb +8 -4
- data/spec/rest-ftp-daemon/features/jobs_spec.rb +68 -0
- data/spec/rest-ftp-daemon/features/routes_spec.rb +20 -0
- data/spec/rest-ftp-daemon/features/status_spec.rb +19 -0
- data/spec/spec_helper.rb +6 -2
- data/spec/support/config.yml +0 -1
- data/spec/support/request_helpers.rb +22 -0
- metadata +53 -3
- data/.ruby-version +0 -1
@@ -0,0 +1,160 @@
|
|
1
|
+
module RestFtpDaemon
|
2
|
+
class RemoteSFTP < Remote
|
3
|
+
attr_reader :sftp
|
4
|
+
|
5
|
+
def initialize url, log_context, options = {}
|
6
|
+
# Call super
|
7
|
+
super
|
8
|
+
|
9
|
+
# Use debug ?
|
10
|
+
@debug = (Settings.at :debug, :sftp) == true
|
11
|
+
|
12
|
+
# Announce object
|
13
|
+
log_info "RemoteSFTP.initialize"
|
14
|
+
end
|
15
|
+
|
16
|
+
def connect
|
17
|
+
# Connect init
|
18
|
+
super
|
19
|
+
log_info "RemoteSFTP.connect [#{@url.user}]@[#{@url.host}]:[#{@url.port}]"
|
20
|
+
|
21
|
+
# Debug level
|
22
|
+
verbosity = @debug ? Logger::INFO : false
|
23
|
+
|
24
|
+
# Connect remote server
|
25
|
+
@sftp = Net::SFTP.start(@url.host, @url.user, password: @url.password, verbose: verbosity)
|
26
|
+
end
|
27
|
+
|
28
|
+
def present? target
|
29
|
+
log_info "RemoteSFTP.present? [#{target.name}]"
|
30
|
+
stat = @sftp.stat! target.full
|
31
|
+
size = "?"
|
32
|
+
|
33
|
+
rescue Net::SFTP::StatusException
|
34
|
+
return false
|
35
|
+
else
|
36
|
+
return stat.size
|
37
|
+
end
|
38
|
+
|
39
|
+
# def remove target
|
40
|
+
# log_info "RemoteSFTP.remove [#{target.name}]"
|
41
|
+
# @sftp.remove target.full
|
42
|
+
# end
|
43
|
+
|
44
|
+
def remove! target
|
45
|
+
log_info "RemoteSFTP.remove! [#{target.name}]"
|
46
|
+
@sftp.remove target.full
|
47
|
+
|
48
|
+
rescue Net::SFTP::StatusException
|
49
|
+
log_info "#{LOG_INDENT}[#{target.name}] file not found"
|
50
|
+
else
|
51
|
+
log_info "#{LOG_INDENT}[#{target.name}] removed"
|
52
|
+
end
|
53
|
+
|
54
|
+
def mkdir directory
|
55
|
+
log_info "RemoteSFTP.mkdir [#{directory}]"
|
56
|
+
@sftp.mkdir! directory
|
57
|
+
|
58
|
+
rescue
|
59
|
+
raise JobTargetPermissionError
|
60
|
+
end
|
61
|
+
|
62
|
+
def chdir_or_create directory, mkdir = false
|
63
|
+
# Init, extract my parent name and my own name
|
64
|
+
log_info "RemoteSFTP.chdir_or_create mkdir[#{mkdir}] dir[#{directory}]"
|
65
|
+
parent, current = Helpers.extract_parent(directory)
|
66
|
+
|
67
|
+
# Access this directory
|
68
|
+
begin
|
69
|
+
# log_info " chdir [/#{directory}]"
|
70
|
+
handle = @sftp.opendir! "./#{directory}"
|
71
|
+
|
72
|
+
rescue Net::SFTP::StatusException => e
|
73
|
+
# If not allowed to create path, that's over, we're stuck
|
74
|
+
return false unless mkdir
|
75
|
+
|
76
|
+
# Recurse upward
|
77
|
+
#log_info "#{LOG_INDENT}upward [#{parent}]"
|
78
|
+
chdir_or_create parent, mkdir
|
79
|
+
|
80
|
+
# Now I was able to chdir into my parent, create the current directory
|
81
|
+
#log_info "#{LOG_INDENT}mkdir [#{directory}]"
|
82
|
+
mkdir directory
|
83
|
+
|
84
|
+
# Finally retry the chdir
|
85
|
+
retry
|
86
|
+
else
|
87
|
+
return true
|
88
|
+
end
|
89
|
+
|
90
|
+
# We should never get here
|
91
|
+
raise JobTargetShouldBeDirectory
|
92
|
+
end
|
93
|
+
|
94
|
+
# def dir_contents directory
|
95
|
+
# # Access this directory
|
96
|
+
# handle = @sftp.opendir! directory
|
97
|
+
# @sftp.readdir! handle
|
98
|
+
# end
|
99
|
+
|
100
|
+
def push source, target, tempname = nil, &callback
|
101
|
+
# Push init
|
102
|
+
raise RestFtpDaemon::JobAssertionFailed, "push/1" if @sftp.nil?
|
103
|
+
|
104
|
+
# Temp file if provided
|
105
|
+
destination = target.clone
|
106
|
+
destination.name = tempname if tempname
|
107
|
+
|
108
|
+
# Do the transfer
|
109
|
+
log_info "RemoteSFTP.push [#{destination.full}]"
|
110
|
+
@sftp.upload! source.full, destination.full do |event, uploader, *args|
|
111
|
+
case event
|
112
|
+
when :open then
|
113
|
+
# args[0] : file metadata
|
114
|
+
when :put then
|
115
|
+
# args[0] : file metadata
|
116
|
+
# args[1] : byte offset in remote file
|
117
|
+
# args[2] : data being written (as string)
|
118
|
+
# puts "writing #{args[2].length} bytes to #{args[0].remote} starting at #{args[1]}"
|
119
|
+
|
120
|
+
# Update the worker activity marker
|
121
|
+
#FIXME worker_is_still_active
|
122
|
+
|
123
|
+
# Update job status after this block transfer
|
124
|
+
yield args[2].length, destination.name
|
125
|
+
|
126
|
+
when :close then
|
127
|
+
# args[0] : file metadata
|
128
|
+
when :mkdir
|
129
|
+
# args[0] : remote path name
|
130
|
+
when :finish
|
131
|
+
end
|
132
|
+
|
133
|
+
end
|
134
|
+
|
135
|
+
# flags = 0x0001 + 0x0002
|
136
|
+
flags = 0x00000001
|
137
|
+
|
138
|
+
# Rename if needed
|
139
|
+
if tempname
|
140
|
+
log_info "RemoteSFTP.push rename to\t[#{target.name}]"
|
141
|
+
@sftp.rename! destination.full, target.full, flags
|
142
|
+
end
|
143
|
+
|
144
|
+
# progress:
|
145
|
+
# Net::SFTP::StatusException
|
146
|
+
end
|
147
|
+
|
148
|
+
def close
|
149
|
+
# Close init
|
150
|
+
super
|
151
|
+
|
152
|
+
# @sftp.close
|
153
|
+
end
|
154
|
+
|
155
|
+
def connected?
|
156
|
+
@sftp && !@sftp.closed?
|
157
|
+
end
|
158
|
+
|
159
|
+
end
|
160
|
+
end
|
data/lib/rest-ftp-daemon/uri.rb
CHANGED
@@ -1,12 +1,19 @@
|
|
1
1
|
module URI
|
2
|
+
|
2
3
|
class FTPS < Generic
|
3
4
|
DEFAULT_PORT = 21
|
4
5
|
end
|
5
|
-
|
6
|
-
end
|
7
|
-
module URI
|
6
|
+
|
8
7
|
class FTPES < Generic
|
9
|
-
DEFAULT_PORT = 990
|
8
|
+
# DEFAULT_PORT = 990
|
9
|
+
DEFAULT_PORT = 21
|
10
|
+
end
|
11
|
+
|
12
|
+
class SFTP < Generic
|
13
|
+
DEFAULT_PORT = 22
|
10
14
|
end
|
15
|
+
|
16
|
+
@@schemes["FTPS"] = FTPS
|
11
17
|
@@schemes["FTPES"] = FTPES
|
18
|
+
@@schemes["SFTP"] = SFTP
|
12
19
|
end
|
@@ -6,7 +6,7 @@
|
|
6
6
|
- presented = present job, with: RestFtpDaemon::API::Entities::JobPresenter, hide_params: true
|
7
7
|
- bitrate = job.get :transfer_bitrate
|
8
8
|
|
9
|
-
- trclass =
|
9
|
+
- trclass = DASHBOARD_JOB_STYLES[job.status]
|
10
10
|
|
11
11
|
- unless job.error.nil?
|
12
12
|
- trclass = "warning"
|
@@ -10,6 +10,7 @@ module RestFtpDaemon
|
|
10
10
|
def initialize wid
|
11
11
|
# Logger
|
12
12
|
@logger = RestFtpDaemon::LoggerPool.instance.get :workers
|
13
|
+
@log_worker_status_changes = true
|
13
14
|
|
14
15
|
# Worker name
|
15
16
|
@wid = wid
|
@@ -17,7 +18,7 @@ module RestFtpDaemon
|
|
17
18
|
# Set thread context
|
18
19
|
Thread.current.thread_variable_set :wid, wid
|
19
20
|
Thread.current.thread_variable_set :started_at, Time.now
|
20
|
-
worker_status
|
21
|
+
worker_status WORKER_STATUS_STARTING
|
21
22
|
end
|
22
23
|
|
23
24
|
protected
|
@@ -40,9 +41,16 @@ module RestFtpDaemon
|
|
40
41
|
end
|
41
42
|
end
|
42
43
|
|
43
|
-
def worker_status status
|
44
|
+
def worker_status status, extra = ""
|
45
|
+
# Update thread variables
|
44
46
|
Thread.current.thread_variable_set :status, status
|
45
47
|
Thread.current.thread_variable_set :updted_at, Time.now
|
48
|
+
|
49
|
+
# Nothin' to log if "silent"
|
50
|
+
return unless @log_worker_status_changes
|
51
|
+
|
52
|
+
# Log this status change
|
53
|
+
log_info "worker: #{status} #{extra}"
|
46
54
|
end
|
47
55
|
|
48
56
|
def worker_jid jid
|
@@ -5,6 +5,10 @@ module RestFtpDaemon
|
|
5
5
|
# Generic worker initialize
|
6
6
|
super
|
7
7
|
|
8
|
+
# Use debug ?
|
9
|
+
@debug = (Settings.at :debug, :conchita) == true
|
10
|
+
@log_worker_status_changes = @debug
|
11
|
+
|
8
12
|
# Conchita configuration
|
9
13
|
@conchita = Settings.conchita
|
10
14
|
if !@conchita.is_a? Hash
|
@@ -21,23 +25,25 @@ module RestFtpDaemon
|
|
21
25
|
protected
|
22
26
|
|
23
27
|
def work
|
24
|
-
|
28
|
+
# Announce we are working
|
29
|
+
worker_status WORKER_STATUS_CLEANING
|
25
30
|
|
26
31
|
# Cleanup queues according to configured max-age
|
27
|
-
$queue.expire JOB_STATUS_FINISHED, maxage(JOB_STATUS_FINISHED)
|
28
|
-
$queue.expire JOB_STATUS_FAILED, maxage(JOB_STATUS_FAILED)
|
29
|
-
$queue.expire JOB_STATUS_QUEUED, maxage(JOB_STATUS_QUEUED)
|
32
|
+
$queue.expire JOB_STATUS_FINISHED, maxage(JOB_STATUS_FINISHED), @debug
|
33
|
+
$queue.expire JOB_STATUS_FAILED, maxage(JOB_STATUS_FAILED), @debug
|
34
|
+
$queue.expire JOB_STATUS_QUEUED, maxage(JOB_STATUS_QUEUED), @debug
|
30
35
|
|
31
36
|
# Force garbage collector
|
32
|
-
worker_status :collecting
|
33
37
|
GC.start if @conchita["garbage_collector"]
|
34
38
|
|
35
39
|
rescue StandardError => e
|
36
40
|
log_error "EXCEPTION: #{e.inspect}"
|
37
41
|
sleep 1
|
38
42
|
else
|
43
|
+
# Restore previous status
|
44
|
+
worker_status WORKER_STATUS_WAITING
|
45
|
+
|
39
46
|
# Sleep for a few seconds
|
40
|
-
worker_status :sleeping
|
41
47
|
sleep @conchita[:timer]
|
42
48
|
end
|
43
49
|
|
@@ -17,14 +17,13 @@ module RestFtpDaemon
|
|
17
17
|
|
18
18
|
def work
|
19
19
|
# Wait for a job to come into the queue
|
20
|
-
worker_status
|
21
|
-
log_info "waiting
|
20
|
+
worker_status WORKER_STATUS_WAITING
|
21
|
+
#log_info "waiting"
|
22
22
|
job = $queue.pop
|
23
23
|
|
24
24
|
# Prepare the job for processing
|
25
|
-
worker_status
|
25
|
+
worker_status WORKER_STATUS_RUNNING, "job [#{job.id}]"
|
26
26
|
worker_jid job.id
|
27
|
-
log_info "working with job [#{job.id}]"
|
28
27
|
job.wid = Thread.current.thread_variable_get :wid
|
29
28
|
|
30
29
|
# Processs this job protected by a timeout
|
@@ -33,8 +32,7 @@ module RestFtpDaemon
|
|
33
32
|
end
|
34
33
|
|
35
34
|
# Processing done
|
36
|
-
worker_status
|
37
|
-
log_info "finished with job [#{job.id}]"
|
35
|
+
worker_status WORKER_STATUS_FINISHED, "job [#{job.id}]"
|
38
36
|
worker_jid nil
|
39
37
|
job.wid = nil
|
40
38
|
|
@@ -42,8 +40,8 @@ module RestFtpDaemon
|
|
42
40
|
$queue.counter_inc :jobs_processed
|
43
41
|
|
44
42
|
rescue RestFtpDaemon::JobTimeout => ex
|
45
|
-
log_error "JOB TIMED OUT",
|
46
|
-
worker_status
|
43
|
+
log_error "JOB TIMED OUT", ex.backtrace
|
44
|
+
worker_status WORKER_STATUS_TIMEOUT
|
47
45
|
worker_jid nil
|
48
46
|
job.wid = nil
|
49
47
|
|
@@ -51,14 +49,13 @@ module RestFtpDaemon
|
|
51
49
|
sleep 1
|
52
50
|
|
53
51
|
rescue StandardError => ex
|
54
|
-
log_error "JOB UNHDNALED EXCEPTION: #{ex.message}",
|
55
|
-
worker_status
|
52
|
+
log_error "JOB UNHDNALED EXCEPTION: #{ex.message}", ex.backtrace
|
53
|
+
worker_status WORKER_STATUS_CRASHED
|
56
54
|
job.oops_after_crash ex unless job.nil?
|
57
55
|
sleep 1
|
58
56
|
|
59
57
|
else
|
60
58
|
# Clean job status
|
61
|
-
worker_status :free
|
62
59
|
job.wid = nil
|
63
60
|
|
64
61
|
end
|
data/rest-ftp-daemon.gemspec
CHANGED
@@ -16,7 +16,9 @@ Gem::Specification.new do |spec|
|
|
16
16
|
spec.homepage = "http://github.com/bmedici/rest-ftp-daemon"
|
17
17
|
spec.licenses = ["MIT"]
|
18
18
|
|
19
|
-
spec.files = `git ls-files -z`.split("\x0")
|
19
|
+
spec.files = `git ls-files -z`.split("\x0").reject do |f|
|
20
|
+
f == 'dashboard.png'
|
21
|
+
end
|
20
22
|
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
21
23
|
spec.require_paths = ["lib"]
|
22
24
|
spec.version = APP_VER
|
@@ -28,6 +30,8 @@ Gem::Specification.new do |spec|
|
|
28
30
|
spec.add_development_dependency "rake"
|
29
31
|
spec.add_development_dependency "rspec", "~> 3.1"
|
30
32
|
spec.add_development_dependency "http", "~> 0.8"
|
33
|
+
spec.add_development_dependency "rubocop", "~> 0.32.0"
|
34
|
+
spec.add_development_dependency "pry"
|
31
35
|
|
32
36
|
spec.add_runtime_dependency "thin", "~> 1.6"
|
33
37
|
spec.add_runtime_dependency "grape"
|
@@ -35,6 +39,7 @@ Gem::Specification.new do |spec|
|
|
35
39
|
spec.add_runtime_dependency "settingslogic"
|
36
40
|
spec.add_runtime_dependency "haml"
|
37
41
|
spec.add_runtime_dependency "json"
|
42
|
+
spec.add_runtime_dependency "net-sftp"
|
38
43
|
spec.add_runtime_dependency "double-bag-ftps"
|
39
44
|
spec.add_runtime_dependency "facter"
|
40
45
|
spec.add_runtime_dependency "sys-cpu"
|
data/rest-ftp-daemon.yml.sample
CHANGED
@@ -4,10 +4,10 @@ defaults: &defaults
|
|
4
4
|
workers: 2
|
5
5
|
user: rftpd
|
6
6
|
group: rftpd
|
7
|
-
host: "myhost"
|
7
|
+
#host: "myhost"
|
8
|
+
#appname: "replay" # appname prefix used for NewRelic reporting
|
8
9
|
|
9
10
|
transfer:
|
10
|
-
update_every_kb: 1024 # block size to transfer between counter updates
|
11
11
|
notify_after_sec: 5 # wait at least X seconds between HTTP notifications
|
12
12
|
mkdir: true # build directory tree if missing
|
13
13
|
tempfile: true # transfer to temporary file, rename after sucessful transfer
|
@@ -23,6 +23,8 @@ defaults: &defaults
|
|
23
23
|
|
24
24
|
debug:
|
25
25
|
ftp: false
|
26
|
+
sftp: false
|
27
|
+
conchita: false
|
26
28
|
# newrelic: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
|
27
29
|
|
28
30
|
logs:
|
@@ -5,19 +5,23 @@ describe 'Dashboard', feature: true do
|
|
5
5
|
describe "GET /" do
|
6
6
|
context 'without a password' do
|
7
7
|
it 'is forbidden' do
|
8
|
-
expect(HTTP.accept(:json).get("http://localhost
|
8
|
+
expect(HTTP.accept(:json).get("http://localhost:#{RequestHelpers::PORT}").status).to eq 401
|
9
9
|
end
|
10
10
|
end
|
11
11
|
|
12
12
|
context 'with a password' do
|
13
13
|
it 'can be accessed' do
|
14
14
|
expect(
|
15
|
-
|
16
|
-
basic_auth(user: 'admin', pass: 'admin').
|
17
|
-
get("http://localhost:5678").status
|
15
|
+
get("/").status
|
18
16
|
).to eq 200
|
19
17
|
end
|
20
18
|
end
|
19
|
+
|
20
|
+
it "has an HTML representation" do
|
21
|
+
expect(
|
22
|
+
get("/", accept: 'html').status
|
23
|
+
).to eq 200
|
24
|
+
end
|
21
25
|
end # GET /
|
22
26
|
|
23
27
|
end
|
@@ -0,0 +1,68 @@
|
|
1
|
+
require "spec_helper"
|
2
|
+
|
3
|
+
describe "Jobs", feature: true do
|
4
|
+
|
5
|
+
describe "GET /jobs" do
|
6
|
+
let!(:response) { get "/jobs" }
|
7
|
+
|
8
|
+
it "responds successfully" do
|
9
|
+
expect(response.status).to eq 200
|
10
|
+
end
|
11
|
+
|
12
|
+
it "exposes an array" do
|
13
|
+
expect(JSON.parse(response.body)).to be_an_instance_of(Array)
|
14
|
+
end
|
15
|
+
end # GET /jobs
|
16
|
+
|
17
|
+
describe "POST /jobs" do
|
18
|
+
def jobs_list
|
19
|
+
JSON.parse get("/jobs").body
|
20
|
+
end
|
21
|
+
|
22
|
+
context "when params are valid" do
|
23
|
+
let(:params) do
|
24
|
+
{
|
25
|
+
source: "/tmp/foo",
|
26
|
+
target: "/tmp/bar"
|
27
|
+
}
|
28
|
+
end
|
29
|
+
|
30
|
+
it "issues a 201 response" do
|
31
|
+
expect(
|
32
|
+
post("/jobs", json: params).status
|
33
|
+
).to eq 201
|
34
|
+
end
|
35
|
+
|
36
|
+
it "exposes the new job id" do
|
37
|
+
response = JSON.parse post("/jobs", json: params)
|
38
|
+
expect(response['id']).not_to be_nil
|
39
|
+
end
|
40
|
+
|
41
|
+
it "assigns a status" do
|
42
|
+
response = JSON.parse post("/jobs", json: params)
|
43
|
+
expect(response['status']).to match(/^(queued|failed)$/)
|
44
|
+
end
|
45
|
+
|
46
|
+
it "creates a new job" do
|
47
|
+
expect {
|
48
|
+
post("/jobs", json: params)
|
49
|
+
}.to change { jobs_list.size }.by(1)
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end # POST /jobs
|
53
|
+
|
54
|
+
describe "GET /jobs/:id" do
|
55
|
+
let(:creation_response) do
|
56
|
+
JSON.parse post("/jobs", json: {source: "/tmp/foo", target: "/tmp/bar"}).body
|
57
|
+
end
|
58
|
+
|
59
|
+
let(:job_id) { creation_response.fetch('id') }
|
60
|
+
|
61
|
+
it "is properly exposed" do
|
62
|
+
expect(
|
63
|
+
get("/jobs/#{job_id}").status
|
64
|
+
).to eq 200
|
65
|
+
end
|
66
|
+
end # GET /jobs/:id
|
67
|
+
|
68
|
+
end
|