rest-ftp-daemon 0.85.2 → 0.90.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/Gemfile.lock +1 -1
- data/README.md +16 -0
- data/bin/rest-ftp-daemon +52 -35
- data/lib/rest-ftp-daemon/api/jobs.rb +20 -13
- data/lib/rest-ftp-daemon/api/root.rb +12 -21
- data/lib/rest-ftp-daemon/constants.rb +3 -3
- data/lib/rest-ftp-daemon/exceptions.rb +2 -0
- data/lib/rest-ftp-daemon/helpers.rb +26 -2
- data/lib/rest-ftp-daemon/job.rb +128 -64
- data/lib/rest-ftp-daemon/job_queue.rb +59 -19
- data/lib/rest-ftp-daemon/notification.rb +4 -7
- data/lib/rest-ftp-daemon/static/css/bootstrap.css +2 -2
- data/lib/rest-ftp-daemon/static/css/main.css +17 -0
- data/lib/rest-ftp-daemon/views/dashboard.haml +43 -4
- data/lib/rest-ftp-daemon/views/dashboard_jobs.haml +55 -74
- data/lib/rest-ftp-daemon/views/dashboard_workers.haml +1 -1
- data/lib/rest-ftp-daemon/worker_pool.rb +1 -1
- data/rest-ftp-daemon.yml.sample +4 -3
- metadata +3 -4
- data/test/helper.rb +0 -34
- data/test/test_rest-ftp-daemon.rb +0 -7
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e1f8cd7be9c2f0d1932b820c3cf38b7dc79de537
|
4
|
+
data.tar.gz: e001c9fb6c9299f3d1f49b15368fac6628fb0fd9
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 0f37294ceee1337337cbc4013fd1f8fa08438d5b6ea11fc698cfa1c59442864f4b5cbe7444deedb265e549a5c9a0c32b2e7b4e845c61c1a659656206369f2045
|
7
|
+
data.tar.gz: 28c38f2a5e8618abf1f24773b91381f4623843b9629a190c967641c4980ebc78b9306d6ef26b4bee2da65b6ce616f40f043f5756892b06356b01c9589e15a4f5
|
data/.gitignore
CHANGED
data/Gemfile.lock
CHANGED
data/README.md
CHANGED
@@ -29,6 +29,8 @@ As of today, its main features are :
|
|
29
29
|
* Automatically clean-up jobs after a configurable amount of time (failed, finished)
|
30
30
|
* Current bitrate on the last blocks chunk updated in the job attributes
|
31
31
|
* Global bitrate on the whole file transfer is re-computed after the transfer finishes
|
32
|
+
* Daemon process is tagged with its name and environement in process lists
|
33
|
+
* Allow basic patterns in source filename to match multiple files (/dir/file*.jpg)
|
32
34
|
|
33
35
|
Expected features in a short-time range :
|
34
36
|
|
@@ -41,6 +43,9 @@ Expected features in a short-time range :
|
|
41
43
|
* Allow to specify random remote/local source/target
|
42
44
|
* Allow more transfer protocols (sFTP, HTTP POST etc)
|
43
45
|
|
46
|
+
Known bugs :
|
47
|
+
|
48
|
+
* As this project is based on SettingsLogic, which in turns uses Syck YAML parser, configuration merge from "defaults" section and environment-specific section is broken. A sub-tree defined for a specific environment, will overwrite the corresponding subtree from "defaults".
|
44
49
|
|
45
50
|
|
46
51
|
Installation
|
@@ -168,6 +173,17 @@ curl -H "Content-Type: application/json" -X POST -D /dev/stdout -d \
|
|
168
173
|
'{"source":"~/file.dmg","priority":"3", target":"ftp://anonymous@localhost/incoming/dest4.dmg","notify":"http://requestb.in/1321axg1"}' "http://localhost:3000/jobs"
|
169
174
|
```
|
170
175
|
|
176
|
+
|
177
|
+
* Get info about a job with ID="q89j.1"
|
178
|
+
|
179
|
+
Both parameters ```q89j.1``` and ```1``` will be accepted as ID in the API. Requests below are equivalent:
|
180
|
+
|
181
|
+
```
|
182
|
+
GET http://localhost:3100/jobs/q89j.1
|
183
|
+
GET http://localhost:3100/jobs/1
|
184
|
+
```
|
185
|
+
|
186
|
+
|
171
187
|
About
|
172
188
|
------------------------------------------------------------------------------------
|
173
189
|
|
data/bin/rest-ftp-daemon
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
2
|
|
3
|
-
# Try to load libs
|
3
|
+
# Try to load libs and constants
|
4
|
+
app_root = File.dirname(__FILE__) + '/../'
|
4
5
|
begin
|
5
6
|
require "thin"
|
6
7
|
require 'optparse'
|
@@ -9,37 +10,24 @@ begin
|
|
9
10
|
rescue LoadError
|
10
11
|
raise "EXITING: some of basic libs were not found: thin, optparse, socket, timeout"
|
11
12
|
end
|
13
|
+
require File.expand_path("#{app_root}/lib/rest-ftp-daemon/constants")
|
12
14
|
puts
|
15
|
+
#puts "--- #{APP_NAME} #{APP_VER}"
|
13
16
|
|
14
|
-
# Detect
|
15
|
-
begin
|
16
|
-
OptionParser.new do |opts|
|
17
|
-
opts.on("-e", "--environment ENV") { |env| APP_ENV = env }
|
18
|
-
opts.on("", "--dev") { APP_ENV = "development" }
|
19
|
-
end.parse(ARGV)
|
20
|
-
rescue OptionParser::InvalidOption => e
|
21
|
-
end
|
22
|
-
|
23
|
-
# Load remaining libs and config subsystem
|
24
|
-
app_root = File.dirname(__FILE__) + '/../'
|
25
|
-
[:constants, :helpers, :config].each do |lib|
|
26
|
-
require File.expand_path("#{app_root}/lib/rest-ftp-daemon/#{lib.to_s}.rb")
|
27
|
-
end
|
28
|
-
|
29
|
-
# Setup options and import current ARGV
|
17
|
+
# Detect options from ARGV
|
30
18
|
options = {}
|
31
19
|
parser = OptionParser.new do |opts|
|
32
20
|
opts.banner = "Usage: #{File.basename $0} [options] start|stop|restart"
|
33
|
-
opts.on("-e", "--environment ENV"
|
34
|
-
opts.on("--dev"
|
35
|
-
opts.on("-p", "--port PORT", "use PORT
|
36
|
-
opts.on("-w", "--workers COUNT", "Use COUNT worker threads
|
37
|
-
opts.on("-d", "--daemonize", "Run daemonized in the background") { |bool| options["daemonize"] = bool
|
21
|
+
opts.on("-e", "--environment ENV") { |env| APP_ENV = env }
|
22
|
+
opts.on("", "--dev") { APP_ENV = "development" }
|
23
|
+
opts.on("-p", "--port PORT", "use PORT") { |port| options["port"] = port.to_i }
|
24
|
+
opts.on("-w", "--workers COUNT", "Use COUNT worker threads") { |count| options["workers"] = count.to_i }
|
25
|
+
opts.on("-d", "--daemonize", "Run daemonized in the background") { |bool| options["daemonize"] = bool }
|
38
26
|
opts.on("-P", "--pid FILE", "File to store PID") { |file| options["pidfile"] = file }
|
39
27
|
opts.on("-u", "--user NAME", "User to run daemon as (use with -g)") { |user| options["user"] = user }
|
40
28
|
opts.on("-g", "--group NAME", "Group to run daemon as (use with -u)"){ |group| options["group"] = group }
|
41
29
|
opts.on_tail("-h", "--help", "Show this message") { puts opts; exit }
|
42
|
-
opts.on_tail('-v', '--version', "Show version") { puts
|
30
|
+
opts.on_tail('-v', '--version', "Show version (#{APP_VER})") { puts APP_VER; exit }
|
43
31
|
end
|
44
32
|
begin
|
45
33
|
parser.order!(ARGV)
|
@@ -52,6 +40,14 @@ rescue OptionParser::InvalidOption => e
|
|
52
40
|
puts "EXITING: option parser: #{e.message}"
|
53
41
|
exit 1
|
54
42
|
end
|
43
|
+
|
44
|
+
# Load helpers and config
|
45
|
+
app_root = File.dirname(__FILE__) + '/../'
|
46
|
+
[:helpers, :config].each do |lib|
|
47
|
+
require File.expand_path("#{app_root}/lib/rest-ftp-daemon/#{lib.to_s}")
|
48
|
+
end
|
49
|
+
|
50
|
+
# Merge options from ARGV into settings
|
55
51
|
Settings.merge!(options)
|
56
52
|
|
57
53
|
# Display compiled configuration
|
@@ -82,6 +78,7 @@ argv = []
|
|
82
78
|
argv << ["-e", Settings.namespace]
|
83
79
|
argv << ["-p", Settings["port"].to_s] unless Settings["port"].nil?
|
84
80
|
argv << ["--pid", Settings.pidfile]
|
81
|
+
argv << ["--tag", "'#{APP_NAME}/#{Settings.namespace}'"]
|
85
82
|
argv << ["--log", Settings["logs"]["thin"].to_s] unless Settings["logs"]["thin"].nil? if Settings["logs"].is_a? Enumerable
|
86
83
|
argv << ["--daemonize"] if [1, true].include? Settings["daemonize"]
|
87
84
|
# User / group
|
@@ -94,17 +91,37 @@ puts "--- Thin ARGV"
|
|
94
91
|
puts argv.flatten.join(' ')
|
95
92
|
puts
|
96
93
|
|
97
|
-
|
94
|
+
|
95
|
+
# Start Thin with this rackup configuration, changing to app_root first
|
98
96
|
begin
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
97
|
+
Dir.chdir app_root
|
98
|
+
r = Thin::Runner.new(argv.flatten).run!
|
99
|
+
rescue RuntimeError => e
|
100
|
+
puts "FAILED: RuntimeError: #{e.message}"
|
101
|
+
rescue Thin::PidFileExist
|
102
|
+
puts "FAILED: daemon was already running (Thin::PidFileExist)"
|
103
|
+
rescue Thin::PidFileNotFound
|
104
|
+
puts "FAILED: daemon was not running (Thin::PidFileNotFound)"
|
105
|
+
rescue SystemExit
|
106
|
+
# Leave some time for the daemon to launch
|
107
|
+
pidfile = Settings.pidfile
|
108
|
+
print "Waiting for pidfile"
|
109
|
+
while !File.file?(pidfile) do
|
110
|
+
print "."
|
111
|
+
sleep 0.25
|
112
|
+
end
|
113
|
+
puts
|
114
|
+
|
115
|
+
# Check that this process is running
|
116
|
+
pid = File.read(pidfile).to_i
|
117
|
+
begin
|
118
|
+
Process.kill(0, pid)
|
119
|
+
puts "Process ##{pid} is running"
|
120
|
+
rescue Errno::EPERM # changed uid
|
121
|
+
puts "No permission to query process ##{pid}!";
|
122
|
+
rescue Errno::ESRCH
|
123
|
+
puts "Process ##{pid} is NOT running."; # or zombied
|
124
|
+
rescue
|
125
|
+
puts "Unable to determine status for ##{pid}: #{$!}"
|
126
|
+
end
|
110
127
|
end
|
@@ -28,13 +28,6 @@ module RestFtpDaemon
|
|
28
28
|
|
29
29
|
helpers do
|
30
30
|
|
31
|
-
def next_job_id
|
32
|
-
@@last_worker_id ||= 0
|
33
|
-
@@last_worker_id += 1
|
34
|
-
#Helpers.identifier(IDENT_JOB_BYTES)
|
35
|
-
#SecureRandom.hex(IDENT_JOB_BYTES)
|
36
|
-
end
|
37
|
-
|
38
31
|
def threads_with_id job_id
|
39
32
|
$threads.list.select do |thread|
|
40
33
|
next unless thread[:job].is_a? Job
|
@@ -45,8 +38,13 @@ module RestFtpDaemon
|
|
45
38
|
def job_describe job_id
|
46
39
|
raise RestFtpDaemon::JobNotFound if ($queue.all_size==0)
|
47
40
|
|
48
|
-
# Find job with this id
|
49
|
-
found = $queue.
|
41
|
+
# Find job with exactly this id
|
42
|
+
found = $queue.find_by_id(job_id)
|
43
|
+
|
44
|
+
# Find job with this id while searching with the current prefix
|
45
|
+
found = $queue.find_by_id(job_id, true) if found.nil?
|
46
|
+
|
47
|
+
# Check that we did find it
|
50
48
|
raise RestFtpDaemon::JobNotFound if found.nil?
|
51
49
|
raise RestFtpDaemon::JobNotFound unless found.is_a? Job
|
52
50
|
|
@@ -71,12 +69,16 @@ module RestFtpDaemon
|
|
71
69
|
|
72
70
|
desc "Get information about a specific job"
|
73
71
|
params do
|
74
|
-
requires :id, type:
|
72
|
+
requires :id, type: String, desc: "job id", regexp: /[^\/]+/
|
73
|
+
# optional :audio do
|
74
|
+
# requires :format, type: Symbol, values: [:mp3, :wav, :aac, :ogg], default: :mp3
|
75
|
+
# end
|
75
76
|
end
|
76
|
-
get '
|
77
|
+
get '*id' do
|
77
78
|
info "GET /jobs/#{params[:id]}"
|
79
|
+
#return params
|
78
80
|
begin
|
79
|
-
response = job_describe params[:id]
|
81
|
+
response = job_describe params[:id]
|
80
82
|
rescue RestFtpDaemon::JobNotFound => exception
|
81
83
|
status 404
|
82
84
|
api_error exception
|
@@ -142,12 +144,17 @@ module RestFtpDaemon
|
|
142
144
|
request.body.rewind
|
143
145
|
params = JSON.parse(request.body.read, symbolize_names: true)
|
144
146
|
|
147
|
+
|
148
|
+
params[:priority] = rand(10)
|
149
|
+
|
145
150
|
# Create a new job
|
146
151
|
# job_id = $last_worker_id += 1
|
147
|
-
job_id =
|
152
|
+
job_id = $queue.generate_id
|
153
|
+
#job = Job.new(params)
|
148
154
|
job = Job.new(job_id, params)
|
149
155
|
|
150
156
|
# And push it to the queue
|
157
|
+
#$queue.push0 job
|
151
158
|
$queue.push job
|
152
159
|
|
153
160
|
rescue JSON::ParserError => exception
|
@@ -25,16 +25,6 @@ module RestFtpDaemon
|
|
25
25
|
Root.logger.info(message, level)
|
26
26
|
end
|
27
27
|
|
28
|
-
def job_list_by_status
|
29
|
-
statuses = {}
|
30
|
-
alljobs = $queue.all.map do |item|
|
31
|
-
next unless item.is_a? Job
|
32
|
-
statuses[item.get_status] ||= 0
|
33
|
-
statuses[item.get_status] +=1
|
34
|
-
end
|
35
|
-
statuses
|
36
|
-
end
|
37
|
-
|
38
28
|
end
|
39
29
|
|
40
30
|
|
@@ -69,7 +59,11 @@ module RestFtpDaemon
|
|
69
59
|
end
|
70
60
|
|
71
61
|
# Jobs to display
|
72
|
-
all_jobs_in_queue = $queue.all
|
62
|
+
# all_jobs_in_queue = $queue.all
|
63
|
+
# all_jobs_in_queue = $queue.ordered_queue
|
64
|
+
# + $queue.popped
|
65
|
+
popped_jobs = $queue.ordered_popped.reverse
|
66
|
+
@jobs_queued = $queue.ordered_queue.reverse
|
73
67
|
|
74
68
|
if params["only"].nil? || params["only"].blank?
|
75
69
|
@only = nil
|
@@ -79,19 +73,15 @@ module RestFtpDaemon
|
|
79
73
|
|
80
74
|
case @only
|
81
75
|
when nil
|
82
|
-
@
|
83
|
-
when :queue
|
84
|
-
|
76
|
+
@jobs_popped = popped_jobs
|
77
|
+
# when :queue
|
78
|
+
# @jobs_popped = $queue.queued
|
85
79
|
else
|
86
|
-
@
|
80
|
+
@jobs_popped = $queue.popped_reverse_sorted_by_status @only
|
87
81
|
end
|
88
82
|
|
89
83
|
# Count jobs for each status
|
90
|
-
@counts =
|
91
|
-
grouped = all_jobs_in_queue.group_by { |job| job.get(:status) }
|
92
|
-
grouped.each do |status, jobs|
|
93
|
-
@counts[status] = jobs.size
|
94
|
-
end
|
84
|
+
@counts = $queue.popped_counts_by_status
|
95
85
|
|
96
86
|
# Get workers status
|
97
87
|
@gworker_statuses = $pool.get_worker_statuses
|
@@ -118,7 +108,8 @@ module RestFtpDaemon
|
|
118
108
|
config: Settings.to_hash,
|
119
109
|
started: APP_STARTED,
|
120
110
|
uptime: (Time.now - APP_STARTED).round(1),
|
121
|
-
|
111
|
+
counters: $queue.counters,
|
112
|
+
status: $queue.popped_counts_by_status,
|
122
113
|
queue_size: $queue.all_size,
|
123
114
|
jobs_queued: $queue.queued.collect(&:id),
|
124
115
|
jobs_popped: $queue.popped.collect(&:id),
|
@@ -1,19 +1,19 @@
|
|
1
1
|
# Terrific constants
|
2
2
|
APP_NAME = "rest-ftp-daemon"
|
3
3
|
APP_CONF = "/etc/#{APP_NAME}.yml"
|
4
|
-
APP_VER = "0.
|
4
|
+
APP_VER = "0.90.1"
|
5
5
|
|
6
6
|
|
7
7
|
# Some global constants
|
8
|
+
IDENT_JOB_LEN = 4
|
8
9
|
IDENT_NOTIF_LEN = 4
|
9
10
|
IDENT_RANDOM_LEN = 8
|
10
11
|
|
11
|
-
|
12
12
|
# Some defaults
|
13
13
|
DEFAULT_CONNECT_TIMEOUT_SEC = 30
|
14
14
|
DEFAULT_UPDATE_EVERY_KB = 2048
|
15
15
|
DEFAULT_WORKERS = 1
|
16
|
-
DEFAULT_LOGS_PROGNAME_TRIM =
|
16
|
+
DEFAULT_LOGS_PROGNAME_TRIM = 12
|
17
17
|
|
18
18
|
|
19
19
|
# Initialize markers
|
@@ -9,12 +9,14 @@ module RestFtpDaemon
|
|
9
9
|
|
10
10
|
class JobException < RestFtpDaemonException; end
|
11
11
|
class JobNotFound < RestFtpDaemonException; end
|
12
|
+
class JobUnresolvedTokens < RestFtpDaemonException; end
|
12
13
|
class JobAssertionFailed < RestFtpDaemonException; end
|
13
14
|
class JobMissingAttribute < RestFtpDaemonException; end
|
14
15
|
class JobSourceNotFound < RestFtpDaemonException; end
|
15
16
|
class JobTargetUnsupported < RestFtpDaemonException; end
|
16
17
|
class JobTargetUnparseable < RestFtpDaemonException; end
|
17
18
|
class JobTargetFileExists < RestFtpDaemonException; end
|
19
|
+
class JobTargetShouldBeDirectory< RestFtpDaemonException; end
|
18
20
|
class JobTooManyOpenFiles < RestFtpDaemonException; end
|
19
21
|
|
20
22
|
end
|
@@ -1,6 +1,5 @@
|
|
1
1
|
require 'securerandom'
|
2
2
|
|
3
|
-
|
4
3
|
module RestFtpDaemon
|
5
4
|
class Helpers
|
6
5
|
|
@@ -16,12 +15,20 @@ module RestFtpDaemon
|
|
16
15
|
def self.identifier len
|
17
16
|
rand(36**len).to_s(36)
|
18
17
|
end
|
19
|
-
#SecureRandom.hex(IDENT_JOB_BYTES)
|
20
18
|
|
21
19
|
def self.tokenize(item)
|
22
20
|
"[#{item}]"
|
23
21
|
end
|
24
22
|
|
23
|
+
def self.contains_tokens(item)
|
24
|
+
/\[[a-zA-Z0-9]+\]/.match(item)
|
25
|
+
end
|
26
|
+
|
27
|
+
def self.extract_filename path
|
28
|
+
m = path.match /\/([^\/]+)$/
|
29
|
+
return m[1] unless m.nil?
|
30
|
+
end
|
31
|
+
|
25
32
|
def self.local_port_used? port
|
26
33
|
ip = '0.0.0.0'
|
27
34
|
timeout = 1
|
@@ -41,6 +48,23 @@ module RestFtpDaemon
|
|
41
48
|
end
|
42
49
|
end
|
43
50
|
|
51
|
+
def self.job_method_label method
|
52
|
+
return if method.nil?
|
53
|
+
klass = case method
|
54
|
+
when :file
|
55
|
+
'label-primary'
|
56
|
+
when :ftp
|
57
|
+
'label-danger'
|
58
|
+
when :ftps
|
59
|
+
'label-warning'
|
60
|
+
when :ftpes
|
61
|
+
'label-warning'
|
62
|
+
else
|
63
|
+
'label-default'
|
64
|
+
end
|
65
|
+
"<div class=\"transfer-method label #{klass}\">#{method.upcase}</div>"
|
66
|
+
end
|
67
|
+
|
44
68
|
# def snakecase
|
45
69
|
# gsub(/::/, '/').
|
46
70
|
# gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
|
data/lib/rest-ftp-daemon/job.rb
CHANGED
@@ -7,35 +7,36 @@ require 'timeout'
|
|
7
7
|
|
8
8
|
module RestFtpDaemon
|
9
9
|
class Job < RestFtpDaemon::Common
|
10
|
+
attr_accessor :id
|
10
11
|
attr_accessor :wid
|
11
12
|
|
12
|
-
def initialize
|
13
|
+
def initialize job_id, params={}
|
13
14
|
# Call super
|
14
15
|
# super()
|
15
16
|
info "Job.initialize"
|
16
17
|
|
17
|
-
#
|
18
|
-
|
19
|
-
|
20
|
-
#
|
21
|
-
@logger = RestFtpDaemon::Logger.new(:workers, "JOB #{id}")
|
18
|
+
# Init context
|
19
|
+
@params = params
|
20
|
+
@id = job_id.to_s
|
21
|
+
#set :id, job_id
|
22
22
|
|
23
23
|
# Protect with a mutex
|
24
24
|
@mutex = Mutex.new
|
25
25
|
|
26
|
-
#
|
27
|
-
@
|
28
|
-
|
26
|
+
# Logger
|
27
|
+
@logger = RestFtpDaemon::Logger.new(:workers, "JOB #{id}")
|
28
|
+
|
29
|
+
# Flag current job
|
29
30
|
set :started_at, Time.now
|
30
31
|
status :created
|
31
32
|
|
32
33
|
# Send first notification
|
33
|
-
info "Job.initialize/notify"
|
34
|
+
#info "Job.initialize/notify"
|
34
35
|
notify "rftpd.queued"
|
35
36
|
end
|
36
37
|
|
37
38
|
def id
|
38
|
-
|
39
|
+
@id
|
39
40
|
end
|
40
41
|
|
41
42
|
def priority
|
@@ -61,15 +62,21 @@ module RestFtpDaemon
|
|
61
62
|
rescue RestFtpDaemon::JobSourceNotFound => exception
|
62
63
|
return oops "rftpd.started", exception, :job_source_not_found
|
63
64
|
|
65
|
+
rescue RestFtpDaemon::JobUnresolvedTokens => exception
|
66
|
+
return oops "rftpd.started", exception, :job_unresolved_tokens
|
67
|
+
|
68
|
+
rescue RestFtpDaemon::JobTargetUnparseable => exception
|
69
|
+
return oops "rftpd.started", exception, :job_target_unparseable
|
70
|
+
|
71
|
+
rescue RestFtpDaemon::JobAssertionFailed => exception
|
72
|
+
return oops "rftpd.started", exception, :job_assertion_failed
|
73
|
+
|
64
74
|
rescue RestFtpDaemon::RestFtpDaemonException => exception
|
65
75
|
return oops "rftpd.started", exception, :job_prepare_failed
|
66
76
|
|
67
77
|
rescue Exception => exception
|
68
78
|
return oops "rftpd.started", exception, :job_prepare_unhandled, true
|
69
79
|
|
70
|
-
rescue exception
|
71
|
-
return oops "rftpd.started", exception, :WOUHOU, true
|
72
|
-
|
73
80
|
else
|
74
81
|
# Update job's status
|
75
82
|
info "Job.process/prepare ok"
|
@@ -90,21 +97,30 @@ module RestFtpDaemon
|
|
90
97
|
status :starting
|
91
98
|
transfer
|
92
99
|
|
93
|
-
rescue
|
94
|
-
return oops "rftpd.ended", exception, :
|
95
|
-
|
96
|
-
rescue Net::FTPPermError => exception
|
97
|
-
return oops "rftpd.ended", exception, :job_ftp_perm_error
|
100
|
+
rescue Errno::EHOSTDOWN => exception
|
101
|
+
return oops "rftpd.ended", exception, :job_host_is_down
|
98
102
|
|
99
103
|
rescue Errno::ECONNREFUSED => exception
|
100
104
|
return oops "rftpd.ended", exception, :job_connexion_refused
|
101
105
|
|
106
|
+
rescue Timeout::Error, Errno::ETIMEDOUT => exception
|
107
|
+
return oops "rftpd.ended", exception, :job_timeout
|
108
|
+
|
109
|
+
rescue Net::FTPPermError => exception
|
110
|
+
return oops "rftpd.ended", exception, :job_perm_error
|
111
|
+
|
102
112
|
rescue Errno::EMFILE => exception
|
103
113
|
return oops "rftpd.ended", exception, :job_too_many_open_files
|
104
114
|
|
105
115
|
rescue RestFtpDaemon::JobTargetFileExists => exception
|
106
116
|
return oops "rftpd.ended", exception, :job_target_file_exists
|
107
117
|
|
118
|
+
rescue RestFtpDaemon::JobTargetShouldBeDirectory => exception
|
119
|
+
return oops "rftpd.ended", exception, :job_target_should_be_directory
|
120
|
+
|
121
|
+
rescue RestFtpDaemon::JobAssertionFailed => exception
|
122
|
+
return oops "rftpd.started", exception, :job_assertion_failed
|
123
|
+
|
108
124
|
rescue RestFtpDaemon::RestFtpDaemonException => exception
|
109
125
|
return oops "rftpd.ended", exception, :job_transfer_failed
|
110
126
|
|
@@ -124,11 +140,14 @@ module RestFtpDaemon
|
|
124
140
|
|
125
141
|
def describe
|
126
142
|
# Update realtime info
|
127
|
-
u = up_time
|
128
|
-
set :uptime, u.round(2) unless u.nil?
|
143
|
+
#u = up_time
|
144
|
+
#set :uptime, u.round(2) unless u.nil?
|
129
145
|
|
130
146
|
# Return the whole structure FIXME
|
131
|
-
@params
|
147
|
+
@params.merge({
|
148
|
+
id: @id,
|
149
|
+
uptime: up_time.round(2)
|
150
|
+
})
|
132
151
|
# @mutex.synchronize do
|
133
152
|
# out = @params.clone
|
134
153
|
# end
|
@@ -144,7 +163,7 @@ module RestFtpDaemon
|
|
144
163
|
protected
|
145
164
|
|
146
165
|
def up_time
|
147
|
-
return if @params[:started_at].nil?
|
166
|
+
return 0 if @params[:started_at].nil?
|
148
167
|
Time.now - @params[:started_at]
|
149
168
|
end
|
150
169
|
|
@@ -175,14 +194,14 @@ module RestFtpDaemon
|
|
175
194
|
end
|
176
195
|
|
177
196
|
def expand_path path
|
178
|
-
File.expand_path
|
197
|
+
File.expand_path replace_tokens(path)
|
179
198
|
end
|
180
199
|
|
181
200
|
def expand_url path
|
182
|
-
URI::parse
|
201
|
+
URI::parse replace_tokens(path) rescue nil
|
183
202
|
end
|
184
203
|
|
185
|
-
def
|
204
|
+
def replace_tokens path
|
186
205
|
# Ensure endpoints are not a nil value
|
187
206
|
return path unless Settings.endpoints.is_a? Enumerable
|
188
207
|
vectors = Settings.endpoints.clone
|
@@ -194,10 +213,14 @@ module RestFtpDaemon
|
|
194
213
|
newpath = path.clone
|
195
214
|
vectors.each do |from, to|
|
196
215
|
next if to.to_s.blank?
|
197
|
-
#info "Job.
|
216
|
+
#info "Job.replace_tokens #{Helpers.tokenize(from)} > #{to}"
|
198
217
|
newpath.gsub! Helpers.tokenize(from), to
|
199
218
|
end
|
200
219
|
|
220
|
+
# Ensure result does not contain tokens after replacement
|
221
|
+
raise RestFtpDaemon::JobUnresolvedTokens if Helpers.contains_tokens newpath
|
222
|
+
|
223
|
+
# All OK, return this
|
201
224
|
return newpath
|
202
225
|
end
|
203
226
|
|
@@ -232,18 +255,26 @@ module RestFtpDaemon
|
|
232
255
|
# Check compliance
|
233
256
|
raise RestFtpDaemon::JobTargetUnparseable if @target_url.nil?
|
234
257
|
raise RestFtpDaemon::JobTargetUnsupported if @target_method.nil?
|
235
|
-
raise RestFtpDaemon::JobSourceNotFound unless File.exists? @source_path
|
258
|
+
#raise RestFtpDaemon::JobSourceNotFound unless File.exists? @source_path
|
236
259
|
end
|
237
260
|
|
238
261
|
def transfer
|
239
|
-
# Method assertions
|
240
|
-
info "Job.transfer checking_source"
|
262
|
+
# Method assertions and init
|
241
263
|
status :checking_source
|
242
|
-
raise RestFtpDaemon::JobAssertionFailed unless @source_path &&
|
264
|
+
raise RestFtpDaemon::JobAssertionFailed unless @source_path && @target_url
|
265
|
+
@transfer_sent = 0
|
266
|
+
set :transfer_source_done, 0
|
243
267
|
|
244
|
-
#
|
268
|
+
# Guess source file names using Dir.glob
|
269
|
+
source_matches = Dir.glob @source_path
|
270
|
+
info "Job.transfer sources: #{source_matches}"
|
271
|
+
raise RestFtpDaemon::JobSourceNotFound if source_matches.empty?
|
272
|
+
set :transfer_source_count, source_matches.count
|
273
|
+
set :transfer_source_files, source_matches
|
245
274
|
|
246
|
-
|
275
|
+
# Guess target file name, and fail if present while we matched multiple sources
|
276
|
+
target_name = Helpers.extract_filename @target_url.path
|
277
|
+
raise RestFtpDaemon::JobTargetShouldBeDirectory if target_name && source_matches.count>1
|
247
278
|
|
248
279
|
# Scheme-aware config
|
249
280
|
ftp_init
|
@@ -251,14 +282,25 @@ module RestFtpDaemon
|
|
251
282
|
# Connect remote server, login and chdir
|
252
283
|
ftp_connect
|
253
284
|
|
254
|
-
# Check
|
255
|
-
|
256
|
-
|
257
|
-
|
285
|
+
# Check source files presence and compute total size, they should be there, coming from Dir.glob()
|
286
|
+
@transfer_total = 0
|
287
|
+
source_matches.each do |filename|
|
288
|
+
# @ftp.close
|
289
|
+
raise RestFtpDaemon::JobSourceNotFound unless File.exists? filename
|
290
|
+
@transfer_total += File.size filename
|
291
|
+
end
|
292
|
+
set :transfer_total, @transfer_total
|
293
|
+
|
294
|
+
# Handle each source file matched, and start a transfer
|
295
|
+
done = 0
|
296
|
+
source_matches.each do |filename|
|
297
|
+
ftp_transfer filename, target_name
|
298
|
+
done += 1
|
299
|
+
set :transfer_source_done, done
|
258
300
|
end
|
259
301
|
|
260
|
-
#
|
261
|
-
|
302
|
+
# Add total transferred to counter
|
303
|
+
$queue.counter_add :transferred, @transfer_total
|
262
304
|
|
263
305
|
# Close FTP connexion
|
264
306
|
info "Job.transfer disconnecting"
|
@@ -320,9 +362,8 @@ module RestFtpDaemon
|
|
320
362
|
info "Job.ftp_connect connect"
|
321
363
|
status :ftp_connect
|
322
364
|
raise RestFtpDaemon::JobAssertionFailed if @ftp.nil? || @target_url.nil?
|
323
|
-
|
324
|
-
|
325
|
-
@ftp.passive = true
|
365
|
+
@ftp.connect(@target_url.host)
|
366
|
+
@ftp.passive = true
|
326
367
|
|
327
368
|
info "Job.ftp_connect login"
|
328
369
|
status :ftp_login
|
@@ -336,55 +377,76 @@ module RestFtpDaemon
|
|
336
377
|
|
337
378
|
def ftp_presence target_name
|
338
379
|
# Method assertions
|
339
|
-
info "Job.ftp_presence"
|
340
380
|
status :ftp_presence
|
341
381
|
raise RestFtpDaemon::JobAssertionFailed if @ftp.nil? || @target_url.nil?
|
342
382
|
|
343
383
|
# Get file list, sometimes the response can be an empty value
|
344
384
|
results = @ftp.list(target_name) rescue nil
|
385
|
+
info "Job.ftp_presence: #{results.inspect}"
|
345
386
|
|
346
387
|
# Result can be nil or a list of files
|
347
388
|
return false if results.nil?
|
348
389
|
return results.count >0
|
349
390
|
end
|
350
391
|
|
351
|
-
def ftp_transfer target_name
|
392
|
+
def ftp_transfer source_match, target_name = nil
|
393
|
+
#target_name
|
352
394
|
# Method assertions
|
353
|
-
info "Job.ftp_transfer
|
395
|
+
info "Job.ftp_transfer source_match: #{source_match}"
|
396
|
+
raise RestFtpDaemon::JobAssertionFailed if @ftp.nil?
|
397
|
+
raise RestFtpDaemon::JobAssertionFailed if source_match.nil?
|
398
|
+
#raise RestFtpDaemon::JobAssertionFailed if @transfer_total.nil?
|
354
399
|
status :ftp_transfer
|
355
|
-
|
400
|
+
|
401
|
+
# Use source filename if target path provided none (typically with multiple sources)
|
402
|
+
target_name ||= Helpers.extract_filename source_match
|
403
|
+
info "Job.ftp_transfer target_name: #{target_name}"
|
404
|
+
|
405
|
+
# Check for target file presence
|
406
|
+
status :checking_target
|
407
|
+
overwrite = !get(:overwrite).nil?
|
408
|
+
present = ftp_presence target_name
|
409
|
+
if present
|
410
|
+
if overwrite
|
411
|
+
# delete it first
|
412
|
+
info "Job.ftp_transfer removing target file"
|
413
|
+
@ftp.delete(target_name)
|
414
|
+
else
|
415
|
+
# won't overwrite then stop here
|
416
|
+
info "Job.ftp_transfer failed: target file exists"
|
417
|
+
@ftp.close
|
418
|
+
raise RestFtpDaemon::JobTargetFileExists
|
419
|
+
end
|
420
|
+
end
|
356
421
|
|
357
422
|
# Read source file size and parameters
|
358
|
-
source_size = File.size @source_path
|
359
|
-
set :transfer_size, source_size
|
360
423
|
update_every_kb = (Settings.transfer.update_every_kb rescue nil) || DEFAULT_UPDATE_EVERY_KB
|
361
424
|
notify_after_sec = Settings.transfer.notify_after_sec rescue nil
|
362
425
|
|
363
426
|
# Start transfer
|
364
|
-
transferred = 0
|
365
427
|
chunk_size = update_every_kb * 1024
|
366
428
|
t0 = tstart = Time.now
|
367
429
|
notified_at = Time.now
|
368
430
|
status :uploading
|
369
|
-
@ftp.putbinaryfile(
|
431
|
+
@ftp.putbinaryfile(source_match, target_name, chunk_size) do |block|
|
370
432
|
# Update counters
|
371
|
-
|
372
|
-
set :transfer_sent,
|
433
|
+
@transfer_sent += block.bytesize
|
434
|
+
set :transfer_sent, @transfer_sent
|
373
435
|
|
374
436
|
# Update bitrate
|
375
|
-
dt = Time.now - t0
|
376
|
-
bitrate0 = (
|
437
|
+
#dt = Time.now - t0
|
438
|
+
bitrate0 = get_bitrate(chunk_size, t0).round(0)
|
377
439
|
set :transfer_bitrate, bitrate0
|
378
440
|
|
379
441
|
# Update job info
|
380
|
-
percent1 = (100.0 *
|
442
|
+
percent1 = (100.0 * @transfer_sent / @transfer_total).round(1)
|
381
443
|
set :progress, percent1
|
382
444
|
|
383
445
|
# Log progress
|
384
446
|
status = []
|
385
447
|
status << "#{percent1} %"
|
386
|
-
status << (Helpers.format_bytes
|
387
|
-
status << (Helpers.format_bytes
|
448
|
+
status << (Helpers.format_bytes @transfer_sent, "B")
|
449
|
+
status << (Helpers.format_bytes @transfer_total, "B")
|
388
450
|
status << (Helpers.format_bytes bitrate0, "bps")
|
389
451
|
info "Job.ftp_transfer" + status.map{|txt| ("%#{DEFAULT_LOGS_PROGNAME_TRIM.to_i}s" % txt)}.join("\t")
|
390
452
|
|
@@ -395,8 +457,8 @@ module RestFtpDaemon
|
|
395
457
|
unless notify_after_sec.nil? || (notified_at + notify_after_sec > Time.now)
|
396
458
|
status = {
|
397
459
|
progress: percent1,
|
398
|
-
transfer_sent:
|
399
|
-
|
460
|
+
transfer_sent: @transfer_sent,
|
461
|
+
transfer_total: @transfer_total,
|
400
462
|
transfer_bitrate: bitrate0
|
401
463
|
}
|
402
464
|
notify "rftpd.progress", 0, status
|
@@ -406,12 +468,10 @@ module RestFtpDaemon
|
|
406
468
|
end
|
407
469
|
|
408
470
|
# Compute final bitrate
|
409
|
-
tbitrate0 = (8 *
|
471
|
+
#tbitrate0 = (8 * @transfer_total.to_f / (Time.now - tstart)).round(0)
|
472
|
+
tbitrate0 = get_bitrate(@transfer_total, tstart).round(0)
|
410
473
|
set :transfer_bitrate, tbitrate0
|
411
474
|
|
412
|
-
# Add total transferred to counter
|
413
|
-
$queue.counter_add :transferred, source_size
|
414
|
-
|
415
475
|
# Done
|
416
476
|
#set :progress, nil
|
417
477
|
info "Job.ftp_transfer finished"
|
@@ -419,12 +479,16 @@ module RestFtpDaemon
|
|
419
479
|
|
420
480
|
def notify signal, error = 0, status = {}
|
421
481
|
RestFtpDaemon::Notification.new get(:notify), {
|
422
|
-
id:
|
482
|
+
id: @id,
|
423
483
|
signal: signal,
|
424
484
|
error: error,
|
425
485
|
status: status,
|
426
486
|
}
|
427
487
|
end
|
428
488
|
|
489
|
+
def get_bitrate total, last_timestamp
|
490
|
+
total.to_f / (Time.now - last_timestamp)
|
491
|
+
end
|
492
|
+
|
429
493
|
end
|
430
494
|
end
|