rest-ftp-daemon 0.90.1 → 0.94.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.ruby-version +1 -1
- data/Gemfile.lock +8 -4
- data/README.md +16 -0
- data/bin/rest-ftp-daemon +1 -1
- data/lib/rest-ftp-daemon.rb +13 -3
- data/lib/rest-ftp-daemon/api/dashboard.rb +60 -0
- data/lib/rest-ftp-daemon/api/debug.rb +25 -0
- data/lib/rest-ftp-daemon/api/job_presenter.rb +33 -0
- data/lib/rest-ftp-daemon/api/jobs.rb +37 -116
- data/lib/rest-ftp-daemon/api/root.rb +39 -107
- data/lib/rest-ftp-daemon/api/routes.rb +16 -0
- data/lib/rest-ftp-daemon/api/status.rb +29 -0
- data/lib/rest-ftp-daemon/constants.rb +5 -3
- data/lib/rest-ftp-daemon/helpers.rb +38 -12
- data/lib/rest-ftp-daemon/job.rb +186 -131
- data/lib/rest-ftp-daemon/job_queue.rb +15 -16
- data/lib/rest-ftp-daemon/notification.rb +1 -2
- data/lib/rest-ftp-daemon/static/css/main.css +45 -4
- data/lib/rest-ftp-daemon/views/dashboard.haml +16 -55
- data/lib/rest-ftp-daemon/views/dashboard_counters.haml +14 -0
- data/lib/rest-ftp-daemon/views/dashboard_headers.haml +29 -0
- data/lib/rest-ftp-daemon/views/dashboard_jobs.haml +37 -21
- data/lib/rest-ftp-daemon/views/dashboard_tokens.haml +6 -1
- data/lib/rest-ftp-daemon/views/dashboard_workers.haml +9 -25
- data/lib/rest-ftp-daemon/worker_pool.rb +1 -1
- data/rest-ftp-daemon.gemspec +1 -0
- metadata +24 -6
- data/lib/rest-ftp-daemon/api/defaults.rb +0 -66
- data/lib/rest-ftp-daemon/api/workers.rb +0 -49
- data/lib/rest-ftp-daemon/static/css/bootstrap.min.old1.css +0 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: eb821dc634f6867b5475e98868b9e6562117ebd2
|
4
|
+
data.tar.gz: 953acf9755629a9ebae2afaac8f1f54438358afb
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e1f92e85d401a25533c0239e8798ea71cc6d3862aec7e176c6ff3dd8486396cab93977abd7d94866820a165ac23f4e4e1a6db7f218cb58db6db4aa43ca79875e
|
7
|
+
data.tar.gz: 6d6d0b77b70bf07a84a39918be28502f45ae71ab8ea6641dbf7ecec9aaa667d212c07963294c2e09f24dd8e91770d7af9d10ddc46928d3750b9aed4baa57eef5
|
data/.ruby-version
CHANGED
@@ -1 +1 @@
|
|
1
|
-
2.1.
|
1
|
+
2.1.5
|
data/Gemfile.lock
CHANGED
@@ -1,10 +1,11 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
rest-ftp-daemon (0.
|
4
|
+
rest-ftp-daemon (0.92)
|
5
5
|
double-bag-ftps
|
6
6
|
facter
|
7
7
|
grape
|
8
|
+
grape-entity
|
8
9
|
haml
|
9
10
|
json
|
10
11
|
settingslogic
|
@@ -16,7 +17,7 @@ GEM
|
|
16
17
|
remote: http://rubygems.org/
|
17
18
|
specs:
|
18
19
|
CFPropertyList (2.2.8)
|
19
|
-
activesupport (4.1.
|
20
|
+
activesupport (4.1.8)
|
20
21
|
i18n (~> 0.6, >= 0.6.9)
|
21
22
|
json (~> 1.7, >= 1.7.7)
|
22
23
|
minitest (~> 5.1)
|
@@ -48,13 +49,16 @@ GEM
|
|
48
49
|
rack-accept
|
49
50
|
rack-mount
|
50
51
|
virtus (>= 1.0.0)
|
52
|
+
grape-entity (0.4.4)
|
53
|
+
activesupport
|
54
|
+
multi_json (>= 1.3.2)
|
51
55
|
haml (4.0.5)
|
52
56
|
tilt
|
53
57
|
hashie (3.3.1)
|
54
58
|
i18n (0.6.11)
|
55
59
|
ice_nine (0.11.0)
|
56
60
|
json (1.8.1)
|
57
|
-
minitest (5.4.
|
61
|
+
minitest (5.4.3)
|
58
62
|
multi_json (1.10.1)
|
59
63
|
multi_xml (0.5.5)
|
60
64
|
rack (1.5.2)
|
@@ -62,7 +66,7 @@ GEM
|
|
62
66
|
rack (>= 0.4)
|
63
67
|
rack-mount (0.8.3)
|
64
68
|
rack (>= 1.0.0)
|
65
|
-
rake (10.
|
69
|
+
rake (10.4.0)
|
66
70
|
settingslogic (2.0.9)
|
67
71
|
sys-cpu (0.7.1)
|
68
72
|
ffi (>= 1.0.0)
|
data/README.md
CHANGED
@@ -184,6 +184,22 @@ GET http://localhost:3100/jobs/1
|
|
184
184
|
```
|
185
185
|
|
186
186
|
|
187
|
+
|
188
|
+
Doc TODO
|
189
|
+
------------------------------------------------------------------------------------
|
190
|
+
|
191
|
+
Document /status
|
192
|
+
|
193
|
+
Document /routes
|
194
|
+
|
195
|
+
Document multiple-files upload
|
196
|
+
|
197
|
+
Document mkdir and overwrite options
|
198
|
+
|
199
|
+
Document counters
|
200
|
+
|
201
|
+
|
202
|
+
|
187
203
|
About
|
188
204
|
------------------------------------------------------------------------------------
|
189
205
|
|
data/bin/rest-ftp-daemon
CHANGED
@@ -63,7 +63,7 @@ puts Settings.to_hash.to_yaml( :Indent => 4, :UseHeader => true, :UseVersion =>
|
|
63
63
|
puts
|
64
64
|
|
65
65
|
# Validate network configuration
|
66
|
-
if ["start"
|
66
|
+
if ["start"].include? command
|
67
67
|
if Settings['port'].nil?
|
68
68
|
puts "ABORTING: Network port is missing"
|
69
69
|
exit 1
|
data/lib/rest-ftp-daemon.rb
CHANGED
@@ -1,6 +1,13 @@
|
|
1
1
|
# Global libs
|
2
2
|
require 'rubygems'
|
3
3
|
require 'json'
|
4
|
+
require 'grape'
|
5
|
+
require 'grape-entity'
|
6
|
+
require 'haml'
|
7
|
+
require "facter"
|
8
|
+
require "sys/cpu"
|
9
|
+
|
10
|
+
|
4
11
|
# require 'celluloid/autostart'
|
5
12
|
|
6
13
|
# My libs
|
@@ -15,7 +22,10 @@ require 'rest-ftp-daemon/worker_pool'
|
|
15
22
|
require 'rest-ftp-daemon/logger'
|
16
23
|
require 'rest-ftp-daemon/job'
|
17
24
|
require 'rest-ftp-daemon/notification'
|
18
|
-
require 'rest-ftp-daemon/api/defaults'
|
19
|
-
require 'rest-ftp-daemon/api/jobs'
|
20
25
|
require 'rest-ftp-daemon/api/root'
|
21
|
-
|
26
|
+
require 'rest-ftp-daemon/api/jobs'
|
27
|
+
require 'rest-ftp-daemon/api/debug'
|
28
|
+
require 'rest-ftp-daemon/api/routes'
|
29
|
+
require 'rest-ftp-daemon/api/dashboard'
|
30
|
+
require 'rest-ftp-daemon/api/status'
|
31
|
+
require 'rest-ftp-daemon/api/job_presenter'
|
@@ -0,0 +1,60 @@
|
|
1
|
+
module RestFtpDaemon
|
2
|
+
module API
|
3
|
+
class Root < Grape::API
|
4
|
+
|
5
|
+
####### DASHBOARD - GET /
|
6
|
+
|
7
|
+
# Server global status
|
8
|
+
get '/' do
|
9
|
+
info "GET /"
|
10
|
+
|
11
|
+
# Initialize UsageWatch
|
12
|
+
Facter.loadfacts
|
13
|
+
@info_load = Sys::CPU.load_avg.first.to_f
|
14
|
+
@info_procs = (Facter.value :processorcount).to_i
|
15
|
+
@info_ipaddr = Facter.value(:ipaddress)
|
16
|
+
@info_memfree = Facter.value(:memoryfree)
|
17
|
+
|
18
|
+
# Compute normalized load
|
19
|
+
if @info_procs.zero?
|
20
|
+
@info_norm = "N/A"
|
21
|
+
else
|
22
|
+
@info_norm = (100 * @info_load / @info_procs).round(1)
|
23
|
+
end
|
24
|
+
|
25
|
+
# Jobs to display
|
26
|
+
popped_jobs = $queue.ordered_popped.reverse
|
27
|
+
@jobs_queued = $queue.ordered_queue.reverse
|
28
|
+
|
29
|
+
if params["only"].nil? || params["only"].blank?
|
30
|
+
@only = nil
|
31
|
+
else
|
32
|
+
@only = params["only"].to_sym
|
33
|
+
end
|
34
|
+
|
35
|
+
if @only.nil?
|
36
|
+
@jobs_popped = popped_jobs
|
37
|
+
else
|
38
|
+
@jobs_popped = $queue.popped_reverse_sorted_by_status @only
|
39
|
+
end
|
40
|
+
|
41
|
+
# Count jobs for each status
|
42
|
+
@counts = $queue.counts_by_status
|
43
|
+
|
44
|
+
# Get workers status
|
45
|
+
@gworker_statuses = $pool.get_worker_statuses
|
46
|
+
|
47
|
+
# Compile haml template
|
48
|
+
output = render :dashboard
|
49
|
+
|
50
|
+
# Send response
|
51
|
+
env['api.format'] = :html
|
52
|
+
format "html"
|
53
|
+
status 200
|
54
|
+
content_type "text/html"
|
55
|
+
body output
|
56
|
+
end
|
57
|
+
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
module RestFtpDaemon
|
2
|
+
module API
|
3
|
+
class Root < Grape::API
|
4
|
+
|
5
|
+
####### GET /debug
|
6
|
+
|
7
|
+
get '/debug' do
|
8
|
+
info "GET /debug"
|
9
|
+
begin
|
10
|
+
raise RestFtpDaemon::DummyException
|
11
|
+
rescue RestFtpDaemon::RestFtpDaemonException => exception
|
12
|
+
status 501
|
13
|
+
api_error exception
|
14
|
+
rescue Exception => exception
|
15
|
+
status 501
|
16
|
+
api_error exception
|
17
|
+
else
|
18
|
+
status 200
|
19
|
+
{}
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
module RestFtpDaemon
|
2
|
+
module API
|
3
|
+
module Entities
|
4
|
+
|
5
|
+
class JobPresenter < Grape::Entity
|
6
|
+
# Job ID
|
7
|
+
expose :id
|
8
|
+
|
9
|
+
# Job specific attributes
|
10
|
+
Job::FIELDS.each do |field|
|
11
|
+
expose field
|
12
|
+
#expose field, unless: lambda { |object, options| object.instance_variable_get("@#{field}").nil? }
|
13
|
+
end
|
14
|
+
|
15
|
+
# Technical fields
|
16
|
+
expose :wid, unless: lambda { |object, options| object.wid.nil? }
|
17
|
+
|
18
|
+
expose :error
|
19
|
+
expose :status
|
20
|
+
|
21
|
+
expose :started_at
|
22
|
+
expose :updated_at
|
23
|
+
expose :age
|
24
|
+
|
25
|
+
# Params
|
26
|
+
# expose :wid, unless: lambda { |object, options| object.wid.nil? }
|
27
|
+
expose :params, unless: :hide_params
|
28
|
+
|
29
|
+
end
|
30
|
+
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -1,84 +1,19 @@
|
|
1
1
|
module RestFtpDaemon
|
2
2
|
module API
|
3
|
+
class Root < Grape::API
|
3
4
|
|
4
|
-
class Jobs < Grape::API
|
5
5
|
|
6
|
+
####### GET /jobs/:id
|
6
7
|
|
7
|
-
####### CLASS CONFIG
|
8
|
-
|
9
|
-
# params do
|
10
|
-
# optional :overwrite, type: Integer, default: false
|
11
|
-
# end
|
12
|
-
|
13
|
-
|
14
|
-
####### INITIALIZATION
|
15
|
-
|
16
|
-
def initialize
|
17
|
-
#$last_worker_id = 0
|
18
|
-
|
19
|
-
# Check that Queue and Pool are available
|
20
|
-
raise RestFtpDaemon::MissingQueue unless defined? $queue
|
21
|
-
raise RestFtpDaemon::MissingPool unless defined? $pool
|
22
|
-
|
23
|
-
super
|
24
|
-
end
|
25
|
-
|
26
|
-
|
27
|
-
####### HELPERS
|
28
|
-
|
29
|
-
helpers do
|
30
|
-
|
31
|
-
def threads_with_id job_id
|
32
|
-
$threads.list.select do |thread|
|
33
|
-
next unless thread[:job].is_a? Job
|
34
|
-
thread[:job].id == job_id
|
35
|
-
end
|
36
|
-
end
|
37
|
-
|
38
|
-
def job_describe job_id
|
39
|
-
raise RestFtpDaemon::JobNotFound if ($queue.all_size==0)
|
40
|
-
|
41
|
-
# Find job with exactly this id
|
42
|
-
found = $queue.find_by_id(job_id)
|
43
|
-
|
44
|
-
# Find job with this id while searching with the current prefix
|
45
|
-
found = $queue.find_by_id(job_id, true) if found.nil?
|
46
|
-
|
47
|
-
# Check that we did find it
|
48
|
-
raise RestFtpDaemon::JobNotFound if found.nil?
|
49
|
-
raise RestFtpDaemon::JobNotFound unless found.is_a? Job
|
50
|
-
|
51
|
-
# Return job description
|
52
|
-
found.describe
|
53
|
-
end
|
54
|
-
|
55
|
-
# def job_delete job_id
|
56
|
-
# end
|
57
|
-
|
58
|
-
def job_list
|
59
|
-
$queue.all.map do |item|
|
60
|
-
next unless item.is_a? Job
|
61
|
-
item.describe
|
62
|
-
end
|
63
|
-
end
|
64
|
-
|
65
|
-
end
|
66
|
-
|
67
|
-
|
68
|
-
####### API DEFINITION
|
69
|
-
|
70
|
-
desc "Get information about a specific job"
|
71
8
|
params do
|
72
|
-
requires :id, type: String, desc:
|
73
|
-
# optional :audio do
|
74
|
-
# requires :format, type: Symbol, values: [:mp3, :wav, :aac, :ogg], default: :mp3
|
75
|
-
# end
|
9
|
+
requires :id, type: String, desc: 'ID of the Job to read', regexp: /[^\/]+/
|
76
10
|
end
|
77
|
-
get '
|
11
|
+
get '/jobs/*id' do
|
78
12
|
info "GET /jobs/#{params[:id]}"
|
79
|
-
#return params
|
80
13
|
begin
|
81
|
-
|
14
|
+
job = job_find params[:id]
|
15
|
+
raise RestFtpDaemon::JobNotFound if job.nil?
|
16
|
+
|
82
17
|
rescue RestFtpDaemon::JobNotFound => exception
|
83
18
|
status 404
|
84
19
|
api_error exception
|
@@ -90,38 +25,20 @@ module RestFtpDaemon
|
|
90
25
|
api_error exception
|
91
26
|
else
|
92
27
|
status 200
|
93
|
-
|
28
|
+
present job, :with => RestFtpDaemon::API::Entities::JobPresenter, type: "complete"
|
94
29
|
end
|
95
|
-
end
|
96
30
|
|
97
|
-
# Delete jobs
|
98
|
-
desc "Kill and remove a specific job"
|
99
|
-
delete ':id' do
|
100
|
-
info "DELETE /jobs/#{params[:name]}"
|
101
|
-
status 501
|
102
|
-
# begin
|
103
|
-
# response = job_delete params[:id].to_i
|
104
|
-
# rescue RestFtpDaemon::JobNotFound => exception
|
105
|
-
# status 404
|
106
|
-
# api_error exception
|
107
|
-
# rescue RestFtpDaemonException => exception
|
108
|
-
# status 500
|
109
|
-
# api_error exception
|
110
|
-
# rescue Exception => exception
|
111
|
-
# status 501
|
112
|
-
# api_error exception
|
113
|
-
# else
|
114
|
-
# status 200
|
115
|
-
# response
|
116
|
-
# end
|
117
31
|
end
|
118
32
|
|
119
|
-
|
120
|
-
|
121
|
-
|
33
|
+
|
34
|
+
####### GET /jobs/
|
35
|
+
|
36
|
+
desc "List all Jobs"
|
37
|
+
|
38
|
+
get '/jobs/' do
|
122
39
|
info "GET /jobs"
|
123
40
|
begin
|
124
|
-
|
41
|
+
jobs = $queue.all
|
125
42
|
rescue RestFtpDaemonException => exception
|
126
43
|
status 501
|
127
44
|
api_error exception
|
@@ -130,33 +47,42 @@ module RestFtpDaemon
|
|
130
47
|
api_error exception
|
131
48
|
else
|
132
49
|
status 200
|
133
|
-
|
50
|
+
present jobs, :with => RestFtpDaemon::API::Entities::JobPresenter
|
134
51
|
end
|
135
52
|
end
|
136
53
|
|
137
54
|
|
138
|
-
|
55
|
+
|
56
|
+
####### POST /jobs/
|
57
|
+
|
139
58
|
desc "Create a new job"
|
140
|
-
post do
|
141
|
-
info "POST /jobs: #{request.body.read}"
|
142
|
-
begin
|
143
|
-
# Extract params
|
144
|
-
request.body.rewind
|
145
|
-
params = JSON.parse(request.body.read, symbolize_names: true)
|
146
59
|
|
60
|
+
params do
|
61
|
+
requires :source, type: String, desc: "Source file pattern"
|
62
|
+
requires :target, type: String, desc: "Target remote path"
|
63
|
+
optional :notify, type: String, desc: "URL to get POST'ed notifications back"
|
64
|
+
optional :priority, type: Integer, desc: "Priority level of the job (lower is stronger)"
|
65
|
+
optional :overwrite, type: Boolean, desc: "wether to overwrites files at target server",
|
66
|
+
default: false
|
67
|
+
optional :mkdir, type: Boolean, desc: "wether to create missing directories on target server",
|
68
|
+
default: false
|
69
|
+
end
|
147
70
|
|
148
|
-
|
71
|
+
post '/jobs/' do
|
72
|
+
info "POST /jobs #{params.inspect}"
|
73
|
+
# request.body.rewind
|
74
|
+
begin
|
149
75
|
|
150
76
|
# Create a new job
|
151
|
-
# job_id = $last_worker_id += 1
|
152
77
|
job_id = $queue.generate_id
|
153
|
-
#job = Job.new(params)
|
154
78
|
job = Job.new(job_id, params)
|
155
79
|
|
156
80
|
# And push it to the queue
|
157
|
-
#$queue.push0 job
|
158
81
|
$queue.push job
|
159
82
|
|
83
|
+
# Increment a counter
|
84
|
+
$queue.counter_inc :jobs_received
|
85
|
+
|
160
86
|
rescue JSON::ParserError => exception
|
161
87
|
status 406
|
162
88
|
api_error exception
|
@@ -168,15 +94,10 @@ params[:priority] = rand(10)
|
|
168
94
|
api_error exception
|
169
95
|
else
|
170
96
|
status 201
|
171
|
-
job
|
97
|
+
present job, :with => RestFtpDaemon::API::Entities::JobPresenter
|
172
98
|
end
|
173
99
|
end
|
174
100
|
|
175
|
-
protected
|
176
|
-
|
177
|
-
# def progname
|
178
|
-
# "API::Jobs"
|
179
|
-
# end
|
180
101
|
|
181
102
|
end
|
182
103
|
end
|