rest-ftp-daemon 0.20.0 → 0.30.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +8 -8
- data/Gemfile +1 -9
- data/Gemfile.lock +55 -10
- data/README.md +5 -4
- data/bin/rest-ftp-daemon +4 -3
- data/config.ru +6 -0
- data/lib/rest-ftp-daemon.rb +10 -309
- data/lib/rest-ftp-daemon/config.rb +8 -0
- data/lib/rest-ftp-daemon/exceptions.rb +21 -0
- data/lib/rest-ftp-daemon/job.rb +173 -0
- data/lib/rest-ftp-daemon/server.rb +237 -0
- data/rest-ftp-daemon.gemspec +7 -29
- metadata +50 -9
- data/VERSION +0 -1
- data/lib/config.rb +0 -3
- data/lib/config.ru +0 -12
- data/lib/errors.rb +0 -12
- data/lib/extend_threads.rb +0 -14
- data/lib/version.rb +0 -3
checksums.yaml
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
---
|
2
2
|
!binary "U0hBMQ==":
|
3
3
|
metadata.gz: !binary |-
|
4
|
-
|
4
|
+
YThkNmNkZTEwNzM2YzcwM2I2N2NiOWE5OTQwNDI1N2M0NTFjZWZhOA==
|
5
5
|
data.tar.gz: !binary |-
|
6
|
-
|
6
|
+
ZTllMzMyNGMzNTE1M2U5ZTBjZTc3MWJmYWNiYWJlNzQwM2NiNmMxNQ==
|
7
7
|
SHA512:
|
8
8
|
metadata.gz: !binary |-
|
9
|
-
|
10
|
-
|
11
|
-
|
9
|
+
ODQzMzc5MzRiZDZjZDAyM2QxZWQyMThjNTUyZmNlZWJhOGFiNTJiODAyYWJi
|
10
|
+
YjExZmZiNGMzMDM5ZDRjMmRjYTMxMTNhZGU0ZWNkODMxMzVjZTQ3YWY2MTk5
|
11
|
+
MjYxM2MwNDU5YzM2OWU1MGVjZjc5NTgwYWVjOTdmZTEzNjQ5MGM=
|
12
12
|
data.tar.gz: !binary |-
|
13
|
-
|
14
|
-
|
15
|
-
|
13
|
+
NTBhZWM0NDM2ODNiY2ZjNTA0YjYxYWE3MTBlYWUwZmM3NjgxOGZhNmNlZDlk
|
14
|
+
ZGEwOWE1ZDM4ZjU1N2FjYTQ2ZTNlMjllOGE5ODlhMTJjNjA5ZGFhYjFlODdj
|
15
|
+
ZDBjM2Y1ZTVmODdiYTk0YjdkMGViMGE1ZDA4ZjQ0NzdhZWUxODE=
|
data/Gemfile
CHANGED
data/Gemfile.lock
CHANGED
@@ -1,20 +1,65 @@
|
|
1
|
+
PATH
|
2
|
+
remote: .
|
3
|
+
specs:
|
4
|
+
rest-ftp-daemon (0.22)
|
5
|
+
grape
|
6
|
+
json
|
7
|
+
|
1
8
|
GEM
|
2
9
|
remote: http://rubygems.org/
|
3
10
|
specs:
|
11
|
+
activesupport (4.1.5)
|
12
|
+
i18n (~> 0.6, >= 0.6.9)
|
13
|
+
json (~> 1.7, >= 1.7.7)
|
14
|
+
minitest (~> 5.1)
|
15
|
+
thread_safe (~> 0.1)
|
16
|
+
tzinfo (~> 1.1)
|
17
|
+
axiom-types (0.1.1)
|
18
|
+
descendants_tracker (~> 0.0.4)
|
19
|
+
ice_nine (~> 0.11.0)
|
20
|
+
thread_safe (~> 0.3, >= 0.3.1)
|
21
|
+
builder (3.2.2)
|
22
|
+
coercible (1.0.0)
|
23
|
+
descendants_tracker (~> 0.0.1)
|
24
|
+
descendants_tracker (0.0.4)
|
25
|
+
thread_safe (~> 0.3, >= 0.3.1)
|
26
|
+
equalizer (0.0.9)
|
27
|
+
grape (0.8.0)
|
28
|
+
activesupport
|
29
|
+
builder
|
30
|
+
hashie (>= 2.1.0)
|
31
|
+
multi_json (>= 1.3.2)
|
32
|
+
multi_xml (>= 0.5.2)
|
33
|
+
rack (>= 1.3.0)
|
34
|
+
rack-accept
|
35
|
+
rack-mount
|
36
|
+
virtus (>= 1.0.0)
|
37
|
+
hashie (3.2.0)
|
38
|
+
i18n (0.6.11)
|
39
|
+
ice_nine (0.11.0)
|
4
40
|
json (1.8.1)
|
41
|
+
minitest (5.4.0)
|
42
|
+
multi_json (1.10.1)
|
43
|
+
multi_xml (0.5.5)
|
5
44
|
rack (1.5.2)
|
6
|
-
rack-
|
7
|
-
rack
|
8
|
-
|
9
|
-
rack (
|
10
|
-
|
11
|
-
|
12
|
-
|
45
|
+
rack-accept (0.4.5)
|
46
|
+
rack (>= 0.4)
|
47
|
+
rack-mount (0.8.3)
|
48
|
+
rack (>= 1.0.0)
|
49
|
+
rake (10.3.2)
|
50
|
+
thread_safe (0.3.4)
|
51
|
+
tzinfo (1.2.2)
|
52
|
+
thread_safe (~> 0.1)
|
53
|
+
virtus (1.0.3)
|
54
|
+
axiom-types (~> 0.1)
|
55
|
+
coercible (~> 1.0)
|
56
|
+
descendants_tracker (~> 0.0, >= 0.0.3)
|
57
|
+
equalizer (~> 0.0, >= 0.0.9)
|
13
58
|
|
14
59
|
PLATFORMS
|
15
60
|
ruby
|
16
61
|
|
17
62
|
DEPENDENCIES
|
18
|
-
bundler (~> 1.
|
19
|
-
|
20
|
-
|
63
|
+
bundler (~> 1.6)
|
64
|
+
rake
|
65
|
+
rest-ftp-daemon!
|
data/README.md
CHANGED
@@ -14,12 +14,13 @@ As of today, its main features are :
|
|
14
14
|
Installation
|
15
15
|
------------------------------------------------------------------------------------
|
16
16
|
|
17
|
-
This project is available as a rubygem, requires on ruby >= 1.9 and rubygems installed.
|
17
|
+
This project is available as a rubygem, requires on ruby >= 1.9.3 and rubygems installed.
|
18
18
|
|
19
19
|
Get and install the gem from rubygems.org:
|
20
20
|
|
21
21
|
```
|
22
|
-
|
22
|
+
# apt-get install ruby2.1 rubygems
|
23
|
+
gem install rest-ftp-daemon --no-ri --no-rdoc
|
23
24
|
```
|
24
25
|
|
25
26
|
Start the daemon:
|
@@ -40,11 +41,11 @@ For now, daemon logs to ```APP_LOGTO``` defined in ```lib/config.rb```
|
|
40
41
|
Usage examples
|
41
42
|
------------------------------------------------------------------------------------
|
42
43
|
|
43
|
-
Start a job to transfer a file named "file.
|
44
|
+
Start a job to transfer a file named "file.iso" to a local FTP server
|
44
45
|
|
45
46
|
```
|
46
47
|
curl -H "Content-Type: application/json" -X POST -D /dev/stdout -d \
|
47
|
-
'{"source":"~/file.
|
48
|
+
'{"source":"~/file.iso","target":"ftp://anonymous@localhost/incoming/dest2.iso"}' "http://localhost:3000/jobs"
|
48
49
|
```
|
49
50
|
|
50
51
|
Start a job to transfer a file named "file.dmg" to a local FTP server
|
data/bin/rest-ftp-daemon
CHANGED
@@ -6,17 +6,18 @@ require 'thin'
|
|
6
6
|
# Initialize some local constants
|
7
7
|
APP_ROOT = File.dirname(__FILE__) + '/../'
|
8
8
|
APP_NAME = 'rest-ftp-daemon'
|
9
|
-
|
9
|
+
|
10
10
|
APP_STARTED = Time.now
|
11
11
|
APP_DEFAULT_PORT = 3000
|
12
12
|
APP_LOGTO = "/tmp/#{APP_NAME}.log"
|
13
13
|
|
14
14
|
# Prepare thin
|
15
|
-
rackup_file = File.expand_path "#{APP_ROOT}/
|
15
|
+
rackup_file = File.expand_path "#{APP_ROOT}/config.ru"
|
16
16
|
argv = ARGV
|
17
17
|
argv << ["-R", rackup_file] unless ARGV.include?("-R")
|
18
18
|
argv << ["-p", APP_DEFAULT_PORT.to_s] unless ARGV.include?("-p")
|
19
19
|
argv << ["-e", "production"] unless ARGV.include?("-e")
|
20
|
-
argv << ["--
|
20
|
+
argv << ["--daemonize"] unless ARGV.include?("-R")
|
21
|
+
#argv << ["--stats", "/stats"] unless ARGV.include?("--stats")
|
21
22
|
|
22
23
|
Thin::Runner.new(argv.flatten).run!
|
data/config.ru
ADDED
data/lib/rest-ftp-daemon.rb
CHANGED
@@ -1,309 +1,10 @@
|
|
1
|
-
#
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
# Logging configuration
|
12
|
-
#use Rack::CommonLogger, logger
|
13
|
-
|
14
|
-
# Some other configuration
|
15
|
-
disable :sessions
|
16
|
-
disable :logging
|
17
|
-
end
|
18
|
-
|
19
|
-
# Server initialization
|
20
|
-
def initialize
|
21
|
-
# Setup logger
|
22
|
-
@logger = Logger.new(APP_LOGTO, 'daily')
|
23
|
-
#@logger = Logger.new
|
24
|
-
#@logger.level = Logger::INFO
|
25
|
-
|
26
|
-
# Other stuff
|
27
|
-
@@last_worker_id = 0
|
28
|
-
@@hostname = `hostname`.chomp
|
29
|
-
|
30
|
-
super
|
31
|
-
end
|
32
|
-
|
33
|
-
# Server global status
|
34
|
-
get "/" do
|
35
|
-
# Debug query
|
36
|
-
info "GET /"
|
37
|
-
|
38
|
-
# Build response
|
39
|
-
content_type :json
|
40
|
-
JSON.pretty_generate get_status
|
41
|
-
end
|
42
|
-
|
43
|
-
# List jobs
|
44
|
-
get "/jobs" do
|
45
|
-
# Debug query
|
46
|
-
info "GET /jobs"
|
47
|
-
|
48
|
-
# Build response
|
49
|
-
content_type :json
|
50
|
-
JSON.pretty_generate get_jobs
|
51
|
-
end
|
52
|
-
|
53
|
-
# Get job info
|
54
|
-
get "/jobs/:id" do
|
55
|
-
# Debug query
|
56
|
-
info "GET /jobs/#{params[:id]}"
|
57
|
-
|
58
|
-
# Find this process by name
|
59
|
-
found = find_job params[:id]
|
60
|
-
|
61
|
-
# Build response
|
62
|
-
error 404 and return if found.nil?
|
63
|
-
content_type :json
|
64
|
-
JSON.pretty_generate found
|
65
|
-
end
|
66
|
-
|
67
|
-
# Delete jobs
|
68
|
-
delete "/jobs/:id" do
|
69
|
-
# Debug query
|
70
|
-
info "DELETE /jobs/#{params[:name]}"
|
71
|
-
|
72
|
-
# Find and kill this job
|
73
|
-
found = delete_job params[:id]
|
74
|
-
|
75
|
-
# Build response
|
76
|
-
error 404 and return if found.nil?
|
77
|
-
content_type :json
|
78
|
-
JSON.pretty_generate found
|
79
|
-
end
|
80
|
-
|
81
|
-
# Spawn a new thread for this new job
|
82
|
-
post '/jobs' do
|
83
|
-
# Extract payload
|
84
|
-
request.body.rewind
|
85
|
-
payload = JSON.parse request.body.read
|
86
|
-
|
87
|
-
# Debug query
|
88
|
-
info "POST /jobs: #{payload.to_json}"
|
89
|
-
|
90
|
-
# Spawn a thread for this job
|
91
|
-
result = new_job payload
|
92
|
-
|
93
|
-
# Build response
|
94
|
-
content_type :json
|
95
|
-
JSON.pretty_generate result
|
96
|
-
end
|
97
|
-
|
98
|
-
protected
|
99
|
-
|
100
|
-
def process_job
|
101
|
-
# Init
|
102
|
-
info "process_job: starting"
|
103
|
-
job = Thread.current.job
|
104
|
-
job_status :started
|
105
|
-
transferred = 0
|
106
|
-
|
107
|
-
# Check source
|
108
|
-
job_source = File.expand_path(job["source"])
|
109
|
-
if !(File.exists? job_source)
|
110
|
-
job_error ERR_JOB_SOURCE_NOTFOUND, :ERR_JOB_SOURCE_NOTFOUND
|
111
|
-
return
|
112
|
-
end
|
113
|
-
info "process_job: job_source: #{job_source}"
|
114
|
-
source_size = File.size job_source
|
115
|
-
job_set :source_size, source_size
|
116
|
-
|
117
|
-
# Check target
|
118
|
-
job_target = job["target"]
|
119
|
-
target = URI(job_target) rescue nil
|
120
|
-
if job_target.nil? || target.nil?
|
121
|
-
job_error ERR_JOB_TARGET_UNPARSEABLE, :ERR_JOB_TARGET_UNPARSEABLE
|
122
|
-
return
|
123
|
-
end
|
124
|
-
info "process_job: job_target: #{job_target}"
|
125
|
-
|
126
|
-
# Split URI
|
127
|
-
target_path = File.dirname target.path
|
128
|
-
target_name = File.basename target.path
|
129
|
-
info "ftp_transfer: job_target.host [#{target.host}]"
|
130
|
-
info "ftp_transfer: target_path [#{target_path}]"
|
131
|
-
info "ftp_transfer: target_name [#{target_name}]"
|
132
|
-
|
133
|
-
# Prepare FTP transfer
|
134
|
-
ftp = Net::FTP.new(target.host)
|
135
|
-
ftp.passive = true
|
136
|
-
ftp.login
|
137
|
-
ftp.chdir(target_path)
|
138
|
-
|
139
|
-
|
140
|
-
# Check if target file is found
|
141
|
-
info "source: checking target file"
|
142
|
-
job_status :checking_target
|
143
|
-
job_error ERR_BUSY, :checking_target
|
144
|
-
|
145
|
-
results = ftp.list(target_name)
|
146
|
-
info "ftp.list: #{results}"
|
147
|
-
unless results.count.zero?
|
148
|
-
job_error ERR_JOB_TARGET_PRESENT, :ERR_JOB_TARGET_PRESENT
|
149
|
-
info "target: existing: ERR_JOB_TARGET_PRESENT"
|
150
|
-
ftp.close
|
151
|
-
return
|
152
|
-
end
|
153
|
-
|
154
|
-
|
155
|
-
# Do transfer
|
156
|
-
info "source: starting stransfer"
|
157
|
-
#Thread.current[:status] = :transferring
|
158
|
-
job_status :uploading
|
159
|
-
job_error ERR_BUSY, :uploading
|
160
|
-
|
161
|
-
begin
|
162
|
-
ftp.putbinaryfile(job_source, target_name, TRANSFER_CHUNK_SIZE) do |block|
|
163
|
-
# Update thread info
|
164
|
-
percent = (100.0 * transferred / source_size).round(1)
|
165
|
-
job_set :progress, percent
|
166
|
-
job_set :transferred, transferred
|
167
|
-
info "transferring [#{percent} %] of [#{target_name}]"
|
168
|
-
|
169
|
-
# Update counters
|
170
|
-
transferred += TRANSFER_CHUNK_SIZE
|
171
|
-
end
|
172
|
-
|
173
|
-
rescue Net::FTPPermError
|
174
|
-
#job_status :failed
|
175
|
-
job_error ERR_JOB_PERMISSION, :ERR_JOB_PERMISSION
|
176
|
-
info "source: FAILED: PERMISSIONS ERROR"
|
177
|
-
|
178
|
-
else
|
179
|
-
#job_status :finished
|
180
|
-
job_error ERR_OK, :finished
|
181
|
-
info "source: finished stransfer"
|
182
|
-
end
|
183
|
-
|
184
|
-
# Close FTP connexion
|
185
|
-
ftp.close
|
186
|
-
end
|
187
|
-
|
188
|
-
def get_status
|
189
|
-
info "> get_status"
|
190
|
-
{
|
191
|
-
app_name: APP_NAME,
|
192
|
-
hostname: @@hostname,
|
193
|
-
version: APP_VER,
|
194
|
-
started: APP_STARTED,
|
195
|
-
uptime: (Time.now - APP_STARTED).round(1),
|
196
|
-
jobs_count: @@workers.list.count,
|
197
|
-
}
|
198
|
-
end
|
199
|
-
|
200
|
-
def get_jobs
|
201
|
-
info "> get_jobs"
|
202
|
-
|
203
|
-
# Collect info's
|
204
|
-
@@workers.list.map { |thread| thread.job }
|
205
|
-
end
|
206
|
-
|
207
|
-
def delete_job id
|
208
|
-
info "> delete_job(#{id})"
|
209
|
-
|
210
|
-
# Find jobs with this id
|
211
|
-
jobs = jobs_with_id id
|
212
|
-
|
213
|
-
# Kill them
|
214
|
-
jobs.each{ |thread| Thread.kill(thread) }
|
215
|
-
|
216
|
-
# Return the first one
|
217
|
-
return nil if jobs.empty?
|
218
|
-
jobs.first.job
|
219
|
-
end
|
220
|
-
|
221
|
-
def find_job id
|
222
|
-
info "> find_job(#{id})"
|
223
|
-
|
224
|
-
# Find jobs with this id
|
225
|
-
jobs = jobs_with_id id
|
226
|
-
|
227
|
-
# Return the first one
|
228
|
-
return nil if jobs.empty?
|
229
|
-
jobs.first.job
|
230
|
-
end
|
231
|
-
|
232
|
-
def jobs_with_id id
|
233
|
-
info "> find_jobs_by_id(#{id})"
|
234
|
-
@@workers.list.select{ |thread| thread[:id].to_s == id.to_s }
|
235
|
-
end
|
236
|
-
|
237
|
-
def new_job context = {}
|
238
|
-
info "new_job"
|
239
|
-
|
240
|
-
# Generate name
|
241
|
-
@@last_worker_id +=1
|
242
|
-
host = @@hostname.split('.')[0]
|
243
|
-
worker_id = @@last_worker_id
|
244
|
-
worker_name = "#{host}-#{Process.pid.to_s}-#{worker_id}"
|
245
|
-
info "new_job: creating thread [#{worker_name}]"
|
246
|
-
|
247
|
-
# Parse parameters
|
248
|
-
job_source = context["source"]
|
249
|
-
job_target = context["target"]
|
250
|
-
return { code: ERR_REQ_SOURCE_MISSING, errmsg: :ERR_REQ_SOURCE_MISSING} if job_source.nil?
|
251
|
-
return { code: ERR_REQ_TARGET_MISSING, errmsg: :ERR_REQ_TARGET_MISSING} if job_target.nil?
|
252
|
-
|
253
|
-
# Parse dest URI
|
254
|
-
target = URI(job_target)
|
255
|
-
info target.scheme
|
256
|
-
return { code: ERR_REQ_TARGET_SCHEME, errmsg: :ERR_REQ_TARGET_SCHEME} unless target.scheme == "ftp"
|
257
|
-
|
258
|
-
# Create thread
|
259
|
-
job = Thread.new(worker_id, worker_name, job) do
|
260
|
-
# Tnitialize thread
|
261
|
-
Thread.abort_on_exception = true
|
262
|
-
job_status :initializing
|
263
|
-
job_error ERR_OK
|
264
|
-
|
265
|
-
# Initialize job info
|
266
|
-
Thread.current[:job] = {}
|
267
|
-
Thread.current[:job].merge! context if context.is_a? Enumerable
|
268
|
-
Thread.current[:id] = worker_id
|
269
|
-
job_set :worker_name, worker_name
|
270
|
-
job_set :created, Time.now
|
271
|
-
|
272
|
-
# Do the job
|
273
|
-
info "new_job: thread running"
|
274
|
-
process_job
|
275
|
-
|
276
|
-
# Sleep a few seconds before dying
|
277
|
-
job_status :graceful_ending
|
278
|
-
sleep THREAD_SLEEP_BEFORE_DIE
|
279
|
-
job_status :ended
|
280
|
-
info "new_job: thread finished"
|
281
|
-
end
|
282
|
-
|
283
|
-
# Keep thread in thread group
|
284
|
-
info "new_job: attaching thread [#{worker_name}] to group"
|
285
|
-
@@workers.add job
|
286
|
-
|
287
|
-
return { code: 0, errmsg: 'success', worker_id: worker_id, context: context }
|
288
|
-
end
|
289
|
-
|
290
|
-
def info msg=""
|
291
|
-
@logger.info msg
|
292
|
-
end
|
293
|
-
|
294
|
-
def job_error error, errmsg = nil
|
295
|
-
job_set :error, error
|
296
|
-
job_set :errmsg, errmsg
|
297
|
-
end
|
298
|
-
def job_status status
|
299
|
-
job_set :status, status
|
300
|
-
end
|
301
|
-
|
302
|
-
def job_set attribute, value, thread = Thread.current
|
303
|
-
thread[:job][attribute] = value if thread[:job].is_a? Enumerable
|
304
|
-
end
|
305
|
-
|
306
|
-
|
307
|
-
end
|
308
|
-
|
309
|
-
# end
|
1
|
+
# Global libs
|
2
|
+
require 'json'
|
3
|
+
require 'grape'
|
4
|
+
require 'net/ftp'
|
5
|
+
|
6
|
+
# My libs
|
7
|
+
require 'rest-ftp-daemon/config'
|
8
|
+
require 'rest-ftp-daemon/exceptions'
|
9
|
+
require 'rest-ftp-daemon/job'
|
10
|
+
require 'rest-ftp-daemon/server'
|
@@ -0,0 +1,21 @@
|
|
1
|
+
module RestFtpDaemon
|
2
|
+
|
3
|
+
class RestFtpDaemonException < StandardError; end
|
4
|
+
|
5
|
+
class DummyException < RestFtpDaemonException; end
|
6
|
+
|
7
|
+
class RequestSourceMissing < RestFtpDaemonException; end
|
8
|
+
class RequestSourceNotFound < RestFtpDaemonException; end
|
9
|
+
class RequestTargetMissing < RestFtpDaemonException; end
|
10
|
+
class RequestTargetScheme < RestFtpDaemonException; end
|
11
|
+
|
12
|
+
class JobPrerequisitesNotMet < RestFtpDaemonException; end
|
13
|
+
|
14
|
+
class JobNotFound < RestFtpDaemonException; end
|
15
|
+
class JobSourceMissing < RestFtpDaemonException; end
|
16
|
+
class JobSourceNotFound < RestFtpDaemonException; end
|
17
|
+
class JobTargetMissing < RestFtpDaemonException; end
|
18
|
+
class JobTargetUnparseable < RestFtpDaemonException; end
|
19
|
+
class JobTargetPermission < RestFtpDaemonException; end
|
20
|
+
|
21
|
+
end
|
@@ -0,0 +1,173 @@
|
|
1
|
+
module RestFtpDaemon
|
2
|
+
class Job
|
3
|
+
|
4
|
+
def initialize(id, params={})
|
5
|
+
# Grab params
|
6
|
+
@params = params
|
7
|
+
@target = nil
|
8
|
+
@source = nil
|
9
|
+
|
10
|
+
# Init context
|
11
|
+
set :id, id
|
12
|
+
set :started_at, Time.now
|
13
|
+
set :status, :initialized
|
14
|
+
end
|
15
|
+
|
16
|
+
def id
|
17
|
+
@params[:id]
|
18
|
+
end
|
19
|
+
|
20
|
+
def process
|
21
|
+
# Init
|
22
|
+
"process [#{@id}] starting"
|
23
|
+
set :status, :starting
|
24
|
+
set :error, 0
|
25
|
+
|
26
|
+
begin
|
27
|
+
# Validate job and params
|
28
|
+
prepare
|
29
|
+
|
30
|
+
# Process
|
31
|
+
transfer
|
32
|
+
|
33
|
+
rescue Net::FTPPermError
|
34
|
+
set :status, :failed
|
35
|
+
set :error, exception.class
|
36
|
+
|
37
|
+
rescue RestFtpDaemonException => exception
|
38
|
+
set :status, :failed
|
39
|
+
set :error, exception.class
|
40
|
+
|
41
|
+
# rescue Exception => exception
|
42
|
+
# set :status, :crashed
|
43
|
+
# set :error, exception.class
|
44
|
+
|
45
|
+
else
|
46
|
+
set :status, :finished
|
47
|
+
set :error, 0
|
48
|
+
end
|
49
|
+
|
50
|
+
end
|
51
|
+
|
52
|
+
def describe
|
53
|
+
# Update realtime info
|
54
|
+
w = wandering_time
|
55
|
+
set :wandering, w.round(2) unless w.nil?
|
56
|
+
|
57
|
+
# Update realtime info
|
58
|
+
u = up_time
|
59
|
+
set :uptime, u.round(2) unless u.nil?
|
60
|
+
|
61
|
+
# Return the whole structure
|
62
|
+
@params
|
63
|
+
end
|
64
|
+
|
65
|
+
def wander time
|
66
|
+
@wander_for = time
|
67
|
+
@wander_started = Time.now
|
68
|
+
sleep time
|
69
|
+
end
|
70
|
+
|
71
|
+
def status text
|
72
|
+
@status = text
|
73
|
+
end
|
74
|
+
|
75
|
+
protected
|
76
|
+
|
77
|
+
def up_time
|
78
|
+
return if @params[:started_at].nil?
|
79
|
+
Time.now - @params[:started_at]
|
80
|
+
end
|
81
|
+
|
82
|
+
def wandering_time
|
83
|
+
return if @wander_started.nil? || @wander_for.nil?
|
84
|
+
@wander_for.to_f - (Time.now - @wander_started)
|
85
|
+
end
|
86
|
+
|
87
|
+
def exception_handler(actor, reason)
|
88
|
+
set :status, :crashed
|
89
|
+
set :error, reason
|
90
|
+
end
|
91
|
+
|
92
|
+
def set attribute, value
|
93
|
+
return unless @params.is_a? Enumerable
|
94
|
+
@params[:updated_at] = Time.now
|
95
|
+
@params[attribute] = value
|
96
|
+
end
|
97
|
+
|
98
|
+
def prepare
|
99
|
+
# Init
|
100
|
+
set :status, :preparing
|
101
|
+
|
102
|
+
# Check source
|
103
|
+
raise JobSourceMissing unless @params["source"]
|
104
|
+
@source = File.expand_path(@params["source"])
|
105
|
+
set :debug_source, @source
|
106
|
+
raise JobSourceNotFound unless File.exists? @source
|
107
|
+
|
108
|
+
# Check target
|
109
|
+
raise JobTargetMissing unless @params["target"]
|
110
|
+
@target = URI(@params["target"]) rescue nil
|
111
|
+
set :debug_target, @target.inspect
|
112
|
+
raise JobTargetUnparseable if @target.nil?
|
113
|
+
end
|
114
|
+
|
115
|
+
def transfer_fake
|
116
|
+
# Init
|
117
|
+
set :status, :faking
|
118
|
+
|
119
|
+
# Work
|
120
|
+
(0..9).each do |i|
|
121
|
+
set :faking, i
|
122
|
+
sleep 0.5
|
123
|
+
end
|
124
|
+
end
|
125
|
+
|
126
|
+
def transfer
|
127
|
+
# Init
|
128
|
+
transferred = 0
|
129
|
+
|
130
|
+
# Ensure @source and @target are there
|
131
|
+
set :status, :checking_source
|
132
|
+
raise JobPrerequisitesNotMet unless @source
|
133
|
+
raise JobPrerequisitesNotMet unless @source
|
134
|
+
target_path = File.dirname @target.path
|
135
|
+
target_name = File.basename @target.path
|
136
|
+
|
137
|
+
# Read source file size
|
138
|
+
source_size = File.size @source
|
139
|
+
set :file_size, source_size
|
140
|
+
|
141
|
+
# Prepare FTP transfer
|
142
|
+
set :status, :checking_target
|
143
|
+
ftp = Net::FTP.new(@target.host)
|
144
|
+
ftp.passive = true
|
145
|
+
ftp.login
|
146
|
+
ftp.chdir(target_path)
|
147
|
+
|
148
|
+
# Check for target file presence
|
149
|
+
results = ftp.list(target_name)
|
150
|
+
#info "ftp.list: #{results}"
|
151
|
+
unless results.count.zero?
|
152
|
+
ftp.close
|
153
|
+
raise JobTargetPermission
|
154
|
+
end
|
155
|
+
|
156
|
+
# Do transfer
|
157
|
+
set :status, :uploading
|
158
|
+
ftp.putbinaryfile(@source, target_name, TRANSFER_CHUNK_SIZE) do |block|
|
159
|
+
# Update counters
|
160
|
+
transferred += block.bytesize
|
161
|
+
|
162
|
+
# Update job info
|
163
|
+
percent = (100.0 * transferred / source_size).round(1)
|
164
|
+
set :file_progress, percent
|
165
|
+
set :file_sent, transferred
|
166
|
+
end
|
167
|
+
|
168
|
+
# Close FTP connexion
|
169
|
+
ftp.close
|
170
|
+
end
|
171
|
+
|
172
|
+
end
|
173
|
+
end
|
@@ -0,0 +1,237 @@
|
|
1
|
+
module RestFtpDaemon
|
2
|
+
|
3
|
+
class API < Grape::API
|
4
|
+
version 'v1', using: :header, vendor: 'ftven'
|
5
|
+
format :json
|
6
|
+
|
7
|
+
|
8
|
+
######################################################################
|
9
|
+
####### INIT
|
10
|
+
######################################################################
|
11
|
+
def initialize
|
12
|
+
# Setup logger
|
13
|
+
@@logger = Logger.new(APP_LOGTO, 'daily')
|
14
|
+
# @@queue = Queue.new
|
15
|
+
|
16
|
+
# Create new thread group
|
17
|
+
@@threads = ThreadGroup.new
|
18
|
+
|
19
|
+
# Other stuff
|
20
|
+
@@last_worker_id = 0
|
21
|
+
super
|
22
|
+
end
|
23
|
+
|
24
|
+
######################################################################
|
25
|
+
####### HELPERS
|
26
|
+
######################################################################
|
27
|
+
helpers do
|
28
|
+
def api_error exception
|
29
|
+
{
|
30
|
+
:error => exception.class,
|
31
|
+
:errmsg => exception.message,
|
32
|
+
:backtrace => exception.backtrace.first,
|
33
|
+
#:backtrace => exception.backtrace,
|
34
|
+
}
|
35
|
+
end
|
36
|
+
|
37
|
+
def info msg=""
|
38
|
+
@@logger.info msg
|
39
|
+
end
|
40
|
+
|
41
|
+
def threads_with_id job_id
|
42
|
+
@@threads.list.select do |thread|
|
43
|
+
next unless thread[:job].is_a? Job
|
44
|
+
thread[:job].id == job_id
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def job_describe job_id
|
49
|
+
# Find threads with tihs id
|
50
|
+
threads = threads_with_id job_id
|
51
|
+
raise RestFtpDaemon::JobNotFound if threads.empty?
|
52
|
+
|
53
|
+
# Find first job with tihs id
|
54
|
+
job = threads.first[:job]
|
55
|
+
raise RestFtpDaemon::JobNotFound unless job.is_a? Job
|
56
|
+
description = job.describe
|
57
|
+
|
58
|
+
# Return job description
|
59
|
+
description
|
60
|
+
end
|
61
|
+
|
62
|
+
def job_delete job_id
|
63
|
+
# Find threads with tihs id
|
64
|
+
threads = threads_with_id job_id
|
65
|
+
raise RestFtpDaemon::JobNotFound if threads.empty?
|
66
|
+
|
67
|
+
# Get description just before terminating the job
|
68
|
+
job = threads.first[:job]
|
69
|
+
raise RestFtpDaemon::JobNotFound unless job.is_a? Job
|
70
|
+
description = job.describe
|
71
|
+
|
72
|
+
# Kill those threads
|
73
|
+
threads.each do |t|
|
74
|
+
Thread.kill(t)
|
75
|
+
end
|
76
|
+
|
77
|
+
# Return job description
|
78
|
+
description
|
79
|
+
end
|
80
|
+
|
81
|
+
def job_list
|
82
|
+
@@threads.list.map do |thread|
|
83
|
+
next unless thread[:job].is_a? Job
|
84
|
+
thread[:job].describe
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
88
|
+
end
|
89
|
+
|
90
|
+
|
91
|
+
######################################################################
|
92
|
+
####### API DEFINITION
|
93
|
+
######################################################################
|
94
|
+
|
95
|
+
# Spawn a new thread for this new job
|
96
|
+
# post '/push' do
|
97
|
+
# @@queue << rand(999)
|
98
|
+
# end
|
99
|
+
|
100
|
+
# Server global status
|
101
|
+
get '/' do
|
102
|
+
info "GET /"
|
103
|
+
|
104
|
+
status 200
|
105
|
+
{
|
106
|
+
app_name: APP_NAME,
|
107
|
+
hostname: `hostname`.chomp,
|
108
|
+
version: RestFtpDaemon::VERSION,
|
109
|
+
started: APP_STARTED,
|
110
|
+
uptime: (Time.now - APP_STARTED).round(1),
|
111
|
+
}
|
112
|
+
end
|
113
|
+
|
114
|
+
# Server test
|
115
|
+
get '/test' do
|
116
|
+
info "GET /tests"
|
117
|
+
begin
|
118
|
+
raise RestFtpDaemon::DummyException
|
119
|
+
rescue RestFtpDaemon::RestFtpDaemonException => exception
|
120
|
+
status 501
|
121
|
+
api_error exception
|
122
|
+
rescue Exception => exception
|
123
|
+
status 501
|
124
|
+
api_error exception
|
125
|
+
else
|
126
|
+
status 200
|
127
|
+
{}
|
128
|
+
end
|
129
|
+
end
|
130
|
+
|
131
|
+
# List jobs
|
132
|
+
get "/jobs" do
|
133
|
+
info "GET /jobs"
|
134
|
+
begin
|
135
|
+
response = job_list
|
136
|
+
rescue RestFtpDaemonException => exception
|
137
|
+
status 501
|
138
|
+
api_error exception
|
139
|
+
rescue Exception => exception
|
140
|
+
status 501
|
141
|
+
api_error exception
|
142
|
+
else
|
143
|
+
status 200
|
144
|
+
response
|
145
|
+
end
|
146
|
+
end
|
147
|
+
|
148
|
+
# Get job info
|
149
|
+
get "/jobs/:id" do
|
150
|
+
info "GET /jobs/#{params[:id]}"
|
151
|
+
begin
|
152
|
+
response = job_describe params[:id].to_i
|
153
|
+
rescue RestFtpDaemon::JobNotFound => exception
|
154
|
+
status 404
|
155
|
+
api_error exception
|
156
|
+
rescue RestFtpDaemonException => exception
|
157
|
+
status 500
|
158
|
+
api_error exception
|
159
|
+
rescue Exception => exception
|
160
|
+
status 501
|
161
|
+
api_error exception
|
162
|
+
else
|
163
|
+
status 200
|
164
|
+
response
|
165
|
+
end
|
166
|
+
end
|
167
|
+
|
168
|
+
# Delete jobs
|
169
|
+
delete "/jobs/:id" do
|
170
|
+
info "DELETE /jobs/#{params[:name]}"
|
171
|
+
begin
|
172
|
+
response = job_delete params[:id].to_i
|
173
|
+
rescue RestFtpDaemon::JobNotFound => exception
|
174
|
+
status 404
|
175
|
+
api_error exception
|
176
|
+
rescue RestFtpDaemonException => exception
|
177
|
+
status 500
|
178
|
+
api_error exception
|
179
|
+
rescue Exception => exception
|
180
|
+
status 501
|
181
|
+
api_error exception
|
182
|
+
else
|
183
|
+
status 200
|
184
|
+
response
|
185
|
+
end
|
186
|
+
end
|
187
|
+
|
188
|
+
# Spawn a new thread for this new job
|
189
|
+
post '/jobs' do
|
190
|
+
info "POST /jobs: #{request.body.read}"
|
191
|
+
begin
|
192
|
+
# Extract params
|
193
|
+
request.body.rewind
|
194
|
+
params = JSON.parse request.body.read
|
195
|
+
|
196
|
+
# Create a new job
|
197
|
+
job_id = @@last_worker_id += 1
|
198
|
+
job = Job.new(job_id, params)
|
199
|
+
|
200
|
+
# Put it inside a thread
|
201
|
+
th = Thread.new(job) do |thread|
|
202
|
+
# Tnitialize thread
|
203
|
+
Thread.abort_on_exception = true
|
204
|
+
Thread.current[:job] = job
|
205
|
+
|
206
|
+
# Do the job
|
207
|
+
job.process
|
208
|
+
|
209
|
+
# Wait for a few seconds before cleaning up the job
|
210
|
+
job.wander RestFtpDaemon::THREAD_SLEEP_BEFORE_DIE
|
211
|
+
end
|
212
|
+
|
213
|
+
# Stack it to the pool
|
214
|
+
#@@queue << job
|
215
|
+
@@threads.add th
|
216
|
+
|
217
|
+
# And start it asynchronously
|
218
|
+
#job.future.process
|
219
|
+
|
220
|
+
rescue JSON::ParserError => exception
|
221
|
+
status 406
|
222
|
+
api_error exception
|
223
|
+
rescue RestFtpDaemonException => exception
|
224
|
+
status 412
|
225
|
+
api_error exception
|
226
|
+
rescue Exception => exception
|
227
|
+
status 501
|
228
|
+
api_error exception
|
229
|
+
else
|
230
|
+
status 201
|
231
|
+
job.describe
|
232
|
+
end
|
233
|
+
end
|
234
|
+
|
235
|
+
end
|
236
|
+
|
237
|
+
end
|
data/rest-ftp-daemon.gemspec
CHANGED
@@ -1,12 +1,11 @@
|
|
1
1
|
# coding: utf-8
|
2
2
|
lib = File.expand_path('../lib', __FILE__)
|
3
3
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
-
require '
|
4
|
+
require 'rest-ftp-daemon/config'
|
5
5
|
|
6
6
|
Gem::Specification.new do |spec|
|
7
7
|
spec.name = "rest-ftp-daemon"
|
8
|
-
spec.
|
9
|
-
spec.date = "2014-08-14"
|
8
|
+
spec.date = Time.now.strftime("%Y-%m-%d")
|
10
9
|
spec.authors = ["Bruno MEDICI"]
|
11
10
|
spec.email = "rest-ftp-daemon@bmconseil.com"
|
12
11
|
spec.description = "This is a pretty simple FTP client daemon, controlled through a RESTfull API"
|
@@ -15,38 +14,17 @@ Gem::Specification.new do |spec|
|
|
15
14
|
spec.licenses = ["MIT"]
|
16
15
|
|
17
16
|
spec.files = `git ls-files -z`.split("\x0")
|
18
|
-
#spec.executables = ["rest-ftp-daemon"]
|
19
|
-
#spec.executables = `git ls-files -- bin/*`.split('\n').map{ |f| File.basename(f) }
|
20
17
|
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
21
18
|
spec.require_paths = ["lib"]
|
19
|
+
spec.version = RestFtpDaemon::VERSION
|
22
20
|
|
23
|
-
|
24
|
-
spec.required_ruby_version = '>= 1.9'
|
21
|
+
spec.required_ruby_version = '>= 1.9.3'
|
25
22
|
|
26
23
|
spec.add_development_dependency "bundler", "~> 1.6"
|
27
24
|
spec.add_development_dependency "rake"
|
28
25
|
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
# "Gemfile",
|
33
|
-
# "Gemfile.lock",
|
34
|
-
# "LICENSE.txt",
|
35
|
-
# "README.md",
|
36
|
-
# "Rakefile",
|
37
|
-
# "VERSION",
|
38
|
-
# "bin/rest-ftp-daemon",
|
39
|
-
# "lib/config.rb",
|
40
|
-
# "lib/config.ru",
|
41
|
-
# "lib/errors.rb",
|
42
|
-
# "lib/extend_threads.rb",
|
43
|
-
# "lib/rest-ftp-daemon.rb",
|
44
|
-
# "rest-ftp-daemon.gemspec",
|
45
|
-
# "test/helper.rb",
|
46
|
-
# "test/test_rest-ftp-daemon.rb"
|
47
|
-
# ]
|
48
|
-
# spec.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
49
|
-
# spec.rubygems_version = "2.4.1"
|
26
|
+
spec.add_runtime_dependency "thin"
|
27
|
+
spec.add_runtime_dependency "grape"
|
28
|
+
spec.add_runtime_dependency "json"
|
50
29
|
|
51
30
|
end
|
52
|
-
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rest-ftp-daemon
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.30.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Bruno MEDICI
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-08-
|
11
|
+
date: 2014-08-29 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -38,6 +38,48 @@ dependencies:
|
|
38
38
|
- - ! '>='
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: thin
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ! '>='
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ! '>='
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: grape
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ! '>='
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ! '>='
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: json
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - ! '>='
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
type: :runtime
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ! '>='
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
41
83
|
description: This is a pretty simple FTP client daemon, controlled through a RESTfull
|
42
84
|
API
|
43
85
|
email: rest-ftp-daemon@bmconseil.com
|
@@ -52,14 +94,13 @@ files:
|
|
52
94
|
- LICENSE.txt
|
53
95
|
- README.md
|
54
96
|
- Rakefile
|
55
|
-
- VERSION
|
56
97
|
- bin/rest-ftp-daemon
|
57
|
-
-
|
58
|
-
- lib/config.ru
|
59
|
-
- lib/errors.rb
|
60
|
-
- lib/extend_threads.rb
|
98
|
+
- config.ru
|
61
99
|
- lib/rest-ftp-daemon.rb
|
62
|
-
- lib/
|
100
|
+
- lib/rest-ftp-daemon/config.rb
|
101
|
+
- lib/rest-ftp-daemon/exceptions.rb
|
102
|
+
- lib/rest-ftp-daemon/job.rb
|
103
|
+
- lib/rest-ftp-daemon/server.rb
|
63
104
|
- rest-ftp-daemon.gemspec
|
64
105
|
- test/helper.rb
|
65
106
|
- test/test_rest-ftp-daemon.rb
|
@@ -75,7 +116,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
75
116
|
requirements:
|
76
117
|
- - ! '>='
|
77
118
|
- !ruby/object:Gem::Version
|
78
|
-
version:
|
119
|
+
version: 1.9.3
|
79
120
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
80
121
|
requirements:
|
81
122
|
- - ! '>='
|
data/VERSION
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
0.9.0
|
data/lib/config.rb
DELETED
data/lib/config.ru
DELETED
@@ -1,12 +0,0 @@
|
|
1
|
-
# Main libs
|
2
|
-
require 'sinatra'
|
3
|
-
require 'sinatra/base'
|
4
|
-
require 'net/ftp'
|
5
|
-
require 'json'
|
6
|
-
|
7
|
-
# My local libs
|
8
|
-
Dir[APP_ROOT+"/lib/*.rb"].each {|file| require File.expand_path file }
|
9
|
-
#Dir[APP_ROOT+"/lib/*/*.rb"].each {|file| require File.expand_path file }
|
10
|
-
|
11
|
-
# Start application
|
12
|
-
run RestFtpDaemon
|
data/lib/errors.rb
DELETED
data/lib/extend_threads.rb
DELETED
data/lib/version.rb
DELETED