rest-ftp-daemon 0.300.3 → 0.302.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +12 -12
- data/README.md +12 -3
- data/bin/rest-ftp-daemon +14 -15
- data/config.ru +2 -8
- data/defaults.yml +83 -59
- data/lib/rest-ftp-daemon.rb +4 -8
- data/lib/rest-ftp-daemon/api/config.rb +3 -1
- data/lib/rest-ftp-daemon/api/dashboard.rb +1 -0
- data/lib/rest-ftp-daemon/api/debug.rb +2 -0
- data/lib/rest-ftp-daemon/api/job_presenter.rb +2 -0
- data/lib/rest-ftp-daemon/api/jobs.rb +2 -0
- data/lib/rest-ftp-daemon/api/root.rb +2 -0
- data/lib/rest-ftp-daemon/api/status.rb +1 -0
- data/lib/rest-ftp-daemon/constants.rb +5 -5
- data/lib/rest-ftp-daemon/job.rb +37 -37
- data/lib/rest-ftp-daemon/logger_pool.rb +39 -16
- data/lib/rest-ftp-daemon/notification.rb +26 -19
- data/lib/rest-ftp-daemon/remote.rb +16 -15
- data/lib/rest-ftp-daemon/remote_ftp.rb +2 -7
- data/lib/rest-ftp-daemon/remote_sftp.rb +1 -4
- data/lib/rest-ftp-daemon/worker_pool.rb +5 -15
- data/lib/rest-ftp-daemon/{worker_conchita.rb → workers/conchita.rb} +20 -22
- data/lib/rest-ftp-daemon/{worker_reporter.rb → workers/reporter.rb} +23 -30
- data/lib/rest-ftp-daemon/{worker_job.rb → workers/transfer.rb} +25 -35
- data/lib/shared/conf.rb +47 -41
- data/lib/{rest-ftp-daemon/worker.rb → shared/worker_base.rb} +42 -28
- data/rest-ftp-daemon.gemspec +5 -3
- data/spec/rest-ftp-daemon/features/dashboard_spec.rb +5 -5
- data/spec/spec_helper.rb +2 -2
- metadata +36 -24
- data/rest-ftp-daemon.sample.yml +0 -71
- data/spec/support/config.yml +0 -25
@@ -1,32 +1,45 @@
|
|
1
|
-
module
|
2
|
-
class
|
1
|
+
module Shared
|
2
|
+
class WorkerBase
|
3
3
|
include Shared::LoggerHelper
|
4
4
|
attr_reader :logger
|
5
|
+
attr_reader :pool
|
6
|
+
attr_reader :wid
|
5
7
|
|
6
8
|
def initialize wid, pool = nil
|
7
9
|
# Logger
|
8
10
|
@logger = RestFtpDaemon::LoggerPool.instance.get :workers
|
9
11
|
@log_worker_status_changes = true
|
10
12
|
|
11
|
-
#
|
12
|
-
@
|
13
|
+
# Configuration
|
14
|
+
@config = {}
|
13
15
|
|
14
16
|
# Set thread context
|
17
|
+
@pool = pool
|
18
|
+
@wid = wid
|
15
19
|
Thread.current.thread_variable_set :pool, pool
|
16
20
|
Thread.current.thread_variable_set :wid, wid
|
17
21
|
Thread.current.thread_variable_set :started_at, Time.now
|
18
22
|
worker_status WORKER_STATUS_STARTING
|
19
23
|
|
20
|
-
#
|
21
|
-
|
22
|
-
|
24
|
+
# Ask worker to init itself, and return if there are errors
|
25
|
+
if worker_init_result = worker_init
|
26
|
+
log_error "worker_init aborting: #{worker_init_result.inspect}", @config
|
27
|
+
else
|
28
|
+
# We're ok, let's start out loop
|
29
|
+
start_loop
|
30
|
+
end
|
31
|
+
end
|
23
32
|
|
24
33
|
protected
|
25
34
|
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
35
|
+
# Worker methods prototypes
|
36
|
+
def worker_init
|
37
|
+
end
|
38
|
+
def worker_after
|
39
|
+
end
|
40
|
+
def worker_process
|
41
|
+
end
|
42
|
+
def worker_config
|
30
43
|
end
|
31
44
|
|
32
45
|
def log_prefix
|
@@ -37,13 +50,16 @@ module RestFtpDaemon
|
|
37
50
|
]
|
38
51
|
end
|
39
52
|
|
40
|
-
def
|
41
|
-
|
42
|
-
|
43
|
-
def start
|
53
|
+
def start_loop
|
54
|
+
log_info "start_loop starting", @config
|
44
55
|
loop do
|
45
56
|
begin
|
46
|
-
work
|
57
|
+
# Do the hard work
|
58
|
+
worker_process
|
59
|
+
|
60
|
+
# Do the cleaning/sleeping stuff
|
61
|
+
worker_after
|
62
|
+
|
47
63
|
rescue StandardError => e
|
48
64
|
log_error "WORKER EXCEPTION: #{e.inspect}"
|
49
65
|
sleep 1
|
@@ -72,26 +88,24 @@ module RestFtpDaemon
|
|
72
88
|
Thread.current.thread_variable_set :updated_at, Time.now
|
73
89
|
end
|
74
90
|
|
75
|
-
|
76
|
-
|
77
|
-
def load_config wid
|
78
|
-
# My debug
|
79
|
-
@debug = (Conf.at :debug, wid) == true
|
91
|
+
def config_section key
|
92
|
+
# Debugging
|
80
93
|
@log_worker_status_changes = @debug
|
81
94
|
|
82
|
-
#
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
return log_info "#{self.class.name}: missing #{wid}/timer value"
|
95
|
+
# Set my configuration
|
96
|
+
if (Conf[key].is_a? Hash) && Conf[key]
|
97
|
+
@config = Conf[key]
|
98
|
+
else
|
99
|
+
log_error "missing [#{key}] configuration"
|
88
100
|
end
|
89
101
|
end
|
90
102
|
|
91
103
|
# NewRelic instrumentation
|
92
104
|
if Conf.newrelic_enabled?
|
93
105
|
include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
|
94
|
-
add_transaction_tracer :
|
106
|
+
add_transaction_tracer :worker_init, category: :task
|
107
|
+
add_transaction_tracer :worker_after, category: :task
|
108
|
+
add_transaction_tracer :worker_process, category: :task
|
95
109
|
end
|
96
110
|
|
97
111
|
end
|
data/rest-ftp-daemon.gemspec
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
Gem::Specification.new do |spec|
|
3
3
|
|
4
4
|
# Project version
|
5
|
-
spec.version = "0.
|
5
|
+
spec.version = "0.302.0"
|
6
6
|
|
7
7
|
# Project description
|
8
8
|
spec.name = "rest-ftp-daemon"
|
@@ -14,18 +14,18 @@ Gem::Specification.new do |spec|
|
|
14
14
|
spec.licenses = ["MIT"]
|
15
15
|
spec.date = Time.now.strftime("%Y-%m-%d")
|
16
16
|
|
17
|
+
|
17
18
|
# List files and executables
|
18
19
|
spec.files = `git ls-files -z`.split("\x0").reject{ |f| f == "dashboard.png"}
|
19
20
|
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
20
21
|
spec.require_paths = ["lib"]
|
21
|
-
spec.required_ruby_version = ">= 2.2"
|
22
|
+
spec.required_ruby_version = ">= 2.2.3"
|
22
23
|
|
23
24
|
|
24
25
|
# Development dependencies
|
25
26
|
spec.add_development_dependency "bundler", "~> 1.6"
|
26
27
|
spec.add_development_dependency "rake"
|
27
28
|
spec.add_development_dependency "rspec"
|
28
|
-
spec.add_development_dependency "http"
|
29
29
|
spec.add_development_dependency "rubocop", "~> 0.32.0"
|
30
30
|
spec.add_development_dependency "pry"
|
31
31
|
|
@@ -35,6 +35,8 @@ Gem::Specification.new do |spec|
|
|
35
35
|
spec.add_runtime_dependency "grape-entity"
|
36
36
|
spec.add_runtime_dependency "settingslogic"
|
37
37
|
spec.add_runtime_dependency "chamber"
|
38
|
+
spec.add_runtime_dependency "rest-client", "~> 1.8"
|
39
|
+
spec.add_runtime_dependency "api-auth"
|
38
40
|
spec.add_runtime_dependency "haml"
|
39
41
|
spec.add_runtime_dependency "json"
|
40
42
|
spec.add_runtime_dependency "net-sftp"
|
@@ -3,11 +3,11 @@ require "spec_helper"
|
|
3
3
|
describe "Dashboard", feature: true do
|
4
4
|
|
5
5
|
describe "GET #{MOUNT_BOARD}" do
|
6
|
-
context 'without a password' do
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
end
|
6
|
+
# context 'without a password' do
|
7
|
+
# it 'is forbidden' do
|
8
|
+
# expect(HTTP.accept(:json).get("http://localhost:#{RequestHelpers::PORT}").status).to eq 401
|
9
|
+
# end
|
10
|
+
# end
|
11
11
|
|
12
12
|
context "with a password" do
|
13
13
|
it "can be accessed" do
|
data/spec/spec_helper.rb
CHANGED
@@ -53,8 +53,8 @@ RSpec.configure do |config|
|
|
53
53
|
|
54
54
|
include RequestHelpers
|
55
55
|
|
56
|
-
def call_server command,
|
57
|
-
system(Pathname(__dir__).join("../bin/rest-ftp-daemon -e test -
|
56
|
+
def call_server command, port = RequestHelpers::PORT
|
57
|
+
system(Pathname(__dir__).join("../bin/rest-ftp-daemon -e test -p #{port} #{command}").to_s, chdir: __dir__) || fail("Could not #{command} server")
|
58
58
|
end
|
59
59
|
|
60
60
|
config.before :suite do
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rest-ftp-daemon
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.302.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Bruno MEDICI
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-06-
|
11
|
+
date: 2016-06-30 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -52,20 +52,6 @@ dependencies:
|
|
52
52
|
- - ">="
|
53
53
|
- !ruby/object:Gem::Version
|
54
54
|
version: '0'
|
55
|
-
- !ruby/object:Gem::Dependency
|
56
|
-
name: http
|
57
|
-
requirement: !ruby/object:Gem::Requirement
|
58
|
-
requirements:
|
59
|
-
- - ">="
|
60
|
-
- !ruby/object:Gem::Version
|
61
|
-
version: '0'
|
62
|
-
type: :development
|
63
|
-
prerelease: false
|
64
|
-
version_requirements: !ruby/object:Gem::Requirement
|
65
|
-
requirements:
|
66
|
-
- - ">="
|
67
|
-
- !ruby/object:Gem::Version
|
68
|
-
version: '0'
|
69
55
|
- !ruby/object:Gem::Dependency
|
70
56
|
name: rubocop
|
71
57
|
requirement: !ruby/object:Gem::Requirement
|
@@ -164,6 +150,34 @@ dependencies:
|
|
164
150
|
- - ">="
|
165
151
|
- !ruby/object:Gem::Version
|
166
152
|
version: '0'
|
153
|
+
- !ruby/object:Gem::Dependency
|
154
|
+
name: rest-client
|
155
|
+
requirement: !ruby/object:Gem::Requirement
|
156
|
+
requirements:
|
157
|
+
- - "~>"
|
158
|
+
- !ruby/object:Gem::Version
|
159
|
+
version: '1.8'
|
160
|
+
type: :runtime
|
161
|
+
prerelease: false
|
162
|
+
version_requirements: !ruby/object:Gem::Requirement
|
163
|
+
requirements:
|
164
|
+
- - "~>"
|
165
|
+
- !ruby/object:Gem::Version
|
166
|
+
version: '1.8'
|
167
|
+
- !ruby/object:Gem::Dependency
|
168
|
+
name: api-auth
|
169
|
+
requirement: !ruby/object:Gem::Requirement
|
170
|
+
requirements:
|
171
|
+
- - ">="
|
172
|
+
- !ruby/object:Gem::Version
|
173
|
+
version: '0'
|
174
|
+
type: :runtime
|
175
|
+
prerelease: false
|
176
|
+
version_requirements: !ruby/object:Gem::Requirement
|
177
|
+
requirements:
|
178
|
+
- - ">="
|
179
|
+
- !ruby/object:Gem::Version
|
180
|
+
version: '0'
|
167
181
|
- !ruby/object:Gem::Dependency
|
168
182
|
name: haml
|
169
183
|
requirement: !ruby/object:Gem::Requirement
|
@@ -333,22 +347,20 @@ files:
|
|
333
347
|
- lib/rest-ftp-daemon/views/dashboard_table.haml
|
334
348
|
- lib/rest-ftp-daemon/views/dashboard_tokens.haml
|
335
349
|
- lib/rest-ftp-daemon/views/dashboard_workers.haml
|
336
|
-
- lib/rest-ftp-daemon/worker.rb
|
337
|
-
- lib/rest-ftp-daemon/worker_conchita.rb
|
338
|
-
- lib/rest-ftp-daemon/worker_job.rb
|
339
350
|
- lib/rest-ftp-daemon/worker_pool.rb
|
340
|
-
- lib/rest-ftp-daemon/
|
351
|
+
- lib/rest-ftp-daemon/workers/conchita.rb
|
352
|
+
- lib/rest-ftp-daemon/workers/reporter.rb
|
353
|
+
- lib/rest-ftp-daemon/workers/transfer.rb
|
341
354
|
- lib/shared/conf.rb
|
342
355
|
- lib/shared/logger_formatter.rb
|
343
356
|
- lib/shared/logger_helper.rb
|
357
|
+
- lib/shared/worker_base.rb
|
344
358
|
- rest-ftp-daemon.gemspec
|
345
|
-
- rest-ftp-daemon.sample.yml
|
346
359
|
- spec/rest-ftp-daemon/features/dashboard_spec.rb
|
347
360
|
- spec/rest-ftp-daemon/features/debug_spec.rb
|
348
361
|
- spec/rest-ftp-daemon/features/jobs_spec.rb
|
349
362
|
- spec/rest-ftp-daemon/features/status_spec.rb
|
350
363
|
- spec/spec_helper.rb
|
351
|
-
- spec/support/config.yml
|
352
364
|
- spec/support/request_helpers.rb
|
353
365
|
homepage: http://github.com/bmedici/rest-ftp-daemon
|
354
366
|
licenses:
|
@@ -362,7 +374,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
362
374
|
requirements:
|
363
375
|
- - ">="
|
364
376
|
- !ruby/object:Gem::Version
|
365
|
-
version:
|
377
|
+
version: 2.2.3
|
366
378
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
367
379
|
requirements:
|
368
380
|
- - ">="
|
@@ -370,7 +382,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
370
382
|
version: '0'
|
371
383
|
requirements: []
|
372
384
|
rubyforge_project:
|
373
|
-
rubygems_version: 2.5.1
|
385
|
+
rubygems_version: 2.4.5.1
|
374
386
|
signing_key:
|
375
387
|
specification_version: 4
|
376
388
|
summary: RESTful FTP client daemon
|
data/rest-ftp-daemon.sample.yml
DELETED
@@ -1,71 +0,0 @@
|
|
1
|
-
defaults: &defaults
|
2
|
-
daemonize: true
|
3
|
-
port: 3000
|
4
|
-
user: rftpd
|
5
|
-
group: rftpd
|
6
|
-
# host: "myhost"
|
7
|
-
# pidfile: "/tmp/rftpd.pid"
|
8
|
-
|
9
|
-
pools:
|
10
|
-
default: 2
|
11
|
-
|
12
|
-
transfer:
|
13
|
-
# notify_after_sec: 5 # wait at least X seconds between HTTP notifications
|
14
|
-
# mkdir: true # build directory tree if missing
|
15
|
-
# tempfile: true # transfer to temporary file, rename after sucessful transfer
|
16
|
-
# overwrite: false # overwrite any target file with the same name
|
17
|
-
# timeout: 1800 # jobs running for longer than X seconds will be killed
|
18
|
-
|
19
|
-
retry:
|
20
|
-
# on_errors:
|
21
|
-
# - ftp_perm_error
|
22
|
-
# - net_temp_error
|
23
|
-
# - conn_reset_by_peer
|
24
|
-
# - conn_timed_out
|
25
|
-
# - conn_refused
|
26
|
-
# - sftp_auth_failed
|
27
|
-
# - conn_host_is_down
|
28
|
-
# - conn_unreachable
|
29
|
-
# - conn_failed
|
30
|
-
# - conn_openssl_error
|
31
|
-
# max_runs: 5
|
32
|
-
# max_age: 1800
|
33
|
-
# delay: 10
|
34
|
-
|
35
|
-
conchita:
|
36
|
-
# timer: 60
|
37
|
-
# garbage_collector: true
|
38
|
-
# clean_failed: 3600
|
39
|
-
# clean_finished: 3600
|
40
|
-
# clean_queued: 86400
|
41
|
-
|
42
|
-
newrelic:
|
43
|
-
licence: ""
|
44
|
-
# prefix: "rftpd" # app prefix
|
45
|
-
platform: "bigbusiness" # app platform name
|
46
|
-
# app_name: "rftpd-bigbusiness-dev" # nickname used for naming app
|
47
|
-
|
48
|
-
debug:
|
49
|
-
ftp: false
|
50
|
-
sftp: false
|
51
|
-
conchita: false
|
52
|
-
reporter: fakse
|
53
|
-
allow_reload: false
|
54
|
-
|
55
|
-
logs:
|
56
|
-
thin: "/var/log/rftpd-environment-thin.log"
|
57
|
-
queue: "/var/log/rftpd-environment-core.log"
|
58
|
-
api: "/var/log/rftpd-environment-core.log"
|
59
|
-
workers: "/var/log/rftpd-environment-work.log"
|
60
|
-
jobs: "/var/log/rftpd-environment-work.log"
|
61
|
-
notify: "/var/log/rftpd-environment-work.log"
|
62
|
-
newrelic:"/var/log/rftpd-environment-newrelic.log"
|
63
|
-
|
64
|
-
|
65
|
-
development:
|
66
|
-
<<: *defaults
|
67
|
-
port: 3400
|
68
|
-
|
69
|
-
production:
|
70
|
-
<<: *defaults
|
71
|
-
port: 3200
|
data/spec/support/config.yml
DELETED
@@ -1,25 +0,0 @@
|
|
1
|
-
test:
|
2
|
-
daemonize: true
|
3
|
-
port: 5678
|
4
|
-
workers: 2
|
5
|
-
adminpwd: "admin"
|
6
|
-
host: local
|
7
|
-
|
8
|
-
transfer:
|
9
|
-
notify_after_sec: 5
|
10
|
-
mkdir: true
|
11
|
-
tempfile: true
|
12
|
-
overwrite: false
|
13
|
-
timeout: 1800
|
14
|
-
|
15
|
-
debug:
|
16
|
-
ftp: false
|
17
|
-
|
18
|
-
logs:
|
19
|
-
thin: "/tmp/rftpd-tests-thin.log"
|
20
|
-
queue: "/tmp/rftpd-tests-core.log"
|
21
|
-
api: "/tmp/rftpd-tests-core.log"
|
22
|
-
workers: "/tmp/rftpd-tests-work.log"
|
23
|
-
jobs: "/tmp/rftpd-tests-work.log"
|
24
|
-
notify: "/tmp/rftpd-tests-work.log"
|
25
|
-
newrelic: "/tmp/rftpd-tests-newrelic.log"
|