rest-ftp-daemon 0.222.0 → 0.230.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +1 -0
  3. data/CODE_OF_CONDUCT.md +13 -0
  4. data/Gemfile.lock +47 -20
  5. data/README.md +160 -94
  6. data/Rakefile +7 -1
  7. data/bin/rest-ftp-daemon +22 -3
  8. data/lib/rest-ftp-daemon.rb +25 -21
  9. data/lib/rest-ftp-daemon/constants.rb +19 -5
  10. data/lib/rest-ftp-daemon/exceptions.rb +2 -1
  11. data/lib/rest-ftp-daemon/helpers.rb +10 -5
  12. data/lib/rest-ftp-daemon/job.rb +181 -304
  13. data/lib/rest-ftp-daemon/job_queue.rb +5 -3
  14. data/lib/rest-ftp-daemon/logger.rb +4 -3
  15. data/lib/rest-ftp-daemon/logger_helper.rb +14 -10
  16. data/lib/rest-ftp-daemon/notification.rb +54 -43
  17. data/lib/rest-ftp-daemon/paginate.rb +2 -2
  18. data/lib/rest-ftp-daemon/path.rb +43 -0
  19. data/lib/rest-ftp-daemon/remote.rb +57 -0
  20. data/lib/rest-ftp-daemon/remote_ftp.rb +141 -0
  21. data/lib/rest-ftp-daemon/remote_sftp.rb +160 -0
  22. data/lib/rest-ftp-daemon/uri.rb +11 -4
  23. data/lib/rest-ftp-daemon/views/dashboard_table.haml +1 -1
  24. data/lib/rest-ftp-daemon/views/dashboard_workers.haml +1 -1
  25. data/lib/rest-ftp-daemon/worker.rb +10 -2
  26. data/lib/rest-ftp-daemon/worker_conchita.rb +12 -6
  27. data/lib/rest-ftp-daemon/worker_job.rb +8 -11
  28. data/rest-ftp-daemon.gemspec +6 -1
  29. data/rest-ftp-daemon.yml.sample +4 -2
  30. data/spec/rest-ftp-daemon/features/dashboard_spec.rb +8 -4
  31. data/spec/rest-ftp-daemon/features/jobs_spec.rb +68 -0
  32. data/spec/rest-ftp-daemon/features/routes_spec.rb +20 -0
  33. data/spec/rest-ftp-daemon/features/status_spec.rb +19 -0
  34. data/spec/spec_helper.rb +6 -2
  35. data/spec/support/config.yml +0 -1
  36. data/spec/support/request_helpers.rb +22 -0
  37. metadata +53 -3
  38. data/.ruby-version +0 -1
@@ -0,0 +1,160 @@
1
+ module RestFtpDaemon
2
+ class RemoteSFTP < Remote
3
+ attr_reader :sftp
4
+
5
+ def initialize url, log_context, options = {}
6
+ # Call super
7
+ super
8
+
9
+ # Use debug ?
10
+ @debug = (Settings.at :debug, :sftp) == true
11
+
12
+ # Announce object
13
+ log_info "RemoteSFTP.initialize"
14
+ end
15
+
16
+ def connect
17
+ # Connect init
18
+ super
19
+ log_info "RemoteSFTP.connect [#{@url.user}]@[#{@url.host}]:[#{@url.port}]"
20
+
21
+ # Debug level
22
+ verbosity = @debug ? Logger::INFO : false
23
+
24
+ # Connect remote server
25
+ @sftp = Net::SFTP.start(@url.host, @url.user, password: @url.password, verbose: verbosity)
26
+ end
27
+
28
+ def present? target
29
+ log_info "RemoteSFTP.present? [#{target.name}]"
30
+ stat = @sftp.stat! target.full
31
+ size = "?"
32
+
33
+ rescue Net::SFTP::StatusException
34
+ return false
35
+ else
36
+ return stat.size
37
+ end
38
+
39
+ # def remove target
40
+ # log_info "RemoteSFTP.remove [#{target.name}]"
41
+ # @sftp.remove target.full
42
+ # end
43
+
44
+ def remove! target
45
+ log_info "RemoteSFTP.remove! [#{target.name}]"
46
+ @sftp.remove target.full
47
+
48
+ rescue Net::SFTP::StatusException
49
+ log_info "#{LOG_INDENT}[#{target.name}] file not found"
50
+ else
51
+ log_info "#{LOG_INDENT}[#{target.name}] removed"
52
+ end
53
+
54
+ def mkdir directory
55
+ log_info "RemoteSFTP.mkdir [#{directory}]"
56
+ @sftp.mkdir! directory
57
+
58
+ rescue
59
+ raise JobTargetPermissionError
60
+ end
61
+
62
+ def chdir_or_create directory, mkdir = false
63
+ # Init, extract my parent name and my own name
64
+ log_info "RemoteSFTP.chdir_or_create mkdir[#{mkdir}] dir[#{directory}]"
65
+ parent, current = Helpers.extract_parent(directory)
66
+
67
+ # Access this directory
68
+ begin
69
+ # log_info " chdir [/#{directory}]"
70
+ handle = @sftp.opendir! "./#{directory}"
71
+
72
+ rescue Net::SFTP::StatusException => e
73
+ # If not allowed to create path, that's over, we're stuck
74
+ return false unless mkdir
75
+
76
+ # Recurse upward
77
+ #log_info "#{LOG_INDENT}upward [#{parent}]"
78
+ chdir_or_create parent, mkdir
79
+
80
+ # Now I was able to chdir into my parent, create the current directory
81
+ #log_info "#{LOG_INDENT}mkdir [#{directory}]"
82
+ mkdir directory
83
+
84
+ # Finally retry the chdir
85
+ retry
86
+ else
87
+ return true
88
+ end
89
+
90
+ # We should never get here
91
+ raise JobTargetShouldBeDirectory
92
+ end
93
+
94
+ # def dir_contents directory
95
+ # # Access this directory
96
+ # handle = @sftp.opendir! directory
97
+ # @sftp.readdir! handle
98
+ # end
99
+
100
+ def push source, target, tempname = nil, &callback
101
+ # Push init
102
+ raise RestFtpDaemon::JobAssertionFailed, "push/1" if @sftp.nil?
103
+
104
+ # Temp file if provided
105
+ destination = target.clone
106
+ destination.name = tempname if tempname
107
+
108
+ # Do the transfer
109
+ log_info "RemoteSFTP.push [#{destination.full}]"
110
+ @sftp.upload! source.full, destination.full do |event, uploader, *args|
111
+ case event
112
+ when :open then
113
+ # args[0] : file metadata
114
+ when :put then
115
+ # args[0] : file metadata
116
+ # args[1] : byte offset in remote file
117
+ # args[2] : data being written (as string)
118
+ # puts "writing #{args[2].length} bytes to #{args[0].remote} starting at #{args[1]}"
119
+
120
+ # Update the worker activity marker
121
+ #FIXME worker_is_still_active
122
+
123
+ # Update job status after this block transfer
124
+ yield args[2].length, destination.name
125
+
126
+ when :close then
127
+ # args[0] : file metadata
128
+ when :mkdir
129
+ # args[0] : remote path name
130
+ when :finish
131
+ end
132
+
133
+ end
134
+
135
+ # flags = 0x0001 + 0x0002
136
+ flags = 0x00000001
137
+
138
+ # Rename if needed
139
+ if tempname
140
+ log_info "RemoteSFTP.push rename to\t[#{target.name}]"
141
+ @sftp.rename! destination.full, target.full, flags
142
+ end
143
+
144
+ # progress:
145
+ # Net::SFTP::StatusException
146
+ end
147
+
148
+ def close
149
+ # Close init
150
+ super
151
+
152
+ # @sftp.close
153
+ end
154
+
155
+ def connected?
156
+ @sftp && !@sftp.closed?
157
+ end
158
+
159
+ end
160
+ end
@@ -1,12 +1,19 @@
1
1
  module URI
2
+
2
3
  class FTPS < Generic
3
4
  DEFAULT_PORT = 21
4
5
  end
5
- @@schemes["FTPS"] = FTPS
6
- end
7
- module URI
6
+
8
7
  class FTPES < Generic
9
- DEFAULT_PORT = 990
8
+ # DEFAULT_PORT = 990
9
+ DEFAULT_PORT = 21
10
+ end
11
+
12
+ class SFTP < Generic
13
+ DEFAULT_PORT = 22
10
14
  end
15
+
16
+ @@schemes["FTPS"] = FTPS
11
17
  @@schemes["FTPES"] = FTPES
18
+ @@schemes["SFTP"] = SFTP
12
19
  end
@@ -6,7 +6,7 @@
6
6
  - presented = present job, with: RestFtpDaemon::API::Entities::JobPresenter, hide_params: true
7
7
  - bitrate = job.get :transfer_bitrate
8
8
 
9
- - trclass = JOB_STYLES[job.status]
9
+ - trclass = DASHBOARD_JOB_STYLES[job.status]
10
10
 
11
11
  - unless job.error.nil?
12
12
  - trclass = "warning"
@@ -12,7 +12,7 @@
12
12
  - wid = vars[:wid]
13
13
  - status = vars[:status]
14
14
  - alive = $pool.worker_alive? wid
15
- - trclass = WORKER_STYLES[status]
15
+ - trclass = DASHBOARD_WORKER_STYLES[status]
16
16
 
17
17
  - unless alive
18
18
  - trclass = "danger"
@@ -10,6 +10,7 @@ module RestFtpDaemon
10
10
  def initialize wid
11
11
  # Logger
12
12
  @logger = RestFtpDaemon::LoggerPool.instance.get :workers
13
+ @log_worker_status_changes = true
13
14
 
14
15
  # Worker name
15
16
  @wid = wid
@@ -17,7 +18,7 @@ module RestFtpDaemon
17
18
  # Set thread context
18
19
  Thread.current.thread_variable_set :wid, wid
19
20
  Thread.current.thread_variable_set :started_at, Time.now
20
- worker_status :starting
21
+ worker_status WORKER_STATUS_STARTING
21
22
  end
22
23
 
23
24
  protected
@@ -40,9 +41,16 @@ module RestFtpDaemon
40
41
  end
41
42
  end
42
43
 
43
- def worker_status status
44
+ def worker_status status, extra = ""
45
+ # Update thread variables
44
46
  Thread.current.thread_variable_set :status, status
45
47
  Thread.current.thread_variable_set :updted_at, Time.now
48
+
49
+ # Nothin' to log if "silent"
50
+ return unless @log_worker_status_changes
51
+
52
+ # Log this status change
53
+ log_info "worker: #{status} #{extra}"
46
54
  end
47
55
 
48
56
  def worker_jid jid
@@ -5,6 +5,10 @@ module RestFtpDaemon
5
5
  # Generic worker initialize
6
6
  super
7
7
 
8
+ # Use debug ?
9
+ @debug = (Settings.at :debug, :conchita) == true
10
+ @log_worker_status_changes = @debug
11
+
8
12
  # Conchita configuration
9
13
  @conchita = Settings.conchita
10
14
  if !@conchita.is_a? Hash
@@ -21,23 +25,25 @@ module RestFtpDaemon
21
25
  protected
22
26
 
23
27
  def work
24
- worker_status :cleaning
28
+ # Announce we are working
29
+ worker_status WORKER_STATUS_CLEANING
25
30
 
26
31
  # Cleanup queues according to configured max-age
27
- $queue.expire JOB_STATUS_FINISHED, maxage(JOB_STATUS_FINISHED)
28
- $queue.expire JOB_STATUS_FAILED, maxage(JOB_STATUS_FAILED)
29
- $queue.expire JOB_STATUS_QUEUED, maxage(JOB_STATUS_QUEUED)
32
+ $queue.expire JOB_STATUS_FINISHED, maxage(JOB_STATUS_FINISHED), @debug
33
+ $queue.expire JOB_STATUS_FAILED, maxage(JOB_STATUS_FAILED), @debug
34
+ $queue.expire JOB_STATUS_QUEUED, maxage(JOB_STATUS_QUEUED), @debug
30
35
 
31
36
  # Force garbage collector
32
- worker_status :collecting
33
37
  GC.start if @conchita["garbage_collector"]
34
38
 
35
39
  rescue StandardError => e
36
40
  log_error "EXCEPTION: #{e.inspect}"
37
41
  sleep 1
38
42
  else
43
+ # Restore previous status
44
+ worker_status WORKER_STATUS_WAITING
45
+
39
46
  # Sleep for a few seconds
40
- worker_status :sleeping
41
47
  sleep @conchita[:timer]
42
48
  end
43
49
 
@@ -17,14 +17,13 @@ module RestFtpDaemon
17
17
 
18
18
  def work
19
19
  # Wait for a job to come into the queue
20
- worker_status :waiting
21
- log_info "waiting for a job"
20
+ worker_status WORKER_STATUS_WAITING
21
+ #log_info "waiting"
22
22
  job = $queue.pop
23
23
 
24
24
  # Prepare the job for processing
25
- worker_status :working
25
+ worker_status WORKER_STATUS_RUNNING, "job [#{job.id}]"
26
26
  worker_jid job.id
27
- log_info "working with job [#{job.id}]"
28
27
  job.wid = Thread.current.thread_variable_get :wid
29
28
 
30
29
  # Processs this job protected by a timeout
@@ -33,8 +32,7 @@ module RestFtpDaemon
33
32
  end
34
33
 
35
34
  # Processing done
36
- worker_status :finished
37
- log_info "finished with job [#{job.id}]"
35
+ worker_status WORKER_STATUS_FINISHED, "job [#{job.id}]"
38
36
  worker_jid nil
39
37
  job.wid = nil
40
38
 
@@ -42,8 +40,8 @@ module RestFtpDaemon
42
40
  $queue.counter_inc :jobs_processed
43
41
 
44
42
  rescue RestFtpDaemon::JobTimeout => ex
45
- log_error "JOB TIMED OUT", lines: ex.backtrace
46
- worker_status :timeout
43
+ log_error "JOB TIMED OUT", ex.backtrace
44
+ worker_status WORKER_STATUS_TIMEOUT
47
45
  worker_jid nil
48
46
  job.wid = nil
49
47
 
@@ -51,14 +49,13 @@ module RestFtpDaemon
51
49
  sleep 1
52
50
 
53
51
  rescue StandardError => ex
54
- log_error "JOB UNHDNALED EXCEPTION: #{ex.message}", lines: ex.backtrace
55
- worker_status :crashed
52
+ log_error "JOB UNHDNALED EXCEPTION: #{ex.message}", ex.backtrace
53
+ worker_status WORKER_STATUS_CRASHED
56
54
  job.oops_after_crash ex unless job.nil?
57
55
  sleep 1
58
56
 
59
57
  else
60
58
  # Clean job status
61
- worker_status :free
62
59
  job.wid = nil
63
60
 
64
61
  end
@@ -16,7 +16,9 @@ Gem::Specification.new do |spec|
16
16
  spec.homepage = "http://github.com/bmedici/rest-ftp-daemon"
17
17
  spec.licenses = ["MIT"]
18
18
 
19
- spec.files = `git ls-files -z`.split("\x0")
19
+ spec.files = `git ls-files -z`.split("\x0").reject do |f|
20
+ f == 'dashboard.png'
21
+ end
20
22
  spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
21
23
  spec.require_paths = ["lib"]
22
24
  spec.version = APP_VER
@@ -28,6 +30,8 @@ Gem::Specification.new do |spec|
28
30
  spec.add_development_dependency "rake"
29
31
  spec.add_development_dependency "rspec", "~> 3.1"
30
32
  spec.add_development_dependency "http", "~> 0.8"
33
+ spec.add_development_dependency "rubocop", "~> 0.32.0"
34
+ spec.add_development_dependency "pry"
31
35
 
32
36
  spec.add_runtime_dependency "thin", "~> 1.6"
33
37
  spec.add_runtime_dependency "grape"
@@ -35,6 +39,7 @@ Gem::Specification.new do |spec|
35
39
  spec.add_runtime_dependency "settingslogic"
36
40
  spec.add_runtime_dependency "haml"
37
41
  spec.add_runtime_dependency "json"
42
+ spec.add_runtime_dependency "net-sftp"
38
43
  spec.add_runtime_dependency "double-bag-ftps"
39
44
  spec.add_runtime_dependency "facter"
40
45
  spec.add_runtime_dependency "sys-cpu"
@@ -4,10 +4,10 @@ defaults: &defaults
4
4
  workers: 2
5
5
  user: rftpd
6
6
  group: rftpd
7
- host: "myhost"
7
+ #host: "myhost"
8
+ #appname: "replay" # appname prefix used for NewRelic reporting
8
9
 
9
10
  transfer:
10
- update_every_kb: 1024 # block size to transfer between counter updates
11
11
  notify_after_sec: 5 # wait at least X seconds between HTTP notifications
12
12
  mkdir: true # build directory tree if missing
13
13
  tempfile: true # transfer to temporary file, rename after sucessful transfer
@@ -23,6 +23,8 @@ defaults: &defaults
23
23
 
24
24
  debug:
25
25
  ftp: false
26
+ sftp: false
27
+ conchita: false
26
28
  # newrelic: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
27
29
 
28
30
  logs:
@@ -5,19 +5,23 @@ describe 'Dashboard', feature: true do
5
5
  describe "GET /" do
6
6
  context 'without a password' do
7
7
  it 'is forbidden' do
8
- expect(HTTP.accept(:json).get("http://localhost:5678").status).to eq 401
8
+ expect(HTTP.accept(:json).get("http://localhost:#{RequestHelpers::PORT}").status).to eq 401
9
9
  end
10
10
  end
11
11
 
12
12
  context 'with a password' do
13
13
  it 'can be accessed' do
14
14
  expect(
15
- HTTP.accept(:json).
16
- basic_auth(user: 'admin', pass: 'admin').
17
- get("http://localhost:5678").status
15
+ get("/").status
18
16
  ).to eq 200
19
17
  end
20
18
  end
19
+
20
+ it "has an HTML representation" do
21
+ expect(
22
+ get("/", accept: 'html').status
23
+ ).to eq 200
24
+ end
21
25
  end # GET /
22
26
 
23
27
  end
@@ -0,0 +1,68 @@
1
+ require "spec_helper"
2
+
3
+ describe "Jobs", feature: true do
4
+
5
+ describe "GET /jobs" do
6
+ let!(:response) { get "/jobs" }
7
+
8
+ it "responds successfully" do
9
+ expect(response.status).to eq 200
10
+ end
11
+
12
+ it "exposes an array" do
13
+ expect(JSON.parse(response.body)).to be_an_instance_of(Array)
14
+ end
15
+ end # GET /jobs
16
+
17
+ describe "POST /jobs" do
18
+ def jobs_list
19
+ JSON.parse get("/jobs").body
20
+ end
21
+
22
+ context "when params are valid" do
23
+ let(:params) do
24
+ {
25
+ source: "/tmp/foo",
26
+ target: "/tmp/bar"
27
+ }
28
+ end
29
+
30
+ it "issues a 201 response" do
31
+ expect(
32
+ post("/jobs", json: params).status
33
+ ).to eq 201
34
+ end
35
+
36
+ it "exposes the new job id" do
37
+ response = JSON.parse post("/jobs", json: params)
38
+ expect(response['id']).not_to be_nil
39
+ end
40
+
41
+ it "assigns a status" do
42
+ response = JSON.parse post("/jobs", json: params)
43
+ expect(response['status']).to match(/^(queued|failed)$/)
44
+ end
45
+
46
+ it "creates a new job" do
47
+ expect {
48
+ post("/jobs", json: params)
49
+ }.to change { jobs_list.size }.by(1)
50
+ end
51
+ end
52
+ end # POST /jobs
53
+
54
+ describe "GET /jobs/:id" do
55
+ let(:creation_response) do
56
+ JSON.parse post("/jobs", json: {source: "/tmp/foo", target: "/tmp/bar"}).body
57
+ end
58
+
59
+ let(:job_id) { creation_response.fetch('id') }
60
+
61
+ it "is properly exposed" do
62
+ expect(
63
+ get("/jobs/#{job_id}").status
64
+ ).to eq 200
65
+ end
66
+ end # GET /jobs/:id
67
+
68
+ end