rest-ftp-daemon 0.222.0 → 0.230.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +1 -0
  3. data/CODE_OF_CONDUCT.md +13 -0
  4. data/Gemfile.lock +47 -20
  5. data/README.md +160 -94
  6. data/Rakefile +7 -1
  7. data/bin/rest-ftp-daemon +22 -3
  8. data/lib/rest-ftp-daemon.rb +25 -21
  9. data/lib/rest-ftp-daemon/constants.rb +19 -5
  10. data/lib/rest-ftp-daemon/exceptions.rb +2 -1
  11. data/lib/rest-ftp-daemon/helpers.rb +10 -5
  12. data/lib/rest-ftp-daemon/job.rb +181 -304
  13. data/lib/rest-ftp-daemon/job_queue.rb +5 -3
  14. data/lib/rest-ftp-daemon/logger.rb +4 -3
  15. data/lib/rest-ftp-daemon/logger_helper.rb +14 -10
  16. data/lib/rest-ftp-daemon/notification.rb +54 -43
  17. data/lib/rest-ftp-daemon/paginate.rb +2 -2
  18. data/lib/rest-ftp-daemon/path.rb +43 -0
  19. data/lib/rest-ftp-daemon/remote.rb +57 -0
  20. data/lib/rest-ftp-daemon/remote_ftp.rb +141 -0
  21. data/lib/rest-ftp-daemon/remote_sftp.rb +160 -0
  22. data/lib/rest-ftp-daemon/uri.rb +11 -4
  23. data/lib/rest-ftp-daemon/views/dashboard_table.haml +1 -1
  24. data/lib/rest-ftp-daemon/views/dashboard_workers.haml +1 -1
  25. data/lib/rest-ftp-daemon/worker.rb +10 -2
  26. data/lib/rest-ftp-daemon/worker_conchita.rb +12 -6
  27. data/lib/rest-ftp-daemon/worker_job.rb +8 -11
  28. data/rest-ftp-daemon.gemspec +6 -1
  29. data/rest-ftp-daemon.yml.sample +4 -2
  30. data/spec/rest-ftp-daemon/features/dashboard_spec.rb +8 -4
  31. data/spec/rest-ftp-daemon/features/jobs_spec.rb +68 -0
  32. data/spec/rest-ftp-daemon/features/routes_spec.rb +20 -0
  33. data/spec/rest-ftp-daemon/features/status_spec.rb +19 -0
  34. data/spec/spec_helper.rb +6 -2
  35. data/spec/support/config.yml +0 -1
  36. data/spec/support/request_helpers.rb +22 -0
  37. metadata +53 -3
  38. data/.ruby-version +0 -1
@@ -165,14 +165,16 @@ module RestFtpDaemon
165
165
  @waiting.size
166
166
  end
167
167
 
168
- def expire status, maxage
168
+ def expire status, maxage, verbose = false
169
169
  # FIXME: clean both @jobs and @queue
170
170
  # Init
171
171
  return if status.nil? || maxage <= 0
172
172
 
173
173
  # Compute oldest possible birthday
174
174
  before = Time.now - maxage.to_i
175
- # info "conchita_clean: SELECT status[#{status.to_s}] before[#{before}]"
175
+
176
+ # Verbose output ?
177
+ log_info "JobQueue.expire \t[#{status.to_s}] \tbefore \t[#{before}]" if verbose
176
178
 
177
179
  @mutex.synchronize do
178
180
  # Delete jobs from the queue when they match status and age limits
@@ -185,7 +187,7 @@ module RestFtpDaemon
185
187
 
186
188
  # Ok, we have to clean it up ..
187
189
  log_info "expire [#{status.to_s}] [#{maxage}] > [#{job.id}] [#{job.updated_at}]"
188
- log_info " & unqueued" if @queue.delete(job)
190
+ log_info "#{LOG_INDENT}unqueued" if @queue.delete(job)
189
191
 
190
192
  true
191
193
  end
@@ -22,9 +22,9 @@ class Logger
22
22
 
23
23
  end
24
24
 
25
-
26
25
  # Prepend plain message to output
27
- output.unshift (prefix1 + message.strip)
26
+ #output.unshift (prefix1 + message.strip)
27
+ output.unshift (prefix1 + message)
28
28
 
29
29
  # Send all this to logger
30
30
  add context[:level], output
@@ -43,7 +43,8 @@ class Logger
43
43
 
44
44
  def build_from_array prefix, lines
45
45
  lines.map do |value|
46
- text = value.to_s.strip[0..LOG_TRIM_LINE]
46
+ #text = value.to_s.strip[0..LOG_TRIM_LINE]
47
+ text = value.to_s[0..LOG_TRIM_LINE]
47
48
  "#{prefix}#{text}"
48
49
  end
49
50
  end
@@ -4,23 +4,27 @@ module RestFtpDaemon
4
4
  protected
5
5
 
6
6
  def log_info message, lines = []
7
- logger.info_with_id message, log_context.merge({
8
- from: self.class.to_s,
9
- lines: lines,
10
- level: Logger::INFO
11
- })
7
+ log message, lines, Logger::INFO
12
8
  end
13
9
 
14
10
  def log_error message, lines = []
15
- logger.info_with_id message, log_context.merge({
16
- from: self.class.to_s,
17
- lines: lines,
18
- level: Logger::ERROR
19
- })
11
+ log message, lines, Logger::ERROR
20
12
  end
21
13
 
22
14
  def log_context
23
15
  {}
24
16
  end
17
+
18
+ private
19
+
20
+ def log message, lines, level
21
+ context = log_context || {}
22
+ logger.info_with_id message, context.merge({
23
+ from: self.class.to_s,
24
+ lines: lines,
25
+ level: level
26
+ })
27
+ end
28
+
25
29
  end
26
30
  end
@@ -12,6 +12,10 @@ module RestFtpDaemon
12
12
  attr_accessor :job
13
13
 
14
14
  def initialize url, params
15
+ # Remember params
16
+ @url = url
17
+ @params = params
18
+
15
19
  # Generate a random key
16
20
  @id = Helpers.identifier(NOTIFY_IDENTIFIER_LEN)
17
21
  @jid = nil
@@ -19,64 +23,71 @@ module RestFtpDaemon
19
23
  # Logger
20
24
  @logger = RestFtpDaemon::LoggerPool.instance.get :notify
21
25
 
26
+ # Handle the notification
27
+ log_info "initialized [#{@url}]"
28
+ process
29
+ end
30
+
31
+ protected
32
+
33
+ def process
22
34
  # Check context
23
- if url.nil?
24
- log_info "skipping (missing url): #{params.inspect}"
35
+ if @url.nil?
36
+ log_info "skipping (missing url): #{@params.inspect}"
25
37
  return
26
- elsif params[:event].nil?
27
- log_info "skipping (missing event): #{params.inspect}"
38
+ elsif @params[:event].nil?
39
+ log_info "skipping (missing event): #{@params.inspect}"
28
40
  return
29
41
  end
30
42
 
31
43
  # Build body and extract job ID if provided
32
- body = {
33
- id: params[:id].to_s,
34
- signal: "#{NOTIFY_PREFIX}.#{params[:event]}",
35
- error: params[:error],
44
+ flags = {
45
+ id: @params[:id].to_s,
46
+ signal: "#{NOTIFY_PREFIX}.#{@params[:event]}",
47
+ error: @params[:error],
36
48
  host: Settings.host.to_s,
37
49
  }
38
- body[:status] = params[:status] if params[:status].is_a? Enumerable
39
- body[:message] = params[:message].to_s unless params[:message].nil?
40
- @jid = params[:id]
41
- log_info "initialized"
50
+ flags[:status] = @params[:status] if @params[:status].is_a? Enumerable
51
+ flags[:message] = @params[:message].to_s unless @params[:message].nil?
52
+ @jid = @params[:id]
42
53
 
43
-
44
- # Send message in a thread
54
+ # Spawn a dedicated thread
45
55
  Thread.new do
46
- # Prepare query
47
- uri = URI(url)
48
- headers = {
49
- "Content-Type" => "application/json",
50
- "Accept" => "application/json",
51
- "User-Agent" => "#{APP_NAME} - #{APP_VER}"
52
- }
53
- data = body.to_json
54
- log_info "sending #{data}"
55
-
56
-
57
- # Prepare HTTP client
58
- http = Net::HTTP.new uri.host, uri.port
59
- # http.initialize_http_header({'User-Agent' => APP_NAME})
60
-
61
- # Post notification
62
- response = http.post uri.path, data, headers
63
-
64
- # Handle server response / multi-lines
65
- response_lines = response.body.lines
66
-
67
- if response_lines.size > 1
68
- human_size = Helpers.format_bytes(response.body.bytesize, "B")
69
- #human_size = 0
70
- log_info "received [#{response.code}] #{human_size} (#{response_lines.size} lines)", response_lines
71
- else
72
- log_info "received [#{response.code}] #{response.body.strip}"
73
- end
56
+ send flags
57
+ end # end Thread
58
+ end
74
59
 
60
+ def send flags
61
+ # Prepare query
62
+ headers = {
63
+ "Content-Type" => "application/json",
64
+ "Accept" => "application/json",
65
+ "User-Agent" => "#{APP_NAME} - #{APP_VER}"
66
+ }
67
+ data = flags.to_json
68
+
69
+ # Send notification through HTTP
70
+ uri = URI @url
71
+ http = Net::HTTP.new uri.host, uri.port
72
+
73
+ # Post notification, handle server response / multi-lines
74
+ log_info "sending #{data}"
75
+ response = http.post uri.path, data, headers
76
+ response_lines = response.body.lines
77
+
78
+ if response_lines.size > 1
79
+ human_size = Helpers.format_bytes(response.body.bytesize, "B")
80
+ log_info "received [#{response.code}] #{human_size} (#{response_lines.size} lines)", response_lines
81
+ else
82
+ log_info "received [#{response.code}] #{response.body.strip}"
75
83
  end
76
84
 
85
+ # Handle exceptions
86
+ rescue StandardError => ex
87
+ log_error "EXCEPTION: #{ex.inspect}"
88
+
77
89
  end
78
90
 
79
- protected
80
91
 
81
92
  def log_context
82
93
  {
@@ -18,7 +18,7 @@ module RestFtpDaemon
18
18
  @total = @data.count
19
19
 
20
20
  # Count pages
21
- @pages = (@total.to_f / PAGINATE_MAX).ceil
21
+ @pages = (@total.to_f / DEFAULT_PAGE_SIZE).ceil
22
22
  @pages = 1 if @pages < 1
23
23
  end
24
24
 
@@ -39,7 +39,7 @@ module RestFtpDaemon
39
39
  end
40
40
 
41
41
  def subset
42
- size = PAGINATE_MAX.to_i
42
+ size = DEFAULT_PAGE_SIZE.to_i
43
43
  offset = (@page-1) * size
44
44
  @data[offset, size]
45
45
  end
@@ -0,0 +1,43 @@
1
+ module RestFtpDaemon
2
+ class Path
3
+ attr_accessor :name
4
+ attr_accessor :dir
5
+
6
+ def initialize full, strip_leading_slash = false
7
+ # Extract path parts
8
+ @name = extract_filename full.to_s
9
+ @dir = extract_dirname full.to_s
10
+
11
+ # Remove leading slash if needed
12
+ strip_leading_slash_from_dir! if strip_leading_slash
13
+ end
14
+
15
+ def full
16
+ return @name if @dir.nil? || @dir.empty?
17
+ return File.join @dir, @name
18
+ end
19
+
20
+ def size
21
+ File.size full if File.exists? full
22
+ end
23
+
24
+ private
25
+
26
+ def extract_filename path
27
+ # match everything that's after a slash at the end of the string
28
+ m = path.match /\/?([^\/]+)$/
29
+ return m[1].to_s unless m.nil?
30
+ end
31
+
32
+ def extract_dirname path
33
+ # match all the beginning of the string up to the last slash
34
+ m = path.match(/^(.*)\/[^\/]*$/)
35
+ return m[1].to_s unless m.nil?
36
+ end
37
+
38
+ def strip_leading_slash_from_dir!
39
+ @dir.to_s.gsub!(/^\//, '')
40
+ end
41
+
42
+ end
43
+ end
@@ -0,0 +1,57 @@
1
+ module RestFtpDaemon
2
+ class Remote
3
+ include LoggerHelper
4
+ attr_reader :logger
5
+ attr_reader :log_context
6
+
7
+ def initialize url, log_context, options = {}
8
+ # Logger
9
+ @log_context = log_context || {}
10
+ @logger = RestFtpDaemon::LoggerPool.instance.get :jobs
11
+
12
+ # Extract URL parts
13
+ @url = url
14
+ @url.user ||= "anonymous"
15
+
16
+ # Annnounce object
17
+ log_info "Remote.initialize [#{url.to_s}]"
18
+ end
19
+
20
+ def connect
21
+ # Debug mode ?
22
+ debug_header if @debug
23
+ end
24
+
25
+ def close
26
+ # Debug mode ?
27
+ puts "-------------------- SESSION CLOSING --------------------------" if @debug
28
+ end
29
+
30
+ private
31
+
32
+ # def log_context
33
+ # @log_context
34
+ # end
35
+
36
+ def myname
37
+ self.class.to_s
38
+ end
39
+
40
+ def debug_header
41
+ # Output header to STDOUT
42
+ puts
43
+ puts "-------------------- SESSION STARTING -------------------------"
44
+ #puts "job id\t #{@id}"
45
+ #puts "source\t #{@source}"
46
+ #puts "target\t #{@target}"
47
+ puts "class\t #{myname}"
48
+ #puts "class\t #{myname}"
49
+ puts "host\t #{@url.host}"
50
+ puts "user\t #{@url.user}"
51
+ puts "port\t #{@url.port}"
52
+ puts "options\t #{@options.inspect}"
53
+ puts "---------------------------------------------------------------"
54
+ end
55
+
56
+ end
57
+ end
@@ -0,0 +1,141 @@
1
+ module RestFtpDaemon
2
+ class RemoteFTP < Remote
3
+ attr_reader :ftp
4
+
5
+ def initialize url, log_context, options = {}
6
+ # Call super
7
+ super
8
+
9
+ # Use debug ?
10
+ @debug = (Settings.at :debug, :ftp) == true
11
+
12
+ # Create FTP object
13
+ if options[:ftpes]
14
+ prepare_ftpes
15
+ else
16
+ prepare_ftp
17
+ end
18
+ @ftp.passive = true
19
+ @ftp.debug_mode = !!@debug
20
+
21
+ # Config
22
+ @chunk_size = DEFAULT_FTP_CHUNK.to_i * 1024
23
+
24
+ # Announce object
25
+ log_info "RemoteFTP.initialize chunk_size:#{@chunk_size}"
26
+ end
27
+
28
+ def connect
29
+ # Connect init
30
+ super
31
+
32
+ # Connect remote server
33
+ @ftp.connect @url.host, @url.port
34
+ @ftp.login @url.user, @url.password
35
+ end
36
+
37
+ def present? target
38
+ size = @ftp.size target.full
39
+ log_info "RemoteFTP.present? [#{target.name}]"
40
+
41
+ rescue Net::FTPPermError
42
+ # log_info "RemoteFTP.present? [#{target.name}] NOT_FOUND"
43
+ return false
44
+ else
45
+ return size
46
+ end
47
+
48
+ def remove! target
49
+ log_info "RemoteFTP.remove! [#{target.name}]"
50
+ @ftp.delete target.full
51
+ rescue Net::FTPPermError
52
+ log_info "#{LOG_INDENT}[#{target.name}] file not found"
53
+ else
54
+ log_info "#{LOG_INDENT}[#{target.name}] removed"
55
+ end
56
+
57
+ def mkdir directory
58
+ log_info "RemoteFTP.mkdir [#{directory}]"
59
+ @ftp.mkdir directory
60
+
61
+ rescue
62
+ raise JobTargetPermissionError
63
+ end
64
+
65
+ def chdir_or_create directory, mkdir = false
66
+ # Init, extract my parent name and my own name
67
+ log_info "RemoteFTP.chdir_or_create mkdir[#{mkdir}] dir[#{directory}]"
68
+ parent, current = Helpers.extract_parent(directory)
69
+
70
+ fulldir = "/#{directory}"
71
+
72
+ # Access this directory
73
+ begin
74
+ @ftp.chdir "/#{directory}"
75
+
76
+ rescue Net::FTPPermError => e
77
+ # If not allowed to create path, that's over, we're stuck
78
+ return false unless mkdir
79
+
80
+ #log_info "#{LOG_INDENT}upward [#{parent}]"
81
+ chdir_or_create parent, mkdir
82
+
83
+ # Now I was able to chdir into my parent, create the current directory
84
+ #log_info "#{LOG_INDENT}mkdir [#{directory}]"
85
+ mkdir "/#{directory}"
86
+
87
+ # Finally retry the chdir
88
+ retry
89
+ else
90
+ return true
91
+ end
92
+ end
93
+
94
+ def push source, target, tempname = nil, &callback
95
+ # Push init
96
+ raise RestFtpDaemon::JobAssertionFailed, "push/1" if @ftp.nil?
97
+
98
+ # Temp file if provided
99
+ destination = target.clone
100
+ destination.name = tempname if tempname
101
+
102
+ # Do the transfer
103
+ log_info "RemoteFTP.push to [#{destination.name}]"
104
+
105
+ @ftp.putbinaryfile source.full, target.name, @chunk_size do |data|
106
+ # Update the worker activity marker
107
+ #FIXME worker_is_still_active
108
+
109
+ # Update job status after this block transfer
110
+ yield data.bytesize, destination.name
111
+ end
112
+
113
+ end
114
+
115
+ def close
116
+ # Close init
117
+ super
118
+
119
+ # Close FTP connexion and free up memory
120
+ @ftp.close
121
+ end
122
+
123
+ def connected?
124
+ !@ftp.welcome.nil?
125
+ end
126
+
127
+ private
128
+
129
+ def prepare_ftp
130
+ @ftp = Net::FTP.new
131
+ end
132
+
133
+ def prepare_ftpes
134
+ @ftp = DoubleBagFTPS.new
135
+ @ftp.ssl_context = DoubleBagFTPS.create_ssl_context(verify_mode: OpenSSL::SSL::VERIFY_NONE)
136
+ @ftp.ftps_mode = DoubleBagFTPS::EXPLICIT
137
+ end
138
+
139
+
140
+ end
141
+ end