rest-ftp-daemon 0.85.2 → 0.90.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 1c6ed0547c0017afce76bb7b5ffc55e5ddb48981
4
- data.tar.gz: 3a40535718c17d96e76a2f332d7e71eef4b3e2ce
3
+ metadata.gz: e1f8cd7be9c2f0d1932b820c3cf38b7dc79de537
4
+ data.tar.gz: e001c9fb6c9299f3d1f49b15368fac6628fb0fd9
5
5
  SHA512:
6
- metadata.gz: 77d115e07981f5bf41d5ef0240e50c813f2a908c5015aff532538a8c56fa9fdf4e2bbb06cbee60e2a51199923f1e38655adbee68d549f28ecdf8866ab2ab6d2b
7
- data.tar.gz: 98e520e5b73394dd7f3cb187673b51650290fcd50fb9c19dd7badc89837814571f35550c08aef9e5c4a7de8534858f35e5b08390ea00f5ed0c29406b90f91662
6
+ metadata.gz: 0f37294ceee1337337cbc4013fd1f8fa08438d5b6ea11fc698cfa1c59442864f4b5cbe7444deedb265e549a5c9a0c32b2e7b4e845c61c1a659656206369f2045
7
+ data.tar.gz: 28c38f2a5e8618abf1f24773b91381f4623843b9629a190c967641c4980ebc78b9306d6ef26b4bee2da65b6ce616f40f043f5756892b06356b01c9589e15a4f5
data/.gitignore CHANGED
@@ -4,3 +4,4 @@ pkg
4
4
  *.log
5
5
  tmp/
6
6
  work/
7
+ rest-ftp-daemon.yml
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- rest-ftp-daemon (0.84)
4
+ rest-ftp-daemon (0.88)
5
5
  double-bag-ftps
6
6
  facter
7
7
  grape
data/README.md CHANGED
@@ -29,6 +29,8 @@ As of today, its main features are :
29
29
  * Automatically clean-up jobs after a configurable amount of time (failed, finished)
30
30
  * Current bitrate on the last blocks chunk updated in the job attributes
31
31
  * Global bitrate on the whole file transfer is re-computed after the transfer finishes
32
+ * Daemon process is tagged with its name and environement in process lists
33
+ * Allow basic patterns in source filename to match multiple files (/dir/file*.jpg)
32
34
 
33
35
  Expected features in a short-time range :
34
36
 
@@ -41,6 +43,9 @@ Expected features in a short-time range :
41
43
  * Allow to specify random remote/local source/target
42
44
  * Allow more transfer protocols (sFTP, HTTP POST etc)
43
45
 
46
+ Known bugs :
47
+
48
+ * As this project is based on SettingsLogic, which in turns uses Syck YAML parser, configuration merge from "defaults" section and environment-specific section is broken. A sub-tree defined for a specific environment, will overwrite the corresponding subtree from "defaults".
44
49
 
45
50
 
46
51
  Installation
@@ -168,6 +173,17 @@ curl -H "Content-Type: application/json" -X POST -D /dev/stdout -d \
168
173
  '{"source":"~/file.dmg","priority":"3", target":"ftp://anonymous@localhost/incoming/dest4.dmg","notify":"http://requestb.in/1321axg1"}' "http://localhost:3000/jobs"
169
174
  ```
170
175
 
176
+
177
+ * Get info about a job with ID="q89j.1"
178
+
179
+ Both parameters ```q89j.1``` and ```1``` will be accepted as ID in the API. Requests below are equivalent:
180
+
181
+ ```
182
+ GET http://localhost:3100/jobs/q89j.1
183
+ GET http://localhost:3100/jobs/1
184
+ ```
185
+
186
+
171
187
  About
172
188
  ------------------------------------------------------------------------------------
173
189
 
data/bin/rest-ftp-daemon CHANGED
@@ -1,6 +1,7 @@
1
1
  #!/usr/bin/env ruby
2
2
 
3
- # Try to load libs
3
+ # Try to load libs and constants
4
+ app_root = File.dirname(__FILE__) + '/../'
4
5
  begin
5
6
  require "thin"
6
7
  require 'optparse'
@@ -9,37 +10,24 @@ begin
9
10
  rescue LoadError
10
11
  raise "EXITING: some of basic libs were not found: thin, optparse, socket, timeout"
11
12
  end
13
+ require File.expand_path("#{app_root}/lib/rest-ftp-daemon/constants")
12
14
  puts
15
+ #puts "--- #{APP_NAME} #{APP_VER}"
13
16
 
14
- # Detect environment from options
15
- begin
16
- OptionParser.new do |opts|
17
- opts.on("-e", "--environment ENV") { |env| APP_ENV = env }
18
- opts.on("", "--dev") { APP_ENV = "development" }
19
- end.parse(ARGV)
20
- rescue OptionParser::InvalidOption => e
21
- end
22
-
23
- # Load remaining libs and config subsystem
24
- app_root = File.dirname(__FILE__) + '/../'
25
- [:constants, :helpers, :config].each do |lib|
26
- require File.expand_path("#{app_root}/lib/rest-ftp-daemon/#{lib.to_s}.rb")
27
- end
28
-
29
- # Setup options and import current ARGV
17
+ # Detect options from ARGV
30
18
  options = {}
31
19
  parser = OptionParser.new do |opts|
32
20
  opts.banner = "Usage: #{File.basename $0} [options] start|stop|restart"
33
- opts.on("-e", "--environment ENV", "Environment (#{APP_ENV if defined? APP_ENV})")
34
- opts.on("--dev", "Force development environment")
35
- opts.on("-p", "--port PORT", "use PORT (#{Settings.port})") { |port| options["port"] = port.to_i }
36
- opts.on("-w", "--workers COUNT", "Use COUNT worker threads (#{Settings['workers']})") { |count| options["workers"] = count.to_i }
37
- opts.on("-d", "--daemonize", "Run daemonized in the background") { |bool| options["daemonize"] = bool || true }
21
+ opts.on("-e", "--environment ENV") { |env| APP_ENV = env }
22
+ opts.on("", "--dev") { APP_ENV = "development" }
23
+ opts.on("-p", "--port PORT", "use PORT") { |port| options["port"] = port.to_i }
24
+ opts.on("-w", "--workers COUNT", "Use COUNT worker threads") { |count| options["workers"] = count.to_i }
25
+ opts.on("-d", "--daemonize", "Run daemonized in the background") { |bool| options["daemonize"] = bool }
38
26
  opts.on("-P", "--pid FILE", "File to store PID") { |file| options["pidfile"] = file }
39
27
  opts.on("-u", "--user NAME", "User to run daemon as (use with -g)") { |user| options["user"] = user }
40
28
  opts.on("-g", "--group NAME", "Group to run daemon as (use with -u)"){ |group| options["group"] = group }
41
29
  opts.on_tail("-h", "--help", "Show this message") { puts opts; exit }
42
- opts.on_tail('-v', '--version', "Show version") { puts Settings['app_ver']; exit }
30
+ opts.on_tail('-v', '--version', "Show version (#{APP_VER})") { puts APP_VER; exit }
43
31
  end
44
32
  begin
45
33
  parser.order!(ARGV)
@@ -52,6 +40,14 @@ rescue OptionParser::InvalidOption => e
52
40
  puts "EXITING: option parser: #{e.message}"
53
41
  exit 1
54
42
  end
43
+
44
+ # Load helpers and config
45
+ app_root = File.dirname(__FILE__) + '/../'
46
+ [:helpers, :config].each do |lib|
47
+ require File.expand_path("#{app_root}/lib/rest-ftp-daemon/#{lib.to_s}")
48
+ end
49
+
50
+ # Merge options from ARGV into settings
55
51
  Settings.merge!(options)
56
52
 
57
53
  # Display compiled configuration
@@ -82,6 +78,7 @@ argv = []
82
78
  argv << ["-e", Settings.namespace]
83
79
  argv << ["-p", Settings["port"].to_s] unless Settings["port"].nil?
84
80
  argv << ["--pid", Settings.pidfile]
81
+ argv << ["--tag", "'#{APP_NAME}/#{Settings.namespace}'"]
85
82
  argv << ["--log", Settings["logs"]["thin"].to_s] unless Settings["logs"]["thin"].nil? if Settings["logs"].is_a? Enumerable
86
83
  argv << ["--daemonize"] if [1, true].include? Settings["daemonize"]
87
84
  # User / group
@@ -94,17 +91,37 @@ puts "--- Thin ARGV"
94
91
  puts argv.flatten.join(' ')
95
92
  puts
96
93
 
97
- # Start Thin with this rackup configuration
94
+
95
+ # Start Thin with this rackup configuration, changing to app_root first
98
96
  begin
99
- Thin::Runner.new(argv.flatten).run!
100
- rescue RuntimeError => e
101
- puts "FAILED: RuntimeError: #{e.message}"
102
- rescue Thin::PidFileExist
103
- puts "FAILED: daemon was already running (Thin::PidFileExist)"
104
- rescue Thin::PidFileNotFound
105
- puts "FAILED: daemon was not running (Thin::PidFileNotFound)"
106
- rescue SystemExit
107
- # puts "EXITING: daemon in the background (SystemExit)"
108
- else
109
- # puts "FAILED: process ending"
97
+ Dir.chdir app_root
98
+ r = Thin::Runner.new(argv.flatten).run!
99
+ rescue RuntimeError => e
100
+ puts "FAILED: RuntimeError: #{e.message}"
101
+ rescue Thin::PidFileExist
102
+ puts "FAILED: daemon was already running (Thin::PidFileExist)"
103
+ rescue Thin::PidFileNotFound
104
+ puts "FAILED: daemon was not running (Thin::PidFileNotFound)"
105
+ rescue SystemExit
106
+ # Leave some time for the daemon to launch
107
+ pidfile = Settings.pidfile
108
+ print "Waiting for pidfile"
109
+ while !File.file?(pidfile) do
110
+ print "."
111
+ sleep 0.25
112
+ end
113
+ puts
114
+
115
+ # Check that this process is running
116
+ pid = File.read(pidfile).to_i
117
+ begin
118
+ Process.kill(0, pid)
119
+ puts "Process ##{pid} is running"
120
+ rescue Errno::EPERM # changed uid
121
+ puts "No permission to query process ##{pid}!";
122
+ rescue Errno::ESRCH
123
+ puts "Process ##{pid} is NOT running."; # or zombied
124
+ rescue
125
+ puts "Unable to determine status for ##{pid}: #{$!}"
126
+ end
110
127
  end
@@ -28,13 +28,6 @@ module RestFtpDaemon
28
28
 
29
29
  helpers do
30
30
 
31
- def next_job_id
32
- @@last_worker_id ||= 0
33
- @@last_worker_id += 1
34
- #Helpers.identifier(IDENT_JOB_BYTES)
35
- #SecureRandom.hex(IDENT_JOB_BYTES)
36
- end
37
-
38
31
  def threads_with_id job_id
39
32
  $threads.list.select do |thread|
40
33
  next unless thread[:job].is_a? Job
@@ -45,8 +38,13 @@ module RestFtpDaemon
45
38
  def job_describe job_id
46
39
  raise RestFtpDaemon::JobNotFound if ($queue.all_size==0)
47
40
 
48
- # Find job with this id
49
- found = $queue.all.select { |job| job.id == job_id }.first
41
+ # Find job with exactly this id
42
+ found = $queue.find_by_id(job_id)
43
+
44
+ # Find job with this id while searching with the current prefix
45
+ found = $queue.find_by_id(job_id, true) if found.nil?
46
+
47
+ # Check that we did find it
50
48
  raise RestFtpDaemon::JobNotFound if found.nil?
51
49
  raise RestFtpDaemon::JobNotFound unless found.is_a? Job
52
50
 
@@ -71,12 +69,16 @@ module RestFtpDaemon
71
69
 
72
70
  desc "Get information about a specific job"
73
71
  params do
74
- requires :id, type: Integer, desc: "job id"
72
+ requires :id, type: String, desc: "job id", regexp: /[^\/]+/
73
+ # optional :audio do
74
+ # requires :format, type: Symbol, values: [:mp3, :wav, :aac, :ogg], default: :mp3
75
+ # end
75
76
  end
76
- get ':id' do
77
+ get '*id' do
77
78
  info "GET /jobs/#{params[:id]}"
79
+ #return params
78
80
  begin
79
- response = job_describe params[:id].to_i
81
+ response = job_describe params[:id]
80
82
  rescue RestFtpDaemon::JobNotFound => exception
81
83
  status 404
82
84
  api_error exception
@@ -142,12 +144,17 @@ module RestFtpDaemon
142
144
  request.body.rewind
143
145
  params = JSON.parse(request.body.read, symbolize_names: true)
144
146
 
147
+
148
+ params[:priority] = rand(10)
149
+
145
150
  # Create a new job
146
151
  # job_id = $last_worker_id += 1
147
- job_id = next_job_id
152
+ job_id = $queue.generate_id
153
+ #job = Job.new(params)
148
154
  job = Job.new(job_id, params)
149
155
 
150
156
  # And push it to the queue
157
+ #$queue.push0 job
151
158
  $queue.push job
152
159
 
153
160
  rescue JSON::ParserError => exception
@@ -25,16 +25,6 @@ module RestFtpDaemon
25
25
  Root.logger.info(message, level)
26
26
  end
27
27
 
28
- def job_list_by_status
29
- statuses = {}
30
- alljobs = $queue.all.map do |item|
31
- next unless item.is_a? Job
32
- statuses[item.get_status] ||= 0
33
- statuses[item.get_status] +=1
34
- end
35
- statuses
36
- end
37
-
38
28
  end
39
29
 
40
30
 
@@ -69,7 +59,11 @@ module RestFtpDaemon
69
59
  end
70
60
 
71
61
  # Jobs to display
72
- all_jobs_in_queue = $queue.all
62
+ # all_jobs_in_queue = $queue.all
63
+ # all_jobs_in_queue = $queue.ordered_queue
64
+ # + $queue.popped
65
+ popped_jobs = $queue.ordered_popped.reverse
66
+ @jobs_queued = $queue.ordered_queue.reverse
73
67
 
74
68
  if params["only"].nil? || params["only"].blank?
75
69
  @only = nil
@@ -79,19 +73,15 @@ module RestFtpDaemon
79
73
 
80
74
  case @only
81
75
  when nil
82
- @jobs = all_jobs_in_queue
83
- when :queue
84
- @jobs = $queue.queued
76
+ @jobs_popped = popped_jobs
77
+ # when :queue
78
+ # @jobs_popped = $queue.queued
85
79
  else
86
- @jobs = $queue.by_status (@only)
80
+ @jobs_popped = $queue.popped_reverse_sorted_by_status @only
87
81
  end
88
82
 
89
83
  # Count jobs for each status
90
- @counts = {}
91
- grouped = all_jobs_in_queue.group_by { |job| job.get(:status) }
92
- grouped.each do |status, jobs|
93
- @counts[status] = jobs.size
94
- end
84
+ @counts = $queue.popped_counts_by_status
95
85
 
96
86
  # Get workers status
97
87
  @gworker_statuses = $pool.get_worker_statuses
@@ -118,7 +108,8 @@ module RestFtpDaemon
118
108
  config: Settings.to_hash,
119
109
  started: APP_STARTED,
120
110
  uptime: (Time.now - APP_STARTED).round(1),
121
- status: job_list_by_status,
111
+ counters: $queue.counters,
112
+ status: $queue.popped_counts_by_status,
122
113
  queue_size: $queue.all_size,
123
114
  jobs_queued: $queue.queued.collect(&:id),
124
115
  jobs_popped: $queue.popped.collect(&:id),
@@ -1,19 +1,19 @@
1
1
  # Terrific constants
2
2
  APP_NAME = "rest-ftp-daemon"
3
3
  APP_CONF = "/etc/#{APP_NAME}.yml"
4
- APP_VER = "0.85.2"
4
+ APP_VER = "0.90.1"
5
5
 
6
6
 
7
7
  # Some global constants
8
+ IDENT_JOB_LEN = 4
8
9
  IDENT_NOTIF_LEN = 4
9
10
  IDENT_RANDOM_LEN = 8
10
11
 
11
-
12
12
  # Some defaults
13
13
  DEFAULT_CONNECT_TIMEOUT_SEC = 30
14
14
  DEFAULT_UPDATE_EVERY_KB = 2048
15
15
  DEFAULT_WORKERS = 1
16
- DEFAULT_LOGS_PROGNAME_TRIM = 9
16
+ DEFAULT_LOGS_PROGNAME_TRIM = 12
17
17
 
18
18
 
19
19
  # Initialize markers
@@ -9,12 +9,14 @@ module RestFtpDaemon
9
9
 
10
10
  class JobException < RestFtpDaemonException; end
11
11
  class JobNotFound < RestFtpDaemonException; end
12
+ class JobUnresolvedTokens < RestFtpDaemonException; end
12
13
  class JobAssertionFailed < RestFtpDaemonException; end
13
14
  class JobMissingAttribute < RestFtpDaemonException; end
14
15
  class JobSourceNotFound < RestFtpDaemonException; end
15
16
  class JobTargetUnsupported < RestFtpDaemonException; end
16
17
  class JobTargetUnparseable < RestFtpDaemonException; end
17
18
  class JobTargetFileExists < RestFtpDaemonException; end
19
+ class JobTargetShouldBeDirectory< RestFtpDaemonException; end
18
20
  class JobTooManyOpenFiles < RestFtpDaemonException; end
19
21
 
20
22
  end
@@ -1,6 +1,5 @@
1
1
  require 'securerandom'
2
2
 
3
-
4
3
  module RestFtpDaemon
5
4
  class Helpers
6
5
 
@@ -16,12 +15,20 @@ module RestFtpDaemon
16
15
  def self.identifier len
17
16
  rand(36**len).to_s(36)
18
17
  end
19
- #SecureRandom.hex(IDENT_JOB_BYTES)
20
18
 
21
19
  def self.tokenize(item)
22
20
  "[#{item}]"
23
21
  end
24
22
 
23
+ def self.contains_tokens(item)
24
+ /\[[a-zA-Z0-9]+\]/.match(item)
25
+ end
26
+
27
+ def self.extract_filename path
28
+ m = path.match /\/([^\/]+)$/
29
+ return m[1] unless m.nil?
30
+ end
31
+
25
32
  def self.local_port_used? port
26
33
  ip = '0.0.0.0'
27
34
  timeout = 1
@@ -41,6 +48,23 @@ module RestFtpDaemon
41
48
  end
42
49
  end
43
50
 
51
+ def self.job_method_label method
52
+ return if method.nil?
53
+ klass = case method
54
+ when :file
55
+ 'label-primary'
56
+ when :ftp
57
+ 'label-danger'
58
+ when :ftps
59
+ 'label-warning'
60
+ when :ftpes
61
+ 'label-warning'
62
+ else
63
+ 'label-default'
64
+ end
65
+ "<div class=\"transfer-method label #{klass}\">#{method.upcase}</div>"
66
+ end
67
+
44
68
  # def snakecase
45
69
  # gsub(/::/, '/').
46
70
  # gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
@@ -7,35 +7,36 @@ require 'timeout'
7
7
 
8
8
  module RestFtpDaemon
9
9
  class Job < RestFtpDaemon::Common
10
+ attr_accessor :id
10
11
  attr_accessor :wid
11
12
 
12
- def initialize(id, params={})
13
+ def initialize job_id, params={}
13
14
  # Call super
14
15
  # super()
15
16
  info "Job.initialize"
16
17
 
17
- # Generate new Job.id
18
- # $queue.counter_add :transferred, source_size
19
-
20
- # Logger
21
- @logger = RestFtpDaemon::Logger.new(:workers, "JOB #{id}")
18
+ # Init context
19
+ @params = params
20
+ @id = job_id.to_s
21
+ #set :id, job_id
22
22
 
23
23
  # Protect with a mutex
24
24
  @mutex = Mutex.new
25
25
 
26
- # Init context
27
- @params = params
28
- set :id, id
26
+ # Logger
27
+ @logger = RestFtpDaemon::Logger.new(:workers, "JOB #{id}")
28
+
29
+ # Flag current job
29
30
  set :started_at, Time.now
30
31
  status :created
31
32
 
32
33
  # Send first notification
33
- info "Job.initialize/notify"
34
+ #info "Job.initialize/notify"
34
35
  notify "rftpd.queued"
35
36
  end
36
37
 
37
38
  def id
38
- get :id
39
+ @id
39
40
  end
40
41
 
41
42
  def priority
@@ -61,15 +62,21 @@ module RestFtpDaemon
61
62
  rescue RestFtpDaemon::JobSourceNotFound => exception
62
63
  return oops "rftpd.started", exception, :job_source_not_found
63
64
 
65
+ rescue RestFtpDaemon::JobUnresolvedTokens => exception
66
+ return oops "rftpd.started", exception, :job_unresolved_tokens
67
+
68
+ rescue RestFtpDaemon::JobTargetUnparseable => exception
69
+ return oops "rftpd.started", exception, :job_target_unparseable
70
+
71
+ rescue RestFtpDaemon::JobAssertionFailed => exception
72
+ return oops "rftpd.started", exception, :job_assertion_failed
73
+
64
74
  rescue RestFtpDaemon::RestFtpDaemonException => exception
65
75
  return oops "rftpd.started", exception, :job_prepare_failed
66
76
 
67
77
  rescue Exception => exception
68
78
  return oops "rftpd.started", exception, :job_prepare_unhandled, true
69
79
 
70
- rescue exception
71
- return oops "rftpd.started", exception, :WOUHOU, true
72
-
73
80
  else
74
81
  # Update job's status
75
82
  info "Job.process/prepare ok"
@@ -90,21 +97,30 @@ module RestFtpDaemon
90
97
  status :starting
91
98
  transfer
92
99
 
93
- rescue Timeout::Error => exception
94
- return oops "rftpd.ended", exception, :job_timeout_error
95
-
96
- rescue Net::FTPPermError => exception
97
- return oops "rftpd.ended", exception, :job_ftp_perm_error
100
+ rescue Errno::EHOSTDOWN => exception
101
+ return oops "rftpd.ended", exception, :job_host_is_down
98
102
 
99
103
  rescue Errno::ECONNREFUSED => exception
100
104
  return oops "rftpd.ended", exception, :job_connexion_refused
101
105
 
106
+ rescue Timeout::Error, Errno::ETIMEDOUT => exception
107
+ return oops "rftpd.ended", exception, :job_timeout
108
+
109
+ rescue Net::FTPPermError => exception
110
+ return oops "rftpd.ended", exception, :job_perm_error
111
+
102
112
  rescue Errno::EMFILE => exception
103
113
  return oops "rftpd.ended", exception, :job_too_many_open_files
104
114
 
105
115
  rescue RestFtpDaemon::JobTargetFileExists => exception
106
116
  return oops "rftpd.ended", exception, :job_target_file_exists
107
117
 
118
+ rescue RestFtpDaemon::JobTargetShouldBeDirectory => exception
119
+ return oops "rftpd.ended", exception, :job_target_should_be_directory
120
+
121
+ rescue RestFtpDaemon::JobAssertionFailed => exception
122
+ return oops "rftpd.started", exception, :job_assertion_failed
123
+
108
124
  rescue RestFtpDaemon::RestFtpDaemonException => exception
109
125
  return oops "rftpd.ended", exception, :job_transfer_failed
110
126
 
@@ -124,11 +140,14 @@ module RestFtpDaemon
124
140
 
125
141
  def describe
126
142
  # Update realtime info
127
- u = up_time
128
- set :uptime, u.round(2) unless u.nil?
143
+ #u = up_time
144
+ #set :uptime, u.round(2) unless u.nil?
129
145
 
130
146
  # Return the whole structure FIXME
131
- @params
147
+ @params.merge({
148
+ id: @id,
149
+ uptime: up_time.round(2)
150
+ })
132
151
  # @mutex.synchronize do
133
152
  # out = @params.clone
134
153
  # end
@@ -144,7 +163,7 @@ module RestFtpDaemon
144
163
  protected
145
164
 
146
165
  def up_time
147
- return if @params[:started_at].nil?
166
+ return 0 if @params[:started_at].nil?
148
167
  Time.now - @params[:started_at]
149
168
  end
150
169
 
@@ -175,14 +194,14 @@ module RestFtpDaemon
175
194
  end
176
195
 
177
196
  def expand_path path
178
- File.expand_path replace_token(path)
197
+ File.expand_path replace_tokens(path)
179
198
  end
180
199
 
181
200
  def expand_url path
182
- URI::parse replace_token(path) rescue nil
201
+ URI::parse replace_tokens(path) rescue nil
183
202
  end
184
203
 
185
- def replace_token path
204
+ def replace_tokens path
186
205
  # Ensure endpoints are not a nil value
187
206
  return path unless Settings.endpoints.is_a? Enumerable
188
207
  vectors = Settings.endpoints.clone
@@ -194,10 +213,14 @@ module RestFtpDaemon
194
213
  newpath = path.clone
195
214
  vectors.each do |from, to|
196
215
  next if to.to_s.blank?
197
- #info "Job.replace_token #{Helpers.tokenize(from)} > #{to}"
216
+ #info "Job.replace_tokens #{Helpers.tokenize(from)} > #{to}"
198
217
  newpath.gsub! Helpers.tokenize(from), to
199
218
  end
200
219
 
220
+ # Ensure result does not contain tokens after replacement
221
+ raise RestFtpDaemon::JobUnresolvedTokens if Helpers.contains_tokens newpath
222
+
223
+ # All OK, return this
201
224
  return newpath
202
225
  end
203
226
 
@@ -232,18 +255,26 @@ module RestFtpDaemon
232
255
  # Check compliance
233
256
  raise RestFtpDaemon::JobTargetUnparseable if @target_url.nil?
234
257
  raise RestFtpDaemon::JobTargetUnsupported if @target_method.nil?
235
- raise RestFtpDaemon::JobSourceNotFound unless File.exists? @source_path
258
+ #raise RestFtpDaemon::JobSourceNotFound unless File.exists? @source_path
236
259
  end
237
260
 
238
261
  def transfer
239
- # Method assertions
240
- info "Job.transfer checking_source"
262
+ # Method assertions and init
241
263
  status :checking_source
242
- raise RestFtpDaemon::JobAssertionFailed unless @source_path && @target_url
264
+ raise RestFtpDaemon::JobAssertionFailed unless @source_path && @target_url
265
+ @transfer_sent = 0
266
+ set :transfer_source_done, 0
243
267
 
244
- # Init
268
+ # Guess source file names using Dir.glob
269
+ source_matches = Dir.glob @source_path
270
+ info "Job.transfer sources: #{source_matches}"
271
+ raise RestFtpDaemon::JobSourceNotFound if source_matches.empty?
272
+ set :transfer_source_count, source_matches.count
273
+ set :transfer_source_files, source_matches
245
274
 
246
- target_name = File.basename @target_url.path
275
+ # Guess target file name, and fail if present while we matched multiple sources
276
+ target_name = Helpers.extract_filename @target_url.path
277
+ raise RestFtpDaemon::JobTargetShouldBeDirectory if target_name && source_matches.count>1
247
278
 
248
279
  # Scheme-aware config
249
280
  ftp_init
@@ -251,14 +282,25 @@ module RestFtpDaemon
251
282
  # Connect remote server, login and chdir
252
283
  ftp_connect
253
284
 
254
- # Check for target file presence
255
- if get(:overwrite).nil? && (ftp_presence target_name)
256
- @ftp.close
257
- raise RestFtpDaemon::JobTargetFileExists
285
+ # Check source files presence and compute total size, they should be there, coming from Dir.glob()
286
+ @transfer_total = 0
287
+ source_matches.each do |filename|
288
+ # @ftp.close
289
+ raise RestFtpDaemon::JobSourceNotFound unless File.exists? filename
290
+ @transfer_total += File.size filename
291
+ end
292
+ set :transfer_total, @transfer_total
293
+
294
+ # Handle each source file matched, and start a transfer
295
+ done = 0
296
+ source_matches.each do |filename|
297
+ ftp_transfer filename, target_name
298
+ done += 1
299
+ set :transfer_source_done, done
258
300
  end
259
301
 
260
- # Do transfer
261
- ftp_transfer target_name
302
+ # Add total transferred to counter
303
+ $queue.counter_add :transferred, @transfer_total
262
304
 
263
305
  # Close FTP connexion
264
306
  info "Job.transfer disconnecting"
@@ -320,9 +362,8 @@ module RestFtpDaemon
320
362
  info "Job.ftp_connect connect"
321
363
  status :ftp_connect
322
364
  raise RestFtpDaemon::JobAssertionFailed if @ftp.nil? || @target_url.nil?
323
-
324
- ret = @ftp.connect(@target_url.host)
325
- @ftp.passive = true
365
+ @ftp.connect(@target_url.host)
366
+ @ftp.passive = true
326
367
 
327
368
  info "Job.ftp_connect login"
328
369
  status :ftp_login
@@ -336,55 +377,76 @@ module RestFtpDaemon
336
377
 
337
378
  def ftp_presence target_name
338
379
  # Method assertions
339
- info "Job.ftp_presence"
340
380
  status :ftp_presence
341
381
  raise RestFtpDaemon::JobAssertionFailed if @ftp.nil? || @target_url.nil?
342
382
 
343
383
  # Get file list, sometimes the response can be an empty value
344
384
  results = @ftp.list(target_name) rescue nil
385
+ info "Job.ftp_presence: #{results.inspect}"
345
386
 
346
387
  # Result can be nil or a list of files
347
388
  return false if results.nil?
348
389
  return results.count >0
349
390
  end
350
391
 
351
- def ftp_transfer target_name
392
+ def ftp_transfer source_match, target_name = nil
393
+ #target_name
352
394
  # Method assertions
353
- info "Job.ftp_transfer starting"
395
+ info "Job.ftp_transfer source_match: #{source_match}"
396
+ raise RestFtpDaemon::JobAssertionFailed if @ftp.nil?
397
+ raise RestFtpDaemon::JobAssertionFailed if source_match.nil?
398
+ #raise RestFtpDaemon::JobAssertionFailed if @transfer_total.nil?
354
399
  status :ftp_transfer
355
- raise RestFtpDaemon::JobAssertionFailed if @ftp.nil? || @source_path.nil?
400
+
401
+ # Use source filename if target path provided none (typically with multiple sources)
402
+ target_name ||= Helpers.extract_filename source_match
403
+ info "Job.ftp_transfer target_name: #{target_name}"
404
+
405
+ # Check for target file presence
406
+ status :checking_target
407
+ overwrite = !get(:overwrite).nil?
408
+ present = ftp_presence target_name
409
+ if present
410
+ if overwrite
411
+ # delete it first
412
+ info "Job.ftp_transfer removing target file"
413
+ @ftp.delete(target_name)
414
+ else
415
+ # won't overwrite then stop here
416
+ info "Job.ftp_transfer failed: target file exists"
417
+ @ftp.close
418
+ raise RestFtpDaemon::JobTargetFileExists
419
+ end
420
+ end
356
421
 
357
422
  # Read source file size and parameters
358
- source_size = File.size @source_path
359
- set :transfer_size, source_size
360
423
  update_every_kb = (Settings.transfer.update_every_kb rescue nil) || DEFAULT_UPDATE_EVERY_KB
361
424
  notify_after_sec = Settings.transfer.notify_after_sec rescue nil
362
425
 
363
426
  # Start transfer
364
- transferred = 0
365
427
  chunk_size = update_every_kb * 1024
366
428
  t0 = tstart = Time.now
367
429
  notified_at = Time.now
368
430
  status :uploading
369
- @ftp.putbinaryfile(@source_path, target_name, chunk_size) do |block|
431
+ @ftp.putbinaryfile(source_match, target_name, chunk_size) do |block|
370
432
  # Update counters
371
- transferred += block.bytesize
372
- set :transfer_sent, transferred
433
+ @transfer_sent += block.bytesize
434
+ set :transfer_sent, @transfer_sent
373
435
 
374
436
  # Update bitrate
375
- dt = Time.now - t0
376
- bitrate0 = (8 * chunk_size/dt).round(0)
437
+ #dt = Time.now - t0
438
+ bitrate0 = get_bitrate(chunk_size, t0).round(0)
377
439
  set :transfer_bitrate, bitrate0
378
440
 
379
441
  # Update job info
380
- percent1 = (100.0 * transferred / source_size).round(1)
442
+ percent1 = (100.0 * @transfer_sent / @transfer_total).round(1)
381
443
  set :progress, percent1
382
444
 
383
445
  # Log progress
384
446
  status = []
385
447
  status << "#{percent1} %"
386
- status << (Helpers.format_bytes transferred, "B")
387
- status << (Helpers.format_bytes source_size, "B")
448
+ status << (Helpers.format_bytes @transfer_sent, "B")
449
+ status << (Helpers.format_bytes @transfer_total, "B")
388
450
  status << (Helpers.format_bytes bitrate0, "bps")
389
451
  info "Job.ftp_transfer" + status.map{|txt| ("%#{DEFAULT_LOGS_PROGNAME_TRIM.to_i}s" % txt)}.join("\t")
390
452
 
@@ -395,8 +457,8 @@ module RestFtpDaemon
395
457
  unless notify_after_sec.nil? || (notified_at + notify_after_sec > Time.now)
396
458
  status = {
397
459
  progress: percent1,
398
- transfer_sent: transferred,
399
- transfer_size: source_size,
460
+ transfer_sent: @transfer_sent,
461
+ transfer_total: @transfer_total,
400
462
  transfer_bitrate: bitrate0
401
463
  }
402
464
  notify "rftpd.progress", 0, status
@@ -406,12 +468,10 @@ module RestFtpDaemon
406
468
  end
407
469
 
408
470
  # Compute final bitrate
409
- tbitrate0 = (8 * source_size.to_f / (Time.now - tstart)).round(0)
471
+ #tbitrate0 = (8 * @transfer_total.to_f / (Time.now - tstart)).round(0)
472
+ tbitrate0 = get_bitrate(@transfer_total, tstart).round(0)
410
473
  set :transfer_bitrate, tbitrate0
411
474
 
412
- # Add total transferred to counter
413
- $queue.counter_add :transferred, source_size
414
-
415
475
  # Done
416
476
  #set :progress, nil
417
477
  info "Job.ftp_transfer finished"
@@ -419,12 +479,16 @@ module RestFtpDaemon
419
479
 
420
480
  def notify signal, error = 0, status = {}
421
481
  RestFtpDaemon::Notification.new get(:notify), {
422
- id: get(:id),
482
+ id: @id,
423
483
  signal: signal,
424
484
  error: error,
425
485
  status: status,
426
486
  }
427
487
  end
428
488
 
489
+ def get_bitrate total, last_timestamp
490
+ total.to_f / (Time.now - last_timestamp)
491
+ end
492
+
429
493
  end
430
494
  end