rest-ftp-daemon 0.435.2 → 0.501.0

data/lib/rest-ftp-daemon/jobs/video.rb

@@ -29,7 +29,7 @@ module RestFtpDaemon
     def do_work
       # Guess source files from disk
       set_status JOB_STATUS_TRANSFORMING
-      sources = @source_loc.scan_files
+      sources = @source_loc.local_files
       raise RestFtpDaemon::SourceNotFound if sources.empty?
 
       # Add the source file name if none found in the target path
@@ -38,8 +38,9 @@ module RestFtpDaemon
       log_info "JobVideo.work target_final.path [#{target_final.path}]"
 
       # Ensure target directory exists
-      log_info "JobVideo.work mkdir_p [#{@target_loc.dir}]"
-      FileUtils.mkdir_p @target_loc.dir
+      t_dir = @target_loc.filedir
+      log_info "JobVideo.work mkdir_p [#{t_dir}]"
+      FileUtils.mkdir_p t_dir
 
       # Do the work, for each file
       set_info INFO_SOURCE_CURRENT, @source_loc.name
@@ -81,10 +82,10 @@ module RestFtpDaemon
       log_info "JobVideo.ffmpeg_command [#{FFMPEG.ffmpeg_binary}] [#{source.name}] > [#{target.name}]", options
 
       # Build command
-      movie.transcode(target.path, options) do |ffmpeg_progress|
+      movie.transcode(target.filepath, options) do |ffmpeg_progress|
         # set_info :work, :ffmpeg_progress, ffmpeg_progress
         set_info INFO_TRANFER_PROGRESS, (100.0 * ffmpeg_progress).round(1)
-        log_debug "progress #{ffmpeg_progress}"
+        log_info "progress #{ffmpeg_progress}"
       end
     end
 
@@ -117,7 +118,6 @@ module RestFtpDaemon
   end
 end
 
-
 # require "stringio"
 # def capture_stderr
 #   real_stderr, $stderr = $stderr, StringIO.new
@@ -125,4 +125,4 @@ end
 #   $stderr.string
 # ensure
 #   $stderr = real_stderr
-# end
+# end

data/lib/rest-ftp-daemon/launcher.rb

@@ -24,4 +24,4 @@ module RestFtpDaemon
     end
 
   end
-end
+end

data/lib/rest-ftp-daemon/location.rb

@@ -8,8 +8,9 @@ module RestFtpDaemon
     # Accessors
     attr_accessor :name
 
-    attr_reader :original
+    attr_reader :url
     attr_reader :uri
+    attr_reader :tokens
     attr_reader :scheme
     attr_reader :dir
 
@@ -22,37 +23,34 @@ module RestFtpDaemon
     delegate :scheme, :host, :port, :user, :password, :to_s,
       to: :uri
 
-    def initialize original
-      unless original.is_a? String
-        raise RestFtpDaemon::AssertionFailed, "location/original/string: #{original.inspect}"
-      end
+    def initialize url
+      # Check parameters
+      # unless url.is_a? String
+      #   raise RestFtpDaemon::AssertionFailed, "location/init/string: #{url.inspect}"
+      # end   
+      debug nil
 
-      # Strip spaces before/after, copying original "path" at the same time
-      @original = original
-      location_uri = original.strip
+      @url = url
+      debug :url, url
+      @tokens = []
 
-      # Replace tokens, fix scheme for local paths
-      resolve_tokens! location_uri
-      fix_scheme! location_uri
+      # Detect tokens in string
+      @tokens = detect_tokens(url)
+      debug :tokens, @tokens.inspect
 
-      # Ensure result does not contain tokens after replacement
-      detected_tokens = detect_tokens(location_uri)
-      unless detected_tokens.empty?
-        raise RestFtpDaemon::JobUnresolvedTokens, 'unresolved tokens: ' + detected_tokens.join(' ')
-      end
+      # First resolve tokens
+      resolve_tokens! url
+
+      # Build URI from parameters
+      build_uri url
+
+      # Extract dir and name
+      build_dir_name
 
-      # Parse URL and do specific initializations
-      parse_url location_uri
+      # Specific initializations
       case @uri
-      when URI::FILE  then init_file
       when URI::S3    then init_aws               # Match AWS URL with BUCKET.s3.amazonaws.com
       end
-
-      # Check that scheme is supported
-      unless @uri.scheme
-        raise RestFtpDaemon::SchemeUnsupported, url
-        # raise RestFtpDaemon::SchemeUnsupported, @uri
-      end
     end
 
     def is? kind
@@ -60,13 +58,18 @@ module RestFtpDaemon
     end
 
     def path
-      return @name if @dir.nil?
-      File.join(@dir.to_s, @name.to_s)
+      File.join(@dir.to_s, name.to_s)
+    end
+
+    def filedir
+      "/#{@dir}"
+    end
+    def filepath
+      "/#{path}"
     end
-    alias :to_s :path
 
-    def scan_files
-      Dir.glob(path).collect do |file|
+    def local_files
+      Dir.glob("/#{path}").collect do |file|
         next unless File.readable? file
         next unless File.file? file
         # Create a new location object
@@ -76,15 +79,19 @@ module RestFtpDaemon
 
     def size
       return unless uri.is_a? URI::FILE
-      local_fil_path = path
-      return unless File.exist? local_fil_path
-      return File.size local_fil_path
+      local_file_path = filepath
+      return unless File.exist? local_file_path
+      return File.size local_file_path
     end
 
     def generate_temp_name!
       @name = "#{@name}.temp-#{identifier(JOB_TEMPFILE_LEN)}"
     end
 
+    # def scheme? condition
+    #   return @uri.scheme == condition
+    # end
+
   private
 
     def tokenize item
@@ -92,6 +99,28 @@ module RestFtpDaemon
       "[#{item}]"
     end
 
+    def build_uri url
+      # Fix scheme if URL as none
+      url.gsub! /^\/(.*)/, 'file:/\1'
+
+      # Parse that URL
+      @uri = URI.parse url # rescue nil
+      raise RestFtpDaemon::LocationParseError, location_path unless @uri
+
+      # Remove unnecessary double slahes
+      @uri.path.gsub!(/\/+/, '/')
+
+      # Check we finally have a scheme
+      debug :scheme, @uri.scheme 
+      debug :path, @uri.path
+      debug :host, @uri.host
+      debug :to_s, @uri.to_s
+
+      # Raise if still no scheme #FIXME
+      raise RestFtpDaemon::SchemeUnsupported, url unless @uri.scheme
+      # raise RestFtpDaemon::LocationParseError, base unless @uri
+    end
+
     def resolve_tokens! path
       # Gther endpoints, and copy path string to alter it later
       endpoints = BmcDaemonLib::Conf[:endpoints]
@@ -107,39 +136,21 @@ module RestFtpDaemon
         next if to.to_s.empty?
         path.gsub! tokenize(from), to
       end
-    end
-
-    def fix_scheme! path
-      # path.gsub!(/^\/(.*)/, 'file:///\1')
-      path.gsub! /^\/(.*)/, 'file:/\1'
-    end
-
-    def parse_url path
-      # Parse that URL
-      @uri = URI.parse path # rescue nil
-      raise RestFtpDaemon::LocationParseError, location_path unless @uri
-
-      # Path cleanup
-      cleaned  = @uri.path.clone
-
-      # remove_leading_slashes
-      cleaned.gsub! /^\//, ''
-
-      # remove_multiple_slashes
-      cleaned.gsub! /([^:])\/\//, '\1/'
 
-      # Store URL parts
-      @dir      = extract_dirname  cleaned
-      @name     = extract_filename cleaned
+      # Ensure result does not contain tokens after replacement
+      detected = detect_tokens(path)
+      unless detected.empty?
+        raise RestFtpDaemon::JobUnresolvedTokens, 'unresolved tokens: ' + detected.join(' ')
+      end
     end
 
-    # def init_username
-    #   @uri.user ||= "anonymous"
-    # end
+    def build_dir_name
+      # Store URL parts (remove leading slashes in dir)
+      @dir      = extract_dirname(@uri.path).gsub(/^\//, '')
+      @name     = extract_filename(@uri.path)
 
-    def init_file
-      # Dir is absolute
-      @dir = File.join('/', @dir.to_s)
+      debug :dir, dir
+      debug :name, name
     end
 
     def init_aws
@@ -156,9 +167,6 @@ module RestFtpDaemon
       # Credentials from config
       @aws_id     = Conf.at(:credentials, @uri.host, :id)
       @aws_secret = Conf.at(:credentials, @uri.host, :secret)
-
-      # Clear @dir
-      @dir = nil
     end
 
     def extract_filename path
@@ -174,8 +182,24 @@ module RestFtpDaemon
     end
 
     def detect_tokens item
-      item.scan /\[[^\[\]]*\]/
+      # item.scan /\[([^\[\]]*)\]/
+      item.scan(/\[([^\[\]]*)\]/).map(&:first)
+    end
+
+    def debug var, val = nil
+      # Read conf if not already cached
+      @debug ||= Conf.at(:debug, :location)
+
+      # Skip if no debug requeste
+      return unless @debug
+
+      # Dump line
+      if var.nil?
+        printf("|%s \n", "-"*100) 
+      else
+        printf("| %-15s: %s \n", var, val)
+      end
     end
 
   end
-end
+end

data/lib/rest-ftp-daemon/metrics.rb

@@ -27,7 +27,7 @@ module RestFtpDaemon
         # Init
         counts = {}
 
-        RestFtpDaemon::WorkerPool.instance.worker_variables.group_by do |wid, vars|
+        WorkerPool.instance.worker_variables.group_by do |wid, vars|
           vars[:status]
         end.each do |status, workers|
           counts[status] = workers.count
@@ -38,4 +38,4 @@ module RestFtpDaemon
       end
 
   end
-end
+end

data/lib/rest-ftp-daemon/notification.rb

@@ -111,4 +111,4 @@ module RestFtpDaemon
     end
 
   end
-end
+end

data/lib/rest-ftp-daemon/paginate.rb

@@ -64,4 +64,4 @@ module RestFtpDaemon
     end
 
   end
-end
+end

data/lib/rest-ftp-daemon/remote/base.rb

@@ -3,6 +3,7 @@ module RestFtpDaemon
   module Remote
     class RemoteBase
       include BmcDaemonLib::LoggerHelper
+      include CommonHelpers
 
       # Class options
       attr_reader :log_prefix
@@ -25,12 +26,16 @@ module RestFtpDaemon
         log_pipe :transfer
 
         # Annnounce object
-        log_info "RemoteBase.initialize debug[#{debug}] target[#{target.path}] "
+        log_info "initialize debug[#{debug}] target[#{target.path}] "
 
         # Prepare real object
         prepare
       end
 
+      def size_if_exists target
+        false
+      end
+
       def prepare
       end
 
@@ -53,7 +58,7 @@ module RestFtpDaemon
       end
 
       def close
-        log_debug "RemoteBase.close"
+        log_debug "close"
 
         # Debug mode ?
         return unless @debug
@@ -85,4 +90,4 @@ module RestFtpDaemon
 
     end
   end
-end
+end

data/lib/rest-ftp-daemon/remote/ftp.rb

@@ -10,6 +10,8 @@ module RestFtpDaemon
       attr_reader :ftp
 
       def prepare
+        super
+        
         # Create FTP object
         if @ftpes
           prepare_ftpes
@@ -19,11 +21,8 @@ module RestFtpDaemon
         @ftp.passive = true
         @ftp.debug_mode = @debug
 
-        # Config
-        @chunk_size = JOB_FTP_CHUNKMB.to_i * 1024
-
         # Announce object
-        log_debug "RemoteFTP.prepare chunk_size:#{@chunk_size}"
+        log_debug "prepare chunk_size:#{format_bytes(JOB_FTP_CHUNKMB, "B")}"
       end
 
       def connect
@@ -34,9 +33,9 @@ module RestFtpDaemon
         @ftp.login @target.user, @target.password
       end
 
-      def present? target
-        size = @ftp.size target.path
-        log_debug "RemoteFTP.present? [#{target.name}]"
+      def size_if_exists target
+        size = @ftp.size target.filepath
+        log_debug "size_if_exists [#{target.name}]"
 
       rescue Net::FTPPermError
         return false
@@ -45,15 +44,15 @@ module RestFtpDaemon
       end
 
       def remove! target
-        @ftp.delete target.path
+        @ftp.delete target.filepath
       rescue Net::FTPPermError
-        log_debug "RemoteFTP.remove! [#{target.name}] not found"
+        log_debug "remove! [#{target.name}] not found"
       else
-        log_debug "RemoteFTP.remove! [#{target.name}] removed"
+        log_debug "remove! [#{target.name}] removed"
       end
 
       def mkdir directory
-        log_debug "RemoteFTP.mkdir [#{directory}]"
+        log_debug "mkdir [#{directory}]"
         @ftp.mkdir directory
 
       rescue StandardError => ex
@@ -63,7 +62,7 @@ module RestFtpDaemon
       def chdir_or_create thedir, mkdir = true
         # Init, extract my parent name and my own name
         parent, current = split_path(thedir)
-        log_debug "RemoteFTP.chdir_or_create mkdir[#{mkdir}] dir[#{thedir}] parent[#{parent}] current[#{current}]"
+        log_debug "chdir_or_create mkdir[#{mkdir}] dir[#{thedir}] parent[#{parent}] current[#{current}]"
 
         # Access this directory
         begin
@@ -104,21 +103,22 @@ module RestFtpDaemon
         end
 
         # Move to the directory
-        log_debug "RemoteFTP.upload chdir [#{dest.dir}]"
-        @ftp.chdir "/#{dest.dir}"
+        log_debug "upload chdir [#{dest.filedir}]"
+        @ftp.chdir dest.filedir
 
         # Do the transfer
-        log_debug "RemoteFTP.upload putbinaryfile [#{dest.name}]"
-        @ftp.putbinaryfile source.path, dest.name, @chunk_size do |data|
+        log_debug "upload putbinaryfile [#{source.filepath}] [#{dest.name}]"
+        @ftp.putbinaryfile source.filepath, dest.name, JOB_FTP_CHUNKMB do |data|
           # Update job status after this block transfer
           yield data.bytesize, dest.name
         end
 
         # Move the file back to its original name
         if use_temp_name
-          log_debug "RemoteFTP.upload rename [#{dest.name}] > [#{target.name}]"
+          log_debug "upload rename [#{dest.name}] > [#{target.name}]"
           @ftp.rename dest.name, target.name
         end
+
       end
 
       def close
@@ -147,4 +147,4 @@ module RestFtpDaemon
 
     end
   end
-end
+end