rest-ftp-daemon 0.30.1 → 0.41

data/lib/rest-ftp-daemon/common.rb

@@ -0,0 +1,49 @@
+module RestFtpDaemon
+
+  class Common
+
+  protected
+
+    def initialize
+      # Logger
+      @logger = ActiveSupport::Logger.new APP_LOGTO, 'daily'
+    end
+
+    def id
+    end
+
+    def progname
+    end
+
+    def info message, level = 0
+      # progname = "Job [#{id}]" unless id.nil?
+      # progname = "Worker [#{id}]" unless worker_id.nil?
+      @logger.add(Logger::INFO, "#{'  '*(level+1)} #{message}", progname)
+    end
+
+    def notify signal, error = 0, status = {}
+      # Skip is not callback URL defined
+      url = get :notify
+      if url.nil?
+        info "Skipping notification (no valid url provided) sig[#{signal}] e[#{error}] s#{status.inspect}"
+        return
+      end
+
+      # Build notification
+      n = RestFtpDaemon::Notification.new
+      n.job_id = id
+      n.url = url
+      n.signal = signal
+      n.error = error.inspect
+      n.status = status
+
+      # Now, send the notification
+      info "Queuing notification key[#{n.key}] sig[#{signal}] url[#{url}]"
+      Thread.new(n) do |thread|
+        n.notify
+      end
+
+    end
+
+  end
+end

data/lib/rest-ftp-daemon/config.rb

@@ -1,8 +1,15 @@
 module RestFtpDaemon
   # Global config
-  VERSION = "0.30.1"
+  NAME = 'rest-ftp-daemon'
+  VERSION = "0.41"
+  PORT = 3000
 
   # Transfer config
   TRANSFER_CHUNK_SIZE = 100000
-  THREAD_SLEEP_BEFORE_DIE = 30
+  THREAD_SLEEP_BEFORE_DIE = 10
+
+  # Logging
+  APP_LOGTO = "/tmp/#{NAME}.log"
+  LOG_TRIM_PROGNAME = 18
+
 end

data/lib/rest-ftp-daemon/exceptions.rb

@@ -16,6 +16,11 @@ module RestFtpDaemon
   class JobSourceNotFound        < RestFtpDaemonException; end
   class JobTargetMissing         < RestFtpDaemonException; end
   class JobTargetUnparseable     < RestFtpDaemonException; end
-  class JobTargetPermission      < RestFtpDaemonException; end
+  #class JobTargetPermission      < RestFtpDaemonException; end
+  class JobTargetFileExists      < RestFtpDaemonException; end
+
+  class NotificationMissingUrl   < RestFtpDaemonException; end
+  class NotificationMissingSignal   < RestFtpDaemonException; end
+
 
 end

data/lib/rest-ftp-daemon/job.rb

@@ -1,25 +1,50 @@
 module RestFtpDaemon
-  class Job
+  class Job < RestFtpDaemon::Common
 
     def initialize(id, params={})
+      # Call super
+      super()
+
       # Grab params
       @params = params
       @target = nil
       @source = nil
 
+      # Logger
+      #@logger = ActiveSupport::Logger.new APP_LOGTO, 'daily'
+
       # Init context
       set :id, id
       set :started_at, Time.now
       set :status, :initialized
+
+      # Send first notification
+      notify "rftpd.queued"
+    end
+
+    # def job_id
+    #   get :id
+    # end
+
+    def progname
+      job_id = get(:id)
+      "JOB #{job_id}"
     end
 
     def id
-      @params[:id]
+      get :id
+    end
+
+    def priority
+      get :priority
+    end
+    def get_status
+      get :status
     end
 
     def process
       # Init
-      "process [#{@id}] starting"
+      info "Job.process starting"
       set :status, :starting
       set :error, 0
 
@@ -30,11 +55,13 @@ module RestFtpDaemon
         # Process
         transfer
 
-      rescue Net::FTPPermError
+      rescue Net::FTPPermError => exception
+        info "Job.process failed [Net::FTPPermError]"
         set :status, :failed
         set :error, exception.class
 
       rescue RestFtpDaemonException => exception
+        info "Job.process failed [RestFtpDaemonException::#{exception.class}]"
         set :status, :failed
         set :error, exception.class
 
@@ -43,8 +70,14 @@ module RestFtpDaemon
       #   set :error, exception.class
 
       else
+        info "Job.process finished"
+# set :error, 0
+        #set :status, :wandering
+
+        # Wait for a few seconds before marking the job as finished
+        # info "#{prefix} wander for #{RestFtpDaemon::THREAD_SLEEP_BEFORE_DIE} sec"
+        # wander RestFtpDaemon::THREAD_SLEEP_BEFORE_DIE
         set :status, :finished
-        set :error, 0
       end
 
     end
@@ -62,12 +95,6 @@ module RestFtpDaemon
       @params
     end
 
-    def wander time
-      @wander_for = time
-      @wander_started = Time.now
-      sleep time
-    end
-
     def status text
       @status = text
     end
@@ -79,20 +106,33 @@ module RestFtpDaemon
       Time.now - @params[:started_at]
     end
 
+    def wander time
+      info "Job.wander #{time}"
+      @wander_for = time
+      @wander_started = Time.now
+      sleep time
+      info "Job.wandered ok"
+    end
+
     def wandering_time
       return if @wander_started.nil? || @wander_for.nil?
       @wander_for.to_f - (Time.now - @wander_started)
     end
 
-    def exception_handler(actor, reason)
-      set :status, :crashed
-      set :error, reason
-    end
+    # def exception_handler(actor, reason)
+    #   set :status, :crashed
+    #   set :error, reason
+    # end
 
     def set attribute, value
       return unless @params.is_a? Enumerable
       @params[:updated_at] = Time.now
-      @params[attribute] = value
+      @params[attribute.to_s] = value
+    end
+
+    def get attribute
+      return nil unless @params.is_a? Enumerable
+      @params[attribute.to_s]
     end
 
     def prepare
@@ -124,8 +164,9 @@ module RestFtpDaemon
     end
 
     def transfer
-      # Init
+      # Send first notification
       transferred = 0
+      notify "rftpd.started"
 
       # Ensure @source and @target are there
       set :status, :checking_source
@@ -142,15 +183,19 @@ module RestFtpDaemon
       set :status, :checking_target
       ftp = Net::FTP.new(@target.host)
       ftp.passive = true
-      ftp.login
+      ftp.login @target.user, @target.password
       ftp.chdir(target_path)
 
       # Check for target file presence
-      results = ftp.list(target_name)
-      #info "ftp.list: #{results}"
-      unless results.count.zero?
-        ftp.close
-        raise JobTargetPermission
+      if get(:overwrite).nil?
+        results = ftp.list(target_name)
+        #results = ftp.list()
+
+        unless results.count.zero?
+          ftp.close
+          notify "rftpd.ended", RestFtpDaemon::JobTargetFileExists
+          raise RestFtpDaemon::JobTargetFileExists
+        end
       end
 
       # Do transfer
@@ -166,6 +211,7 @@ module RestFtpDaemon
       end
 
       # Close FTP connexion
+      notify "rftpd.ended"
       ftp.close
     end
 

data/lib/rest-ftp-daemon/job_queue.rb

@@ -0,0 +1,105 @@
+require 'thread'
+
+module RestFtpDaemon
+  class JobQueue < Queue
+
+    def initialize
+      @queued = []
+      @popped = []
+
+      @waiting = []
+      @queued.taint          # enable tainted communication
+      @waiting.taint
+      self.taint
+      @mutex = Mutex.new
+    end
+
+    def queued
+      @queued
+    end
+    def queued_size
+      @queued.length
+    end
+
+    def popped
+      @popped
+    end
+    def popped_size
+      @popped.length
+    end
+
+    def all
+      @queued + @popped
+    end
+    def all_size
+      popped_size + queued_size
+    end
+
+    def push(obj)
+      # Check that itme responds to "priorty" method
+      raise "object should respond to priority method" unless obj.respond_to? :priority
+
+      @mutex.synchronize{
+        @queued.push obj
+        begin
+          t = @waiting.shift
+          t.wakeup if t
+        rescue ThreadError
+          retry
+        end
+      }
+    end
+    alias << push
+    alias enq push
+
+    #
+    # Retrieves data from the queue.  If the queue is empty, the calling thread is
+    # suspended until data is pushed onto the queue.  If +non_block+ is true, the
+    # thread isn't suspended, and an exception is raised.
+    #
+    def pop(non_block=false)
+      @mutex.synchronize{
+        while true
+          if @queued.empty?
+            raise ThreadError, "queue empty" if non_block
+            @waiting.push Thread.current
+            @mutex.sleep
+          else
+            return pick
+          end
+        end
+      }
+    end
+    alias shift pop
+    alias deq pop
+
+    def empty?
+      @queued.empty?
+    end
+
+    def clear
+      @queued.clear
+    end
+
+    def num_waiting
+      @waiting.size
+    end
+
+  protected
+
+    def pick
+      # Sort jobs by priority and get the biggest one
+      picked = @queued.sort { |a,b| a.priority.to_i <=> b.priority.to_i }.last
+
+      # Delete it from the queue
+      @queued.delete_if { |item| item == picked } unless picked.nil?
+
+      # Stack it to popped items
+      @popped.push picked
+
+      # Return picked
+      picked
+    end
+
+  end
+end

data/lib/rest-ftp-daemon/logger.rb

@@ -0,0 +1,7 @@
+class Logger
+  def format_message(severity, timestamp, progname, msg)
+    stamp = Time.now.strftime("%Y-%m-%d %H:%M:%S")
+    progname = "%-#{RestFtpDaemon::LOG_TRIM_PROGNAME}s" % progname
+    "#{stamp}  #{severity}  #{progname}  #{msg}\n"
+  end
+end

data/lib/rest-ftp-daemon/notification.rb

@@ -0,0 +1,80 @@
+module RestFtpDaemon
+  class Notification < RestFtpDaemon::Common
+    attr_accessor :job_id
+    attr_accessor :signal
+    attr_accessor :error
+    attr_accessor :message
+    attr_accessor :status
+    attr_accessor :url
+    attr_accessor :job
+    attr_accessor :key
+
+    def initialize
+    #def initialize(job, signal, error, status)
+      # Grab params
+      #@job = job
+      # @signal = signal
+      # @error = error
+      # @status = status
+      @status = {}
+      @error = 0
+      @message = nil
+
+      # Generate a random key
+      @key = SecureRandom.hex(2)
+
+      # Call super
+      super
+
+    end
+
+    def progname
+      "NOTIF #{@key}"
+    end
+
+
+    # def status key, val
+    #   @status[key.to_s] = val
+    # end
+
+    def notify
+      # Check context
+      raise NotificationMissingUrl unless @url
+      raise NotificationMissingSignal unless @signal
+      #sraise NotifImpossible unless @status
+
+      # Params
+      params = {
+        id: @job_id,
+        host: get_hostname,
+        signal: @signal,
+        error: @error,
+        }
+
+      # Add status only if present
+      params["status"] = @status unless @status.empty?
+      params["message"] = @message unless @message.to_s.blank?
+
+      # Log this notification
+      info "notify params: #{params.inspect}"
+
+      # Prepare query
+      uri = URI(@url)
+      headers = {"Content-Type" => "application/json",
+                "Accept" => "application/json"}
+
+      # Post the notification
+      http = Net::HTTP.new(uri.host, uri.port)
+      response = http.post(uri.path, params.to_json, headers)
+
+      info "notify reply: #{response.body.strip}"
+    end
+
+  protected
+
+    def get_hostname
+      `hostname`.chomp
+    end
+
+  end
+end