rest-ftp-daemon 0.243.2 → 0.245

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 98aea753658309483e52707b6492003b9d8421d0
-  data.tar.gz: 0f3c991b458644384c98f4697106107e2df1a114
+  metadata.gz: c9be0d61a767998e3daba2c1006ba9c8b184e303
+  data.tar.gz: 46b7e9173008c7f5e4cb015ae84b1d20ac788cd7
 SHA512:
-  metadata.gz: 2656b3fa5d273616e56bd24eb77b625720b22307192f301f2f5274d28ca5aafd1ce92448e0efc811a90cf0fa97a771606c2c58ec917a9baa441d72f561c74000
-  data.tar.gz: 66e5182acef7425e542add0dbf7967295d496fc23ff5808ba811237be1353c858974689b5c8403c29c1a146f22515d036694dbccbcf4a0430c4ea88eba03d5f2
+  metadata.gz: fec8c694546fc1d7b2991f2d0d8e1fa2bf35669a7e6cd56dffd459b787db66ab9a608205635dce427720022b7c3b94aaade650c110674042d2cb302d0def058f
+  data.tar.gz: 45621e4a9a99a535c5448a5172986d391f76f396feab220efb89979ee1f0be63499b5edd1ca8012b8cae275ae7b9dff7c5997ee67c638b8d7db541219a29b528

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    rest-ftp-daemon (0.243.2)
+    rest-ftp-daemon (0.245)
       double-bag-ftps
       facter
       get_process_mem

data/lib/rest-ftp-daemon/api/dashboard.rb

@@ -43,17 +43,20 @@ module RestFtpDaemon
         @only = params["only"].to_s
 
         # Get jobs for this view, order jobs by their weights
-        result = $queue.filter_jobs(@only).reverse
+        jobs_with_status = $queue.jobs_with_status(@only).reverse
 
         # Provide queue only if no filtering set
-        @queue = []
-        @queue = $queue.queue.reverse if @only.empty?
+        if @only.empty?
+          @jobs_queued = $queue.jobs_queued
+        else
+          @jobs_queued = []
+        end
 
         # Get workers status
         @worker_variables = $pool.worker_variables
 
         # Build paginator
-        @paginate = Paginate.new result
+        @paginate = Paginate.new jobs_with_status
         @paginate.only = params["only"]
         @paginate.page = params["page"]
         @paginate.all = params.keys.include? "all"

data/lib/rest-ftp-daemon/api/jobs.rb

@@ -77,6 +77,7 @@ module RestFtpDaemon
         optional :label, type: String, desc: "Descriptive label for this job"
         optional :notify, type: String, desc: "URL to get POST'ed notifications back"
         optional :priority, type: Integer, desc: "Priority level of the job (lower is stronger)"
+        optional :pool, type: String, desc: "Pool of worker to be used"
         optional :overwrite,
           type: Boolean,
           desc: "Overwrites files at target server",

data/lib/rest-ftp-daemon/constants.rb

@@ -1,7 +1,7 @@
 # Terrific constants
 APP_NAME = "rest-ftp-daemon"
 APP_NICK = "rftpd"
-APP_VER = "0.243.2"
+APP_VER = "0.245"
 
 # Provide default config file information
 APP_LIB = File.expand_path File.dirname(__FILE__)
@@ -20,7 +20,8 @@ EOD
 
 
 # Configuration defaults
-DEFAULT_WORKERS         = 2
+# DEFAULT_WORKERS         = 2
+DEFAULT_POOL            = "default"
 DEFAULT_WORKER_TIMEOUT  = 3600  # 1h
 DEFAULT_SFTP_TIMEOUT    = 600   # 10mn
 DEFAULT_FTP_CHUNK       = 1024  # 1 MB
@@ -91,7 +92,7 @@ DASHBOARD_JOB_STYLES = {
   }
 DASHBOARD_WORKER_STYLES = {
   waiting:              :success,
-  working:              :info,
+  running:              :info,
   crashed:              :danger,
   done:                 :success,
   dead:                 :danger,

data/lib/rest-ftp-daemon/job.rb

@@ -11,7 +11,7 @@ module RestFtpDaemon
       include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
     end
 
-    FIELDS = [:source, :target, :label, :priority, :notify, :overwrite, :mkdir, :tempfile]
+    FIELDS = [:source, :target, :label, :priority, :pool, :notify, :overwrite, :mkdir, :tempfile]
 
     attr_accessor :wid
 
@@ -27,6 +27,7 @@ module RestFtpDaemon
     attr_reader :finished_at
 
     attr_reader :infos
+    attr_reader :pool
 
     FIELDS.each do |name|
       attr_reader name
@@ -58,6 +59,15 @@ module RestFtpDaemon
         instance_variable_set "@#{name}", params[name]
       end
 
+      # Set pool
+      pools = (Settings.pools || {})
+      # Check if pool name exists
+      if (pools.keys.include? params[:pool])
+        @pool = params[:pool].to_s
+      else
+        @pool = DEFAULT_POOL
+      end
+
       # Set job queue, thus reset
       reset
 

data/lib/rest-ftp-daemon/job_queue.rb

@@ -5,7 +5,7 @@ module RestFtpDaemon
     include LoggerHelper
     attr_reader :logger
 
-    attr_reader :queue
+    #attr_reader :queues
     attr_reader :jobs
 
     if Settings.newrelic_enabled?
@@ -14,11 +14,17 @@ module RestFtpDaemon
 
     def initialize
       # Instance variables
-      @queue = []
+      @queues = {}
+      @waitings = {}
+
+      # @queue = []
+      # @waiting = []
+
       @jobs = []
-      @waiting = []
-      @queue.taint          # enable tainted communication
-      @waiting.taint
+
+      @queues.taint          # enable tainted communication
+      @waitings.taint
+
       taint
       @mutex = Mutex.new
 
@@ -65,7 +71,11 @@ module RestFtpDaemon
       end
     end
 
-    def filter_jobs status
+    def jobs_queued
+      @queues
+    end
+
+    def jobs_with_status status
       # No status filter: return all execept queued
       if status.empty?
         @jobs.reject { |job| job.status == JOB_STATUS_QUEUED }
@@ -88,7 +98,7 @@ module RestFtpDaemon
     end
 
     def queued_ids
-      @queue.collect(&:id)
+      @queues.collect{|pool, jobs| jobs.collect(&:id)}
     end
 
     def jobs_ids
@@ -101,7 +111,8 @@ module RestFtpDaemon
       log_info "find_by_id (#{id}, #{prefixed}) > #{id}"
 
       # Search in jobs queues
-      @jobs.reverse.find { |item| item.id == id }
+      #@jobs.reverse.find { |item| item.id == id }
+      @jobs.find { |item| item.id == id }
     end
 
 
@@ -109,24 +120,31 @@ module RestFtpDaemon
       # Check that item responds to "priorty" method
       raise "JobQueue.push: job should respond to priority method" unless job.respond_to? :priority
       raise "JobQueue.push: job should respond to id method" unless job.respond_to? :id
+      raise "JobQueue.push: job should respond to pool method" unless job.respond_to? :pool
       raise "JobQueue.push: job should respond to reset" unless job.respond_to? :reset
 
       @mutex.synchronize do
+        # Get this job's pool @ prepare queue of this pool
+        pool = job.pool
+        myqueue = (@queues[pool] ||= [])
+
         # Store the job into the global jobs list, if not already inside
         @jobs.push(job) unless @jobs.include?(job)
 
         # Push job into the queue, if not already inside
-        @queue.push(job) unless @queue.include?(job)
+        myqueue.push(job) unless myqueue.include?(job)
 
         # Inform the job that it's been queued / reset it
         job.reset
 
         # Refresh queue order
-        sort_queue!
+        #sort_queue!(pool)
+        myqueue.sort_by!(&:weight)
 
         # Try to wake a worker up
         begin
-          t = @waiting.shift
+          @waitings[pool] ||= []
+          t = @waitings[pool].shift
           t.wakeup if t
         rescue ThreadError
           retry
@@ -137,16 +155,18 @@ module RestFtpDaemon
     alias enq     push
     alias requeue push
 
-    def pop non_block = false
+    def pop pool, non_block = false
       @mutex.synchronize do
+        myqueue = (@queues[pool] ||= [])
+        @waitings[pool] ||= []
         loop do
-          if @queue.empty?
-            # info "JobQueue.pop: empty"
+          if myqueue.empty?
+            #puts "JobQueue.pop(#{pool}): empty"
             raise ThreadError, "queue empty" if non_block
-            @waiting.push Thread.current
+            @waitings[pool].push Thread.current
             @mutex.sleep
           else
-            return @queue.pop
+            return myqueue.pop
           end
         end
       end
@@ -202,19 +222,11 @@ module RestFtpDaemon
       "#{@prefix}.#{id}"
     end
 
-    def sort_queue!
-      @mutex_counters.synchronize do
-        @queue.sort_by!(&:weight)
-      end
-    end
-
     if Settings.newrelic_enabled?
       add_transaction_tracer :push,             category: :task
       add_transaction_tracer :pop,              category: :task
-      add_transaction_tracer :sort_queue!,      category: :task
       add_transaction_tracer :expire,           category: :task
       add_transaction_tracer :counts_by_status, category: :task
-      add_transaction_tracer :filter_jobs,      category: :task
     end
 
   end

data/lib/rest-ftp-daemon/views/dashboard_jobs.haml

@@ -32,6 +32,7 @@
         %tr
           %th ID
           %th label
+          %th pool
           %th source
           %th mode
           %th target
@@ -44,15 +45,20 @@
           %th{title: "Priority"} P
           %th{title: "Runs count"} R
 
-        - unless @queue.empty?
-          %tbody.jobs
-            = render :dashboard_table, {jobs: @queue}
 
-            %thead
-              %tr
-                %td{colspan: 13}
-                  %br
+        - @jobs_queued.each do |pool, jobs|
+          - unless jobs.empty?
+            %tbody.jobs
+              = render :dashboard_table, {jobs: jobs}
+
+              %thead
+                %tr
+                  %td{colspan: 14}
+                    %br
 
         - unless jobs.empty?
           %tbody.jobs
             = render :dashboard_table, {jobs: jobs}
+
+
+

data/lib/rest-ftp-daemon/views/dashboard_table.haml

@@ -18,6 +18,8 @@
 
     %td= job.label
 
+    %td= job.pool
+
     %td{title: job.get(:source_path)}
       = Helpers.highlight_tokens job.source
 

data/lib/rest-ftp-daemon/views/dashboard_workers.haml

@@ -5,6 +5,7 @@
 
   %tr
     %th ID
+    %th pool
     %th status
     %th job
     %th.text-right seen
@@ -22,6 +23,7 @@
 
     %tr{class: trclass.to_s}
       %td= wid
+      %td= vars[:pool]
       %td= status
       %td= vars[:jid]
       %td.text-right

data/lib/rest-ftp-daemon/worker.rb

@@ -7,15 +7,17 @@ module RestFtpDaemon
       include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
     end
 
-    def initialize wid
+    def initialize wid, pool = nil
       # Logger
       @logger = RestFtpDaemon::LoggerPool.instance.get :workers
       @log_worker_status_changes = true
 
       # Worker name
-      @wid = wid
+      #@wid = wid
+      @pool = pool
 
       # Set thread context
+      Thread.current.thread_variable_set :pool, pool
       Thread.current.thread_variable_set :wid, wid
       Thread.current.thread_variable_set :started_at, Time.now
       worker_status WORKER_STATUS_STARTING
@@ -25,7 +27,7 @@ module RestFtpDaemon
 
     def log_context
       {
-        wid: @wid,
+        wid: Thread.current.thread_variable_get(:wid),
         jid: Thread.current.thread_variable_get(:jid),
       }
     end

data/lib/rest-ftp-daemon/worker_job.rb

@@ -2,8 +2,9 @@ module RestFtpDaemon
 
   # Worker used to process Jobs
   class JobWorker < Worker
+    #attr_reader :pool
 
-    def initialize wid
+    def initialize wid, pool
       # Generic worker initialize
       super
 
@@ -11,7 +12,7 @@ module RestFtpDaemon
       @timeout = (Settings.transfer.timeout rescue nil) || DEFAULT_WORKER_TIMEOUT
 
       # Start main loop
-      log_info "JobWorker starting", ["timeout: #{@timeout}"]
+      log_info "JobWorker initializing", ["wid: #{wid}", "pool: #{pool}", "timeout: #{@timeout}"]
       start
     end
 
@@ -20,14 +21,14 @@ module RestFtpDaemon
     def work
       # Wait for a job to be available in the queue
       worker_status WORKER_STATUS_WAITING
-      job = $queue.pop
+      job = $queue.pop @pool
 
       # Work on this job
       work_on_job(job)
 
-      # Clean job status and sleep for 1s
+      # Clean job status
       job.wid = nil
-      sleep 1
+      #sleep 1
 
       # If job status requires a retry, just restack it
       on_errors = Settings.at(:retry, :on_errors)

data/lib/rest-ftp-daemon/worker_pool.rb

@@ -4,7 +4,6 @@ module RestFtpDaemon
   class WorkerPool
     include LoggerHelper
     attr_reader :logger
-
     attr_reader :wid
 
     if Settings.newrelic_enabled?
@@ -53,34 +52,38 @@ module RestFtpDaemon
 
     def create_threads
       # Read configuration
-      number_threads = (Settings.workers || DEFAULT_WORKERS)
+      pools = (Settings.pools || {})
 
-      if number_threads < 1
-        log_error "create_threads: one worker is the minimum possible number (#{number_threads} configured)"
-        raise InvalidWorkerNumber
+      # Minimum one worker on DEFAULT_POOL
+      if !(pools.is_a? Hash)
+        log_error "create_threads: one worker is the minimum possible number (#{pools.inspect}"
       end
+      pools[DEFAULT_POOL] ||= 1
 
       # Create workers
-      log_info "WorkerPool creating #{number_threads}x JobWorker, 1x ConchitaWorker"
-
-      # Start worker threads
-      number_threads.times do
-        wid = generate_id
-        @workers[wid] = create_worker_thread wid
+      log_info "WorkerPool creating JobWorker's #{pools.inspect} + ConchitaWorker"
+      pools.each do |pool, count|
+        # Start worker threads for each pool
+        log_info "WorkerPool creating JobWorker [#{pool}] x#{count}"
+        count.times do
+          wid = generate_id
+          @workers[wid] = create_worker_thread wid, pool
+        end
       end
 
       # Start conchita thread
+      log_info "WorkerPool creating ConchitaWorker"
       @conchita = create_conchita_thread
 
     rescue StandardError => ex
       log_error "UNHANDLED EXCEPTION: #{ex.message}", ex.backtrace
     end
 
-    def create_worker_thread wid
+    def create_worker_thread wid, pool
       Thread.new wid do
         begin
-          worker = JobWorker.new wid
-          log_info "JobWorker [#{wid}]: #{worker}"
+          worker = JobWorker.new wid, pool
+          log_info "JobWorker [#{wid}][#{pool}]: #{worker}"
         rescue StandardError => ex
           log_error "JobWorker EXCEPTION: #{ex.message}"
         end

data/spec/rest-ftp-daemon/features/routes_spec.rb

@@ -10,11 +10,6 @@ describe "Routes", feature: true do
       expect(response.status).to eq 200
     end
 
-    it "exposes properly formed JSON" do
-      expect { JSON.parse(response.to_s) }.not_to raise_error
-      expect(JSON.parse(response.to_s)).to all(have_key("options"))
-    end
-
   end # GET /
 
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rest-ftp-daemon
 version: !ruby/object:Gem::Version
-  version: 0.243.2
+  version: '0.245'
 platform: ruby
 authors:
 - Bruno MEDICI