pallets 0.7.0 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c9f4f127abbba26bbab9303e347764dd08495034b64ee9eba41d8b995756cf4c
-  data.tar.gz: 89e22edf0746dde5c63602d0dc25176aae6e6a7a8e7fc2e7f7785c5d8487770e
+  metadata.gz: 46a631b6e48f2f1c4efe5bce09c5d74fa2cb064a78d5e546492c6c8ce1dec843
+  data.tar.gz: 61f6414e16941de41defb30ac8dc9121703f75251e47422239c311a8b7749787
 SHA512:
-  metadata.gz: 4a6399b78b97b5ba1a21d2b73c3e24218d4b0b4244f8384c0bf01a9fa8547f479f8c5ae52899f66765eb66adbb49986a6c455f08fff45178a074d88996d0f748
-  data.tar.gz: e8b8117b9cbfaf10f1a6912128fa9a8b9b746a53664ce120d2d9299e7d7ea416bffdf986578629e75518eebd9ebb251a50cdf4c1a4d066dfd070c7294ee625e8
+  metadata.gz: 4b6f77bd93576dc7dc2c5cf945d52c425a63178740c12fa958f0c18c4ca870da7b02d303c7b8e871504d84a13461f79edc52da2869069322f5a81943bdbd79df
+  data.tar.gz: a2a71f4a2343927a94871523b7475f587bb64181e9c6a4654a40111453e2fc2f20e82a02da44adbc32865f68c53a95dd3a9c6f0b641cfd85c7939f1e06d3521f

data/.travis.yml

@@ -4,8 +4,8 @@ services:
   - redis-server
 cache: bundler
 rvm:
-  - 2.4.6
-  - 2.5.5
-  - 2.6.3
-  - 2.7.0
+  - 2.4.10
+  - 2.5.8
+  - 2.6.6
+  - 2.7.1
 script: bundle exec rspec

data/CHANGELOG.md

@@ -6,6 +6,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.8.0] - 2020-06-09
+### Added
+- sync output in CLI (#49)
+- support for configuring custom loggers (#50)
+
+### Changed
+- improve job scheduling using jobmasks (#52)
+
 ## [0.7.0] - 2020-01-19
 ### Added
 - support for Ruby 2.7 (#46)

data/examples/config_savvy.rb

@@ -1,3 +1,4 @@
+require 'logger'
 require 'pallets'
 
 class AnnounceProcessing
@@ -31,6 +32,8 @@ Pallets.configure do |c|
   # given up. Retry times are exponential and happen after: 7, 22, 87, 262, ...
   c.max_failures = 5
 
+  # Custom loggers can be used too
+  c.logger = Logger.new(STDOUT)
   # Job execution can be wrapped with middleware to provide custom logic.
   # Anything that responds to `call` would do
   c.middleware << AnnounceProcessing

data/lib/pallets.rb

@@ -57,13 +57,6 @@ module Pallets
   end
 
   def self.logger
-    @logger ||= Pallets::Logger.new(STDOUT,
-      level: Pallets::Logger::INFO,
-      formatter: Pallets::Logger::Formatters::Pretty.new
-    )
-  end
-
-  def self.logger=(logger)
-    @logger = logger
+    @logger ||= configuration.logger
   end
 end

data/lib/pallets/backends/base.rb

@@ -6,12 +6,12 @@ module Pallets
         raise NotImplementedError
       end
 
-      def get_context(workflow_id)
+      def get_context(wfid)
         raise NotImplementedError
       end
 
       # Saves a job after successfully processing it
-      def save(workflow_id, job, context_buffer)
+      def save(wfid, jid, job, context_buffer)
         raise NotImplementedError
       end
 
@@ -29,7 +29,7 @@ module Pallets
         raise NotImplementedError
       end
 
-      def run_workflow(workflow_id, jobs_with_dependencies, context)
+      def run_workflow(wfid, jobs, jobmasks, context)
         raise NotImplementedError
       end
     end

data/lib/pallets/backends/redis.rb

@@ -9,6 +9,7 @@ module Pallets
       RETRY_SET_KEY = 'retry-set'
       GIVEN_UP_SET_KEY = 'given-up-set'
       WORKFLOW_QUEUE_KEY = 'workflow-queue:%s'
+      JOBMASK_KEY = 'jobmask:%s'
       CONTEXT_KEY = 'context:%s'
       REMAINING_KEY = 'remaining:%s'
 
@@ -41,11 +42,11 @@ module Pallets
         end
       end
 
-      def save(wfid, job, context_buffer)
+      def save(wfid, jid, job, context_buffer)
         @pool.execute do |client|
           client.evalsha(
             @scripts['save'],
-            [WORKFLOW_QUEUE_KEY % wfid, QUEUE_KEY, RELIABILITY_QUEUE_KEY, RELIABILITY_SET_KEY, CONTEXT_KEY % wfid, REMAINING_KEY % wfid],
+            [WORKFLOW_QUEUE_KEY % wfid, QUEUE_KEY, RELIABILITY_QUEUE_KEY, RELIABILITY_SET_KEY, CONTEXT_KEY % wfid, REMAINING_KEY % wfid, JOBMASK_KEY % jid],
             context_buffer.to_a << job
           )
         end
@@ -81,13 +82,14 @@ module Pallets
         end
       end
 
-      def run_workflow(wfid, jobs_with_order, context_buffer)
+      def run_workflow(wfid, jobs, jobmasks, context_buffer)
         @pool.execute do |client|
           client.multi do
+            jobmasks.each { |jid, jobmask| client.zadd(JOBMASK_KEY % jid, jobmask) }
             client.evalsha(
               @scripts['run_workflow'],
               [WORKFLOW_QUEUE_KEY % wfid, QUEUE_KEY, REMAINING_KEY % wfid],
-              jobs_with_order
+              jobs
             )
             client.hmset(CONTEXT_KEY % wfid, *context_buffer.to_a) unless context_buffer.empty?
           end

data/lib/pallets/backends/scripts/run_workflow.lua

@@ -6,9 +6,8 @@ redis.call("SET", KEYS[3], eta)
 
 -- Queue jobs that are ready to be processed (their score is 0) and
 -- remove queued jobs from the sorted set
-local count = redis.call("ZCOUNT", KEYS[1], 0, 0)
-if count > 0 then
-  local work = redis.call("ZRANGEBYSCORE", KEYS[1], 0, 0)
+local work = redis.call("ZRANGEBYSCORE", KEYS[1], 0, 0)
+if #work > 0 then
   redis.call("LPUSH", KEYS[2], unpack(work))
   redis.call("ZREM", KEYS[1], unpack(work))
 end

data/lib/pallets/backends/scripts/save.lua

@@ -10,24 +10,21 @@ if #ARGV > 0 then
   redis.call("HMSET", KEYS[5], unpack(ARGV))
 end
 
--- Decrement all jobs from the sorted set
-local all_pending = redis.call("ZRANGE", KEYS[1], 0, -1)
-for score, task in pairs(all_pending) do
-  redis.call("ZINCRBY", KEYS[1], -1, task)
-end
+-- Decrement jobs from the sorted set by applying a jobmask
+redis.call("ZUNIONSTORE", KEYS[1], 2, KEYS[1], KEYS[7])
+redis.call("DEL", KEYS[7])
 
 -- Queue jobs that are ready to be processed (their score is 0) and
 -- remove queued jobs from sorted set
-local count = redis.call("ZCOUNT", KEYS[1], 0, 0)
-if count > 0 then
-  local work = redis.call("ZRANGEBYSCORE", KEYS[1], 0, 0)
+local work = redis.call("ZRANGEBYSCORE", KEYS[1], 0, 0)
+if #work > 0 then
   redis.call("LPUSH", KEYS[2], unpack(work))
   redis.call("ZREM", KEYS[1], unpack(work))
 end
 
 -- Decrement ETA and remove it together with the context if all tasks have
 -- been processed (ETA is 0)
-redis.call("DECR", KEYS[6])
-if tonumber(redis.call("GET", KEYS[6])) == 0 then
+local remaining = redis.call("DECR", KEYS[6])
+if remaining == 0 then
   redis.call("DEL", KEYS[5], KEYS[6])
 end

data/lib/pallets/cli.rb

@@ -1,5 +1,7 @@
 require 'optparse'
 
+$stdout.sync = true
+
 module Pallets
   class CLI
     def initialize

data/lib/pallets/configuration.rb

@@ -20,6 +20,9 @@ module Pallets
     # period, it is considered failed, and scheduled to be processed again
     attr_accessor :job_timeout
 
+    # Custom logger used throughout Pallets
+    attr_writer :logger
+
     # Maximum number of failures allowed per job. Can also be configured on a
     # per task basis
     attr_accessor :max_failures
@@ -51,6 +54,13 @@ module Pallets
       @middleware = default_middleware
     end
 
+    def logger
+      @logger || Pallets::Logger.new(STDOUT,
+        level: Pallets::Logger::INFO,
+        formatter: Pallets::Logger::Formatters::Pretty.new
+      )
+    end
+
     def pool_size
       @pool_size || @concurrency + 1
     end

data/lib/pallets/graph.rb

@@ -24,18 +24,11 @@ module Pallets
       nodes.empty?
     end
 
-    # Returns nodes topologically sorted, together with their order (number of
-    # nodes that have to be executed prior)
-    def sorted_with_order
-      # Identify groups of nodes that can be executed concurrently
-      groups = tsort_each.slice_when { |a, b| parents(a) != parents(b) }
-
-      # Assign order to each node
-      i = 0
-      groups.flat_map do |group|
-        group_with_order = group.product([i])
-        i += group.size
-        group_with_order
+    def each
+      return enum_for(__method__) unless block_given?
+
+      tsort_each do |node|
+        yield(node, parents(node))
       end
     end
 

data/lib/pallets/version.rb

@@ -1,3 +1,3 @@
 module Pallets
-  VERSION = "0.7.0"
+  VERSION = "0.8.0"
 end

data/lib/pallets/worker.rb

@@ -116,7 +116,7 @@ module Pallets
     end
 
     def handle_job_success(context, job, job_hash)
-      backend.save(job_hash['wfid'], job, serializer.dump_context(context.buffer))
+      backend.save(job_hash['wfid'], job_hash['jid'], job, serializer.dump_context(context.buffer))
     end
 
     def backoff_in_seconds(count)

data/lib/pallets/workflow.rb

@@ -20,7 +20,7 @@ module Pallets
       raise WorkflowError, "#{self.class.name} has no tasks. Workflows "\
                            "must contain at least one task" if self.class.graph.empty?
 
-      backend.run_workflow(id, jobs_with_order, serializer.dump_context(context.buffer))
+      backend.run_workflow(id, *prepare_jobs, serializer.dump_context(context.buffer))
       id
     end
 
@@ -30,11 +30,21 @@ module Pallets
 
     private
 
-    def jobs_with_order
-      self.class.graph.sorted_with_order.map do |task_alias, order|
-        job = serializer.dump(construct_job(task_alias))
-        [order, job]
+    def prepare_jobs
+      jobs = []
+      jobmasks = Hash.new { |h, k| h[k] = [] }
+      acc = {}
+
+      self.class.graph.each do |task_alias, dependencies|
+        job_hash = construct_job(task_alias)
+        acc[task_alias] = job_hash['jid']
+        job = serializer.dump(job_hash)
+
+        jobs << [dependencies.size, job]
+        dependencies.each { |d| jobmasks[acc[d]] << [-1, job] }
       end
+
+      [jobs, jobmasks]
     end
 
     def construct_job(task_alias)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pallets
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.8.0
 platform: ruby
 authors:
 - Andrei Horak
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-01-19 00:00:00.000000000 Z
+date: 2020-06-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis