pallets 0.10.0 → 0.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 715e8a2c758fa925afce7091b931d4ea71b02f548290d4daf73604cdd463de3d
-  data.tar.gz: 750fd79e4d8aea7920ac4a019ac410d2ec07b910a1cf70e673a628d1604a90de
+  metadata.gz: c7ee0fbd91594f23bc84b81116f867490620f580223195f69fa9ff4f72be2558
+  data.tar.gz: b681bc765933ce36d3732a04eae3b9b7d60c2695b3d47f5704ebcfe96e3f884b
 SHA512:
-  metadata.gz: f3c7d08ad9b0de820f14ec4e3a3bbce77d5e10e9d6bfd99b58273360ce8dc6eeedec8d139d0f9445082848f5b9587dc67b706e49794746cc3208d6cde3aa9516
-  data.tar.gz: '0584d9d3faaebea5c615ee6d9c6dc0397d4e65c04cd80b1722a4cea2794bcf5c61a4762a539a2abf2d21e6a9f52de9a7b8f8e5241615168461bdb674a1465772'
+  metadata.gz: 8dfe6d23000df032d7f65fd81bd7907a3600240eaa6f111a161d1b2fb6d234440c8cc9219203a094ae139847ba00011939b50943e5eaaa04c11dde9128c3bb94
+  data.tar.gz: c054b1abd6d25b0408fa8427550751c8c8d41cd204bed65a968302d2ad5be98ac5ef020a6d096efc52ec58e9fef6f66456528ea72cdf423666916dd3a1394a5f

data/CHANGELOG.md

@@ -6,6 +6,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.11.0] - 2020-10-09
+### Added
+- JSON logger (#62)
+
+### Changed
+- fix for leaking Redis connections (#61)
+
 ## [0.10.0] - 2020-08-30
 ### Added
 - configure polling interval for scheduler (#60)

data/lib/pallets/logger.rb

@@ -9,8 +9,7 @@ module Pallets
         metadata = Thread.current[:pallets_log_metadata]
         return super(message) if metadata.nil?
 
-        formatted_metadata = ' ' + metadata.map { |k, v| "#{k}=#{v}" }.join(' ')
-        super(formatted_metadata) { message }
+        super(metadata) { message }
       end
     end
 
@@ -24,7 +23,19 @@ module Pallets
     module Formatters
       class Pretty < ::Logger::Formatter
         def call(severity, time, metadata, message)
-          "#{time.utc.iso8601(4)} pid=#{Process.pid}#{metadata} #{severity}: #{message}\n"
+          formatted_metadata = ' ' + metadata.map { |k, v| "#{k}=#{v}" }.join(' ') if metadata
+          "#{time.utc.iso8601(4)} pid=#{Process.pid}#{formatted_metadata} #{severity}: #{message}\n"
+        end
+      end
+
+      class Json < ::Logger::Formatter
+        def call(severity, time, metadata, message)
+          {
+            timestamp: time.utc.iso8601(4),
+            pid: Process.pid,
+            severity: severity,
+            message: message
+          }.merge(metadata || {}).to_json + "\n"
         end
       end
     end

data/lib/pallets/manager.rb

@@ -3,8 +3,9 @@ module Pallets
     attr_reader :workers, :scheduler
 
     def initialize(concurrency: Pallets.configuration.concurrency)
-      @workers = concurrency.times.map { Worker.new(self) }
-      @scheduler = Scheduler.new(self)
+      @backend = Pallets.backend
+      @workers = concurrency.times.map { Worker.new(self, @backend) }
+      @scheduler = Scheduler.new(self, @backend)
       @lock = Mutex.new
       @needs_to_stop = false
     end
@@ -48,7 +49,7 @@ module Pallets
 
         return if @needs_to_stop
 
-        worker = Worker.new(self)
+        worker = Worker.new(self, @backend)
         @workers << worker
         worker.start
       end

data/lib/pallets/scheduler.rb

@@ -1,7 +1,8 @@
 module Pallets
   class Scheduler
-    def initialize(manager)
+    def initialize(manager, backend)
       @manager = manager
+      @backend = backend
       @needs_to_stop = false
       @thread = nil
     end
@@ -35,7 +36,7 @@ module Pallets
       loop do
         break if needs_to_stop?
 
-        backend.reschedule_all(Time.now.to_f)
+        @backend.reschedule_all(Time.now.to_f)
         wait_a_bit
       end
     end
@@ -55,9 +56,5 @@ module Pallets
         sleep 1
       end
     end
-
-    def backend
-      @backend ||= Pallets.backend
-    end
   end
 end

data/lib/pallets/version.rb

@@ -1,3 +1,3 @@
 module Pallets
-  VERSION = "0.10.0"
+  VERSION = "0.11.0"
 end

data/lib/pallets/worker.rb

@@ -1,9 +1,8 @@
 module Pallets
   class Worker
-    attr_reader :manager
-
-    def initialize(manager)
+    def initialize(manager, backend)
       @manager = manager
+      @backend = backend
       @current_job = nil
       @needs_to_stop = false
       @thread = nil
@@ -40,7 +39,7 @@ module Pallets
       loop do
         break if needs_to_stop?
 
-        @current_job = backend.pick
+        @current_job = @backend.pick
         # No need to requeue because of the reliability queue
         break if needs_to_stop?
         next if @current_job.nil?
@@ -66,13 +65,13 @@ module Pallets
       rescue
         # We ensure only valid jobs are created. If something fishy reaches this
         # point, just give up on it
-        backend.discard(job)
+        @backend.discard(job)
         Pallets.logger.error "Could not deserialize #{job}. Gave up job"
         return
       end
 
       context = Context[
-        serializer.load_context(backend.get_context(job_hash['wfid']))
+        serializer.load_context(@backend.get_context(job_hash['wfid']))
       ]
 
       task_class = Pallets::Util.constantize(job_hash["task_class"])
@@ -103,9 +102,9 @@ module Pallets
       ))
       if failures < job_hash['max_failures']
         retry_at = Time.now.to_f + backoff_in_seconds(failures)
-        backend.retry(new_job, job, retry_at)
+        @backend.retry(new_job, job, retry_at)
       else
-        backend.give_up(job_hash['wfid'], new_job, job)
+        @backend.give_up(job_hash['wfid'], new_job, job)
       end
     end
 
@@ -114,21 +113,17 @@ module Pallets
         'given_up_at' => Time.now.to_f,
         'reason' => 'returned_false'
       ))
-      backend.give_up(job_hash['wfid'], new_job, job)
+      @backend.give_up(job_hash['wfid'], new_job, job)
     end
 
     def handle_job_success(context, job, job_hash)
-      backend.save(job_hash['wfid'], job_hash['jid'], job, serializer.dump_context(context.buffer))
+      @backend.save(job_hash['wfid'], job_hash['jid'], job, serializer.dump_context(context.buffer))
     end
 
     def backoff_in_seconds(count)
       count ** 4 + rand(6..10)
     end
 
-    def backend
-      @backend ||= Pallets.backend
-    end
-
     def serializer
       @serializer ||= Pallets.serializer
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pallets
 version: !ruby/object:Gem::Version
-  version: 0.10.0
+  version: 0.11.0
 platform: ruby
 authors:
 - Andrei Horak
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-08-30 00:00:00.000000000 Z
+date: 2020-10-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis