insque 0.6.3 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: bc23ed058d9cee97359aa01f09f125531c53dc58
-  data.tar.gz: bac388403d7c9b289723ce47f078d7b9531f89b0
+SHA256:
+  metadata.gz: 4f1434665ea4206ce27e4120c4a08830a43f1e33ee50a7e68c986977f76b6fa4
+  data.tar.gz: 220ebc3acda7272151365db0532938362fddaea061511b5cda4281314014457c
 SHA512:
-  metadata.gz: 898027b6d227a907b8edb87a7a3d3ccfb0fb2ba0e888f63e01429773b2ab363a77668111bba6008d32c1f063003f8d418fda437f85aa177b3dcdb098f2d9a8c6
-  data.tar.gz: cb589c1f9a3d0e77ad0669d8fffdc62cf20a770d70de4dc9b2fcb336bd8ff1a3860aec56c249bdd77e6e28a446cafcf25a7ec2ad1f78fb228a8e5268e3875aeb
+  metadata.gz: 8bbad94b06e41bc20b709a3f70ebe8c8c7171a2dbe2d07cff93f8bd349ee5a95abaa22c1e839177ad79d638e48fd37096f3a9e4a3ec4cf49c9d2481cdd3917d2
+  data.tar.gz: 5b4348daa91b43d70fc609a15741a1c99c0933bcf14d376243f8a00dc20295d9cfb399ee3c1570b61ade85ffd53d1fb7bebfe30be6ec61b5ada01936deab7727

data/insque.local.yml

@@ -3,6 +3,7 @@ version: '3.2'
 services:
   redis:
     image: redis
+    command: redis-server --save "" --appendonly no
     ports:
       - target: 6379
         published: 63790

data/lib/insque/json_formatter.rb

@@ -0,0 +1,21 @@
+module Insque
+  class JsonFormatter < ::Logger::Formatter
+    def call(severity, time, progname, msg, additional_fields = {})
+      message = case msg
+                when ::String
+                  { message: msg }
+                when ::Hash
+                  msg
+                when ::Exception
+                  {
+                    error: "#{ msg.message } (#{ msg.class })",
+                    error_class: "#{msg.class}",
+                    backtrace: (msg.backtrace || []).join("\n\t")
+                  }
+                else
+                  { message: msg.inspect }
+                end
+      "#{message.merge(timestamp: time, level: severity).merge(additional_fields).to_json}\n"
+    end
+  end
+end

data/lib/insque/json_logger.rb

@@ -0,0 +1,19 @@
+module Insque
+  class JsonLogger < ::Logger
+    attr_accessor :additional_fields
+
+    def format_message(severity, datetime, progname, msg)
+      @json_formatter ||= JsonFormatter.new
+      @additional_fields ||= {}
+      (@formatter || @json_formatter).call(severity, datetime, progname, msg, @additional_fields)
+    end
+
+    private
+    # Severity label for logging (max 5 chars).
+    SEV_LABEL = %w(debug info warn error fatal any).each(&:freeze).freeze
+
+    def format_severity(severity)
+      SEV_LABEL[severity] || 'any'
+    end
+  end
+end

data/lib/insque/version.rb

@@ -1,3 +1,3 @@
 module Insque
-  VERSION = "0.6.3"
+  VERSION = "0.7.0"
 end

data/lib/insque.rb

@@ -1,6 +1,8 @@
 require "insque/version"
 require "redis"
-require 'json'
+require "json"
+require "insque/json_formatter"
+require "insque/json_logger"
 require "insque/railtie" if defined?(Rails)
 
 module Insque
@@ -23,10 +25,6 @@ module Insque
     @processing_ttl || DEFAULT_PROCESSING_TTL
   end
 
-  def self.debug= debug
-    @debug = debug
-  end
-
   def self.redis= redis
     @redis = redis
   end
@@ -35,6 +33,18 @@ module Insque
     @redis
   end
 
+  def self.logger= l
+    @logger = l
+  end
+
+  def self.logger
+    unless @logger
+      @logger = JsonLogger.new(STDOUT)
+      @logger.additional_fields = { tag: 'insque' }
+    end
+    @logger
+  end
+
   def self.redis_class= klass
     @redis_class = klass
   end
@@ -72,7 +82,7 @@ module Insque
       keys = recipient.is_a?(Array) ? recipient : [recipient]
     end
     value = { message: "#{@sender}_#{message}", params: params, broadcasted_at: Time.now.utc }
-    log "SENDING: #{value.to_json} TO #{keys.to_json}" if @debug
+    logger.debug sending: value, to: keys
     @redis.multi do |r|
       keys.each {|k| r.lpush k, value.to_json}
     end
@@ -97,19 +107,17 @@ module Insque
 
 private
   def self.do_listen inbox, processing, redis, worker_name, pointer=nil
-    log "#{worker_name} START LISTENING #{inbox}"
+    logger.info starting: worker_name, listening: inbox
     loop do
       redis.setex(pointer, inbox_ttl, inbox) if pointer
       message = redis.brpoplpush(inbox, processing, 0)
       begin
-        log "#{worker_name} RECEIVING: #{message}" if @debug
         parsed_message = JSON.parse message
+        logger.debug receiving: parsed_message, worker: worker_name
         send(parsed_message['message'], parsed_message) 
       rescue NoMethodError
       rescue => e
-        log "#{worker_name} ========== BROKEN_MESSAGE: #{message} =========="
-        log e.inspect
-        log e.backtrace
+        logger.error e
       ensure
         redis.lrem processing, 0, message
       end
@@ -134,9 +142,7 @@ private
             delete << m
           end
         rescue => e
-          log "========== JANITOR_BROKEN_MESSAGE: #{m} =========="
-          log e.inspect
-          log e.backtrace
+          logger.error e
         end
       end
       result = redis.multi do |r|
@@ -144,11 +150,11 @@ private
         delete.each {|m| r.lrem processing, 0, m }
       end
       if result
-        errors.each {|m| log "ERROR: #{m.to_json}" }
-        restart.each {|m| log "RESTART: #{m.to_json}" }
-        log "CLEANING #{inbox} SUCCESSFULL"
+        errors.each {|m| logger.info deleting: m }
+        restart.each {|m| logger.info restarting: m }
+        logger.info cleaning: 'success', inbox: inbox
       else
-        log "CLEANING #{inbox} FAILED"
+        logger.info cleaning: 'failed', inbox: inbox
       end
       sleep(Random.rand((inbox_ttl.to_f / 10).ceil) + 1)
     end
@@ -158,11 +164,6 @@ private
     (@redis_class || Redis).new @redis_config
   end 
 
-  def self.log message
-    print "#{Time.now.utc} #{message}\n"
-    STDOUT.flush if @debug
-  end
-
   def self.create_send_later_handler
     define_singleton_method("#{@sender}_send_later") do |msg|
       Kernel.const_get(msg['params']['class']).unscoped.find(msg['params']['id']).send(msg['params']['method'], *msg['params']['args'])      

data/spec/insque_spec.rb

@@ -36,7 +36,7 @@ RSpec.describe 'insque' do
     system "docker stack deploy -c insque.local.yml insque"
     sleep 10
     Thread.abort_on_exception=true
-    Insque.debug = true
+    Insque.logger.level = :error
     Insque.sender = 'myapp'
     Insque.inbox_ttl = 3
     Insque.redis_config = { host: 'localhost', port: 63790 }
@@ -54,6 +54,9 @@ RSpec.describe 'insque' do
 
   before(:each) do
     Insque.redis.flushall
+  rescue
+    sleep 10
+    Insque.redis.flushall
   end
 
   it "can broadcast without listeners" do
@@ -76,7 +79,7 @@ RSpec.describe 'insque' do
   it "restarts broken message" do
     janitor = Thread.new { Insque.janitor }
     Insque.redis.lpush('{insque}processing_myapp', { message: 'myapp_test', broadcasted_at: 1.hour.ago }.to_json)
-    sleep 3
+    sleep 4
     expect(Insque.redis.llen '{insque}processing_myapp').to eq(0)
     expect(Insque.redis.llen '{insque}inbox_myapp').to eq(1)
     janitor.exit

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: insque
 version: !ruby/object:Gem::Version
-  version: 0.6.3
+  version: 0.7.0
 platform: ruby
 authors:
 - Yuri Gomozov
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-04-21 00:00:00.000000000 Z
+date: 2019-01-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -113,6 +113,8 @@ files:
 - lib/generators/insque/templates/insque.erb
 - lib/generators/insque/templates/redis.yml
 - lib/insque.rb
+- lib/insque/json_formatter.rb
+- lib/insque/json_logger.rb
 - lib/insque/railtie.rb
 - lib/insque/version.rb
 - lib/tasks/insque.rake
@@ -138,7 +140,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.7.8
 signing_key: 
 specification_version: 4
 summary: Redis-based multi-threaded queue