insque 0.6.3 → 0.7.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/insque.local.yml +1 -0
- data/lib/insque/json_formatter.rb +21 -0
- data/lib/insque/json_logger.rb +19 -0
- data/lib/insque/version.rb +1 -1
- data/lib/insque.rb +24 -23
- data/spec/insque_spec.rb +5 -2
- metadata +5 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 4f1434665ea4206ce27e4120c4a08830a43f1e33ee50a7e68c986977f76b6fa4
|
4
|
+
data.tar.gz: 220ebc3acda7272151365db0532938362fddaea061511b5cda4281314014457c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8bbad94b06e41bc20b709a3f70ebe8c8c7171a2dbe2d07cff93f8bd349ee5a95abaa22c1e839177ad79d638e48fd37096f3a9e4a3ec4cf49c9d2481cdd3917d2
|
7
|
+
data.tar.gz: 5b4348daa91b43d70fc609a15741a1c99c0933bcf14d376243f8a00dc20295d9cfb399ee3c1570b61ade85ffd53d1fb7bebfe30be6ec61b5ada01936deab7727
|
data/insque.local.yml
CHANGED
@@ -0,0 +1,21 @@
|
|
1
|
+
module Insque
|
2
|
+
class JsonFormatter < ::Logger::Formatter
|
3
|
+
def call(severity, time, progname, msg, additional_fields = {})
|
4
|
+
message = case msg
|
5
|
+
when ::String
|
6
|
+
{ message: msg }
|
7
|
+
when ::Hash
|
8
|
+
msg
|
9
|
+
when ::Exception
|
10
|
+
{
|
11
|
+
error: "#{ msg.message } (#{ msg.class })",
|
12
|
+
error_class: "#{msg.class}",
|
13
|
+
backtrace: (msg.backtrace || []).join("\n\t")
|
14
|
+
}
|
15
|
+
else
|
16
|
+
{ message: msg.inspect }
|
17
|
+
end
|
18
|
+
"#{message.merge(timestamp: time, level: severity).merge(additional_fields).to_json}\n"
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
module Insque
|
2
|
+
class JsonLogger < ::Logger
|
3
|
+
attr_accessor :additional_fields
|
4
|
+
|
5
|
+
def format_message(severity, datetime, progname, msg)
|
6
|
+
@json_formatter ||= JsonFormatter.new
|
7
|
+
@additional_fields ||= {}
|
8
|
+
(@formatter || @json_formatter).call(severity, datetime, progname, msg, @additional_fields)
|
9
|
+
end
|
10
|
+
|
11
|
+
private
|
12
|
+
# Severity label for logging (max 5 chars).
|
13
|
+
SEV_LABEL = %w(debug info warn error fatal any).each(&:freeze).freeze
|
14
|
+
|
15
|
+
def format_severity(severity)
|
16
|
+
SEV_LABEL[severity] || 'any'
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
data/lib/insque/version.rb
CHANGED
data/lib/insque.rb
CHANGED
@@ -1,6 +1,8 @@
|
|
1
1
|
require "insque/version"
|
2
2
|
require "redis"
|
3
|
-
require
|
3
|
+
require "json"
|
4
|
+
require "insque/json_formatter"
|
5
|
+
require "insque/json_logger"
|
4
6
|
require "insque/railtie" if defined?(Rails)
|
5
7
|
|
6
8
|
module Insque
|
@@ -23,10 +25,6 @@ module Insque
|
|
23
25
|
@processing_ttl || DEFAULT_PROCESSING_TTL
|
24
26
|
end
|
25
27
|
|
26
|
-
def self.debug= debug
|
27
|
-
@debug = debug
|
28
|
-
end
|
29
|
-
|
30
28
|
def self.redis= redis
|
31
29
|
@redis = redis
|
32
30
|
end
|
@@ -35,6 +33,18 @@ module Insque
|
|
35
33
|
@redis
|
36
34
|
end
|
37
35
|
|
36
|
+
def self.logger= l
|
37
|
+
@logger = l
|
38
|
+
end
|
39
|
+
|
40
|
+
def self.logger
|
41
|
+
unless @logger
|
42
|
+
@logger = JsonLogger.new(STDOUT)
|
43
|
+
@logger.additional_fields = { tag: 'insque' }
|
44
|
+
end
|
45
|
+
@logger
|
46
|
+
end
|
47
|
+
|
38
48
|
def self.redis_class= klass
|
39
49
|
@redis_class = klass
|
40
50
|
end
|
@@ -72,7 +82,7 @@ module Insque
|
|
72
82
|
keys = recipient.is_a?(Array) ? recipient : [recipient]
|
73
83
|
end
|
74
84
|
value = { message: "#{@sender}_#{message}", params: params, broadcasted_at: Time.now.utc }
|
75
|
-
|
85
|
+
logger.debug sending: value, to: keys
|
76
86
|
@redis.multi do |r|
|
77
87
|
keys.each {|k| r.lpush k, value.to_json}
|
78
88
|
end
|
@@ -97,19 +107,17 @@ module Insque
|
|
97
107
|
|
98
108
|
private
|
99
109
|
def self.do_listen inbox, processing, redis, worker_name, pointer=nil
|
100
|
-
|
110
|
+
logger.info starting: worker_name, listening: inbox
|
101
111
|
loop do
|
102
112
|
redis.setex(pointer, inbox_ttl, inbox) if pointer
|
103
113
|
message = redis.brpoplpush(inbox, processing, 0)
|
104
114
|
begin
|
105
|
-
log "#{worker_name} RECEIVING: #{message}" if @debug
|
106
115
|
parsed_message = JSON.parse message
|
116
|
+
logger.debug receiving: parsed_message, worker: worker_name
|
107
117
|
send(parsed_message['message'], parsed_message)
|
108
118
|
rescue NoMethodError
|
109
119
|
rescue => e
|
110
|
-
|
111
|
-
log e.inspect
|
112
|
-
log e.backtrace
|
120
|
+
logger.error e
|
113
121
|
ensure
|
114
122
|
redis.lrem processing, 0, message
|
115
123
|
end
|
@@ -134,9 +142,7 @@ private
|
|
134
142
|
delete << m
|
135
143
|
end
|
136
144
|
rescue => e
|
137
|
-
|
138
|
-
log e.inspect
|
139
|
-
log e.backtrace
|
145
|
+
logger.error e
|
140
146
|
end
|
141
147
|
end
|
142
148
|
result = redis.multi do |r|
|
@@ -144,11 +150,11 @@ private
|
|
144
150
|
delete.each {|m| r.lrem processing, 0, m }
|
145
151
|
end
|
146
152
|
if result
|
147
|
-
errors.each {|m|
|
148
|
-
restart.each {|m|
|
149
|
-
|
153
|
+
errors.each {|m| logger.info deleting: m }
|
154
|
+
restart.each {|m| logger.info restarting: m }
|
155
|
+
logger.info cleaning: 'success', inbox: inbox
|
150
156
|
else
|
151
|
-
|
157
|
+
logger.info cleaning: 'failed', inbox: inbox
|
152
158
|
end
|
153
159
|
sleep(Random.rand((inbox_ttl.to_f / 10).ceil) + 1)
|
154
160
|
end
|
@@ -158,11 +164,6 @@ private
|
|
158
164
|
(@redis_class || Redis).new @redis_config
|
159
165
|
end
|
160
166
|
|
161
|
-
def self.log message
|
162
|
-
print "#{Time.now.utc} #{message}\n"
|
163
|
-
STDOUT.flush if @debug
|
164
|
-
end
|
165
|
-
|
166
167
|
def self.create_send_later_handler
|
167
168
|
define_singleton_method("#{@sender}_send_later") do |msg|
|
168
169
|
Kernel.const_get(msg['params']['class']).unscoped.find(msg['params']['id']).send(msg['params']['method'], *msg['params']['args'])
|
data/spec/insque_spec.rb
CHANGED
@@ -36,7 +36,7 @@ RSpec.describe 'insque' do
|
|
36
36
|
system "docker stack deploy -c insque.local.yml insque"
|
37
37
|
sleep 10
|
38
38
|
Thread.abort_on_exception=true
|
39
|
-
Insque.
|
39
|
+
Insque.logger.level = :error
|
40
40
|
Insque.sender = 'myapp'
|
41
41
|
Insque.inbox_ttl = 3
|
42
42
|
Insque.redis_config = { host: 'localhost', port: 63790 }
|
@@ -54,6 +54,9 @@ RSpec.describe 'insque' do
|
|
54
54
|
|
55
55
|
before(:each) do
|
56
56
|
Insque.redis.flushall
|
57
|
+
rescue
|
58
|
+
sleep 10
|
59
|
+
Insque.redis.flushall
|
57
60
|
end
|
58
61
|
|
59
62
|
it "can broadcast without listeners" do
|
@@ -76,7 +79,7 @@ RSpec.describe 'insque' do
|
|
76
79
|
it "restarts broken message" do
|
77
80
|
janitor = Thread.new { Insque.janitor }
|
78
81
|
Insque.redis.lpush('{insque}processing_myapp', { message: 'myapp_test', broadcasted_at: 1.hour.ago }.to_json)
|
79
|
-
sleep
|
82
|
+
sleep 4
|
80
83
|
expect(Insque.redis.llen '{insque}processing_myapp').to eq(0)
|
81
84
|
expect(Insque.redis.llen '{insque}inbox_myapp').to eq(1)
|
82
85
|
janitor.exit
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: insque
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.7.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Yuri Gomozov
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2019-01-25 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: redis
|
@@ -113,6 +113,8 @@ files:
|
|
113
113
|
- lib/generators/insque/templates/insque.erb
|
114
114
|
- lib/generators/insque/templates/redis.yml
|
115
115
|
- lib/insque.rb
|
116
|
+
- lib/insque/json_formatter.rb
|
117
|
+
- lib/insque/json_logger.rb
|
116
118
|
- lib/insque/railtie.rb
|
117
119
|
- lib/insque/version.rb
|
118
120
|
- lib/tasks/insque.rake
|
@@ -138,7 +140,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
138
140
|
version: '0'
|
139
141
|
requirements: []
|
140
142
|
rubyforge_project:
|
141
|
-
rubygems_version: 2.
|
143
|
+
rubygems_version: 2.7.8
|
142
144
|
signing_key:
|
143
145
|
specification_version: 4
|
144
146
|
summary: Redis-based multi-threaded queue
|