pgq 0.1.1 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +1 -1
- data/Gemfile +0 -1
- data/Gemfile.lock +4 -2
- data/LICENSE +1 -1
- data/README.md +1 -17
- data/Rakefile +1 -1
- data/lib/generators/pgq/add_generator.rb +1 -1
- data/lib/pgq/api.rb +51 -14
- data/lib/pgq/consumer_group.rb +19 -19
- data/lib/pgq/utils.rb +18 -55
- data/lib/pgq/version.rb +1 -1
- data/pgq.gemspec +1 -0
- metadata +21 -7
data/.gitignore
CHANGED
data/Gemfile
CHANGED
data/Gemfile.lock
CHANGED
@@ -1,9 +1,10 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
pgq (0.1.
|
4
|
+
pgq (0.1.2)
|
5
5
|
activerecord (>= 2.3.2)
|
6
6
|
activesupport (>= 2.3.2)
|
7
|
+
pg
|
7
8
|
|
8
9
|
GEM
|
9
10
|
remote: http://rubygems.org/
|
@@ -23,7 +24,8 @@ GEM
|
|
23
24
|
builder (3.0.0)
|
24
25
|
diff-lcs (1.1.3)
|
25
26
|
i18n (0.6.0)
|
26
|
-
multi_json (1.3.
|
27
|
+
multi_json (1.3.5)
|
28
|
+
pg (0.13.2)
|
27
29
|
rake (0.9.2.2)
|
28
30
|
rspec (2.10.0)
|
29
31
|
rspec-core (~> 2.10.0)
|
data/LICENSE
CHANGED
@@ -17,4 +17,4 @@ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
|
17
17
|
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
18
|
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
19
|
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
-
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
20
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
CHANGED
@@ -122,24 +122,8 @@ and run:
|
|
122
122
|
|
123
123
|
|
124
124
|
|
125
|
-
|
126
125
|
### Admin interface
|
127
|
-
|
128
|
-
For queues info, run in console:
|
129
|
-
|
130
|
-
> Pgq::Consumer.database.pgq_get_consumer_info
|
131
|
-
|
132
|
-
|
133
|
-
### Failed events
|
134
|
-
When any raise happens in consumer, its produce failed event, which can be retry, or delete.
|
135
|
-
|
136
|
-
Retry manual:
|
137
|
-
|
138
|
-
Pgq::Consumer.retry_failed_events(queue_name)
|
139
|
-
|
140
|
-
Delete manual:
|
141
|
-
|
142
|
-
Pgq::Consumer.delete_failed_events(queue_name)
|
126
|
+
Realized by gem [pgq_web](http://github.com/kostya/pgq_web).
|
143
127
|
|
144
128
|
|
145
129
|
### Divide events between workers, for one consumer class
|
data/Rakefile
CHANGED
data/lib/pgq/api.rb
CHANGED
@@ -11,12 +11,12 @@ module Pgq::Api
|
|
11
11
|
connection.select_value(sanitize_sql_array ["SELECT pgq.drop_queue(?)", queue_name]).to_i
|
12
12
|
end
|
13
13
|
|
14
|
-
def pgq_register_consumer(queue_name,
|
15
|
-
connection.select_value(sanitize_sql_array ["SELECT pgq.register_consumer(?, ?)", queue_name,
|
14
|
+
def pgq_register_consumer(queue_name, consumer_name)
|
15
|
+
connection.select_value(sanitize_sql_array ["SELECT pgq.register_consumer(?, ?)", queue_name, consumer_name]).to_i
|
16
16
|
end
|
17
17
|
|
18
|
-
def pgq_unregister_consumer(queue_name,
|
19
|
-
connection.select_value(sanitize_sql_array ["SELECT pgq.unregister_consumer(?, ?)", queue_name,
|
18
|
+
def pgq_unregister_consumer(queue_name, consumer_name)
|
19
|
+
connection.select_value(sanitize_sql_array ["SELECT pgq.unregister_consumer(?, ?)", queue_name, consumer_name]).to_i
|
20
20
|
end
|
21
21
|
|
22
22
|
# == insert events
|
@@ -29,8 +29,8 @@ module Pgq::Api
|
|
29
29
|
|
30
30
|
# == consuming
|
31
31
|
|
32
|
-
def pgq_next_batch(queue_name,
|
33
|
-
result = connection.select_value(sanitize_sql_array ["SELECT pgq.next_batch(?, ?)", queue_name,
|
32
|
+
def pgq_next_batch(queue_name, consumer_name)
|
33
|
+
result = connection.select_value(sanitize_sql_array ["SELECT pgq.next_batch(?, ?)", queue_name, consumer_name])
|
34
34
|
result ? result.to_i : nil
|
35
35
|
end
|
36
36
|
|
@@ -54,22 +54,22 @@ module Pgq::Api
|
|
54
54
|
|
55
55
|
# == failed events
|
56
56
|
|
57
|
-
def pgq_failed_event_retry(queue_name,
|
58
|
-
connection.select_value(sanitize_sql_array ["SELECT * FROM pgq.failed_event_retry(?, ?, ?)", queue_name,
|
57
|
+
def pgq_failed_event_retry(queue_name, consumer_name, event_id)
|
58
|
+
connection.select_value(sanitize_sql_array ["SELECT * FROM pgq.failed_event_retry(?, ?, ?)", queue_name, consumer_name, event_id])
|
59
59
|
end
|
60
60
|
|
61
|
-
def pgq_failed_event_delete(queue_name,
|
62
|
-
connection.select_value(sanitize_sql_array ["SELECT * FROM pgq.failed_event_delete(?, ?, ?)", queue_name,
|
61
|
+
def pgq_failed_event_delete(queue_name, consumer_name, event_id)
|
62
|
+
connection.select_value(sanitize_sql_array ["SELECT * FROM pgq.failed_event_delete(?, ?, ?)", queue_name, consumer_name, event_id])
|
63
63
|
end
|
64
64
|
|
65
|
-
def pgq_failed_event_count(queue_name,
|
66
|
-
res = connection.select_value(sanitize_sql_array ["SELECT * FROM pgq.failed_event_count(?, ?)", queue_name,
|
65
|
+
def pgq_failed_event_count(queue_name, consumer_name)
|
66
|
+
res = connection.select_value(sanitize_sql_array ["SELECT * FROM pgq.failed_event_count(?, ?)", queue_name, consumer_name])
|
67
67
|
res ? res.to_i : nil
|
68
68
|
end
|
69
69
|
|
70
|
-
def pgq_failed_event_list queue_name,
|
70
|
+
def pgq_failed_event_list queue_name, consumer_name, limit = nil, offset = nil, order = 'desc'
|
71
71
|
order = (order.to_s == 'desc') ? order : 'asc'
|
72
|
-
connection.select_all(sanitize_sql_array ["SELECT * FROM pgq.failed_event_list(?, ?, ?, ?)
|
72
|
+
connection.select_all(sanitize_sql_array ["SELECT * FROM pgq.failed_event_list(?, ?, ?, ?) ORDER BY ev_id #{order.upcase}", queue_name, consumer_name, limit.to_i, offset.to_i])
|
73
73
|
end
|
74
74
|
|
75
75
|
# == info methods
|
@@ -92,4 +92,41 @@ module Pgq::Api
|
|
92
92
|
connection.select_one(sanitize_sql_array ["SELECT *, EXTRACT(epoch FROM last_seen) AS last_seen_sec, EXTRACT(epoch FROM lag) AS lag_sec FROM pgq.get_consumer_info(?)", queue_name]) || {}
|
93
93
|
end
|
94
94
|
|
95
|
+
# == utils
|
96
|
+
|
97
|
+
def pgq_last_event_id(queue_name)
|
98
|
+
ticks = pgq_get_consumer_queue_info(queue_name)
|
99
|
+
table = connection.select_value("SELECT queue_data_pfx AS table FROM pgq.queue WHERE queue_name = #{sanitize(queue_name)}")
|
100
|
+
|
101
|
+
result = nil
|
102
|
+
|
103
|
+
if ticks['current_batch']
|
104
|
+
sql = connection.select_value("SELECT * FROM pgq.batch_event_sql(#{sanitize(ticks['current_batch'].to_i)})")
|
105
|
+
last_event = connection.select_value("SELECT MAX(ev_id) AS count FROM (#{sql}) AS x")
|
106
|
+
result = last_event.to_i
|
107
|
+
end
|
108
|
+
|
109
|
+
[table, result]
|
110
|
+
end
|
111
|
+
|
112
|
+
def pgq_mass_retry_failed_events(queue_name, consumer_name, limit = 5_000)
|
113
|
+
events = pgq_failed_event_list(queue_name, consumer_name, limit, nil, 'asc') || []
|
114
|
+
|
115
|
+
events.each do |event|
|
116
|
+
pgq_failed_event_retry(queue_name, consumer_name, event['ev_id'])
|
117
|
+
end
|
118
|
+
|
119
|
+
events.length
|
120
|
+
end
|
121
|
+
|
122
|
+
def pgq_mass_delete_failed_events(queue_name, consumer_name, limit = 5_000)
|
123
|
+
events = pgq_failed_event_list(queue_name, consumer_name, limit, nil, 'asc') || []
|
124
|
+
|
125
|
+
events.each do |event|
|
126
|
+
pgq_failed_event_delete(queue_name, consumer_name, event['ev_id'])
|
127
|
+
end
|
128
|
+
|
129
|
+
events.length
|
130
|
+
end
|
131
|
+
|
95
132
|
end
|
data/lib/pgq/consumer_group.rb
CHANGED
@@ -4,24 +4,24 @@
|
|
4
4
|
require 'pgq/consumer'
|
5
5
|
|
6
6
|
class Pgq::ConsumerGroup < Pgq::Consumer
|
7
|
+
|
8
|
+
# {'type' => [events]}
|
9
|
+
def perform_group(events_hash)
|
10
|
+
raise "realize me"
|
11
|
+
end
|
12
|
+
|
13
|
+
def perform_events(events)
|
14
|
+
events = sum_events(events)
|
15
|
+
# log_info "consume events (#{self.queue_name}): #{events.map{|k,v| [k, v.size]}.inspect}"
|
16
|
+
perform_group(events) if events.present?
|
17
|
+
end
|
7
18
|
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
# log_info "consume events (#{self.queue_name}): #{events.map{|k,v| [k, v.size]}.inspect}"
|
16
|
-
perform_group(events) if events.present?
|
17
|
-
end
|
19
|
+
def sum_events(events)
|
20
|
+
events.inject({}) do |result, event|
|
21
|
+
result[event.type] ||= []
|
22
|
+
result[event.type] << event
|
23
|
+
result
|
24
|
+
end
|
25
|
+
end
|
18
26
|
|
19
|
-
|
20
|
-
events.inject({}) do |result, event|
|
21
|
-
result[event.type] ||= []
|
22
|
-
result[event.type] << event
|
23
|
-
result
|
24
|
-
end
|
25
|
-
end
|
26
|
-
|
27
|
-
end
|
27
|
+
end
|
data/lib/pgq/utils.rb
CHANGED
@@ -19,31 +19,26 @@ module Pgq::Utils
|
|
19
19
|
# == inspect queue
|
20
20
|
# { type => events_count }
|
21
21
|
def inspect_queue(queue_name)
|
22
|
-
table, last_event =
|
22
|
+
table, last_event = database.pgq_last_event_id(queue_name)
|
23
23
|
|
24
|
-
if last_event
|
25
|
-
|
24
|
+
stats = if last_event
|
25
|
+
connection.select_all <<-SQL
|
26
26
|
SELECT count(*) as count, ev_type
|
27
27
|
FROM #{table}
|
28
28
|
WHERE ev_id > #{last_event.to_i}
|
29
29
|
GROUP BY ev_type
|
30
30
|
SQL
|
31
|
-
|
32
|
-
stats.each do |x|
|
33
|
-
result["#{x['ev_type']}"] = x['count'].to_i
|
34
|
-
end
|
35
|
-
|
36
31
|
else
|
37
|
-
|
32
|
+
connection.select_all <<-SQL
|
38
33
|
SELECT ev_type
|
39
34
|
FROM #{table}
|
40
35
|
GROUP BY ev_type
|
41
36
|
SQL
|
42
|
-
|
43
|
-
stats.each do |x|
|
44
|
-
result["#{x['ev_type']}"] = 0
|
45
|
-
end
|
46
37
|
end
|
38
|
+
|
39
|
+
stats.each do |x|
|
40
|
+
result["#{x['ev_type']}"] = x['count'].to_i
|
41
|
+
end
|
47
42
|
|
48
43
|
result
|
49
44
|
end
|
@@ -55,30 +50,25 @@ module Pgq::Utils
|
|
55
50
|
# show hash stats, for londiste type of storage events
|
56
51
|
# { type => events_count }
|
57
52
|
def inspect_londiste_queue(queue_name)
|
58
|
-
table, last_event =
|
53
|
+
table, last_event = database.pgq_last_event_id(queue_name)
|
59
54
|
|
60
|
-
if last_event
|
61
|
-
|
55
|
+
stats = if last_event
|
56
|
+
connection.select_all <<-SQL
|
62
57
|
SELECT count(*) as count, ev_type, ev_extra1
|
63
58
|
FROM #{table}
|
64
59
|
WHERE ev_id > #{last_event.to_i}
|
65
60
|
GROUP BY ev_type, ev_extra1
|
66
61
|
SQL
|
67
|
-
|
68
|
-
stats.each do |x|
|
69
|
-
result["#{x['ev_extra1']}:#{x['ev_type']}"] = x['count'].to_i
|
70
|
-
end
|
71
|
-
|
72
62
|
else
|
73
|
-
|
63
|
+
connection.select_all <<-SQL
|
74
64
|
SELECT ev_type, ev_extra1
|
75
65
|
FROM #{table}
|
76
66
|
GROUP BY ev_type, ev_extra1 ORDER BY ev_extra1, ev_type
|
77
67
|
SQL
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
68
|
+
end
|
69
|
+
|
70
|
+
stats.each do |x|
|
71
|
+
result["#{x['ev_extra1']}:#{x['ev_type']}"] = x['count'].to_i
|
82
72
|
end
|
83
73
|
|
84
74
|
result
|
@@ -95,38 +85,11 @@ module Pgq::Utils
|
|
95
85
|
|
96
86
|
# == resend failed events in queue
|
97
87
|
def retry_failed_events(queue_name, limit = 5_000)
|
98
|
-
|
99
|
-
|
100
|
-
events.each do |event|
|
101
|
-
database.pgq_failed_event_retry(queue_name, self.consumer_name, event['ev_id'])
|
102
|
-
end
|
103
|
-
|
104
|
-
events.length
|
88
|
+
database.pgq_mass_retry_failed_events(queue_name, self.consumer_name, limit)
|
105
89
|
end
|
106
90
|
|
107
91
|
def delete_failed_events(queue_name, limit = 5_000)
|
108
|
-
|
109
|
-
|
110
|
-
events.each do |event|
|
111
|
-
database.pgq_failed_event_delete(queue_name, self.consumer_name, event['ev_id'])
|
112
|
-
end
|
113
|
-
|
114
|
-
events.length
|
115
|
-
end
|
116
|
-
|
117
|
-
def last_event_id(queue_name)
|
118
|
-
ticks = database.pgq_get_consumer_queue_info(queue_name)
|
119
|
-
table = connection.select_value("SELECT queue_data_pfx AS table FROM pgq.queue WHERE queue_name = #{database.sanitize(queue_name)}")
|
120
|
-
|
121
|
-
result = nil
|
122
|
-
|
123
|
-
if ticks['current_batch']
|
124
|
-
sql = connection.select_value("SELECT * FROM pgq.batch_event_sql(#{database.sanitize(ticks['current_batch'].to_i)})")
|
125
|
-
last_event = connection.select_value("SELECT MAX(ev_id) AS count FROM (#{sql}) AS x")
|
126
|
-
result = last_event.to_i
|
127
|
-
end
|
128
|
-
|
129
|
-
[table, result]
|
92
|
+
database.pgq_mass_delete_failed_events(queue_name, self.consumer_name, limit)
|
130
93
|
end
|
131
94
|
|
132
95
|
end
|
data/lib/pgq/version.rb
CHANGED
data/pgq.gemspec
CHANGED
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: pgq
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 31
|
5
5
|
prerelease: false
|
6
6
|
segments:
|
7
7
|
- 0
|
8
8
|
- 1
|
9
|
-
-
|
10
|
-
version: 0.1.
|
9
|
+
- 2
|
10
|
+
version: 0.1.2
|
11
11
|
platform: ruby
|
12
12
|
authors:
|
13
13
|
- Makarchev Konstantin
|
@@ -15,7 +15,7 @@ autorequire: init
|
|
15
15
|
bindir: bin
|
16
16
|
cert_chain: []
|
17
17
|
|
18
|
-
date: 2012-05-
|
18
|
+
date: 2012-05-26 00:00:00 +04:00
|
19
19
|
default_executable:
|
20
20
|
dependencies:
|
21
21
|
- !ruby/object:Gem::Dependency
|
@@ -51,7 +51,7 @@ dependencies:
|
|
51
51
|
type: :runtime
|
52
52
|
version_requirements: *id002
|
53
53
|
- !ruby/object:Gem::Dependency
|
54
|
-
name:
|
54
|
+
name: pg
|
55
55
|
prerelease: false
|
56
56
|
requirement: &id003 !ruby/object:Gem::Requirement
|
57
57
|
none: false
|
@@ -62,10 +62,10 @@ dependencies:
|
|
62
62
|
segments:
|
63
63
|
- 0
|
64
64
|
version: "0"
|
65
|
-
type: :
|
65
|
+
type: :runtime
|
66
66
|
version_requirements: *id003
|
67
67
|
- !ruby/object:Gem::Dependency
|
68
|
-
name:
|
68
|
+
name: rspec
|
69
69
|
prerelease: false
|
70
70
|
requirement: &id004 !ruby/object:Gem::Requirement
|
71
71
|
none: false
|
@@ -78,6 +78,20 @@ dependencies:
|
|
78
78
|
version: "0"
|
79
79
|
type: :development
|
80
80
|
version_requirements: *id004
|
81
|
+
- !ruby/object:Gem::Dependency
|
82
|
+
name: rake
|
83
|
+
prerelease: false
|
84
|
+
requirement: &id005 !ruby/object:Gem::Requirement
|
85
|
+
none: false
|
86
|
+
requirements:
|
87
|
+
- - ">="
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
hash: 3
|
90
|
+
segments:
|
91
|
+
- 0
|
92
|
+
version: "0"
|
93
|
+
type: :development
|
94
|
+
version_requirements: *id005
|
81
95
|
description: Queues system for AR/Rails based on PgQ Skytools for PostgreSQL, like Resque on Redis. Rails 3! only tested.
|
82
96
|
email: kostya27@gmail.com
|
83
97
|
executables: []
|