pika_que 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +9 -0
- data/.rspec +2 -0
- data/.travis.yml +4 -0
- data/CODE_OF_CONDUCT.md +49 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +21 -0
- data/README.md +41 -0
- data/Rakefile +6 -0
- data/bin/console +14 -0
- data/bin/setup +8 -0
- data/examples/demo.rb +42 -0
- data/examples/demo_delay.rb +41 -0
- data/examples/demo_oneoff.rb +29 -0
- data/examples/demo_priority.rb +52 -0
- data/examples/demo_reporter.rb +19 -0
- data/examples/demo_retry.rb +41 -0
- data/examples/demo_worker.rb +17 -0
- data/examples/dev_worker.rb +19 -0
- data/exe/pika_que +8 -0
- data/lib/active_job/queue_adapters/pika_que_adapter.rb +42 -0
- data/lib/pika_que.rb +44 -0
- data/lib/pika_que/broker.rb +88 -0
- data/lib/pika_que/cli.rb +180 -0
- data/lib/pika_que/codecs/json.rb +22 -0
- data/lib/pika_que/codecs/noop.rb +20 -0
- data/lib/pika_que/codecs/rails.rb +22 -0
- data/lib/pika_que/configuration.rb +110 -0
- data/lib/pika_que/connection.rb +47 -0
- data/lib/pika_que/delay_worker.rb +55 -0
- data/lib/pika_que/errors.rb +5 -0
- data/lib/pika_que/handlers/default_handler.rb +31 -0
- data/lib/pika_que/handlers/delay_handler.rb +124 -0
- data/lib/pika_que/handlers/error_handler.rb +69 -0
- data/lib/pika_que/handlers/retry_handler.rb +186 -0
- data/lib/pika_que/launcher.rb +92 -0
- data/lib/pika_que/logging.rb +33 -0
- data/lib/pika_que/metrics.rb +26 -0
- data/lib/pika_que/metrics/log_metric.rb +23 -0
- data/lib/pika_que/metrics/null_metric.rb +14 -0
- data/lib/pika_que/middleware/active_record.rb +13 -0
- data/lib/pika_que/middleware/chain.rb +90 -0
- data/lib/pika_que/processor.rb +45 -0
- data/lib/pika_que/publisher.rb +23 -0
- data/lib/pika_que/rails.rb +62 -0
- data/lib/pika_que/reporters.rb +18 -0
- data/lib/pika_que/reporters/log_reporter.rb +13 -0
- data/lib/pika_que/runner.rb +24 -0
- data/lib/pika_que/subscriber.rb +80 -0
- data/lib/pika_que/util.rb +17 -0
- data/lib/pika_que/version.rb +3 -0
- data/lib/pika_que/worker.rb +99 -0
- data/pika_que.gemspec +37 -0
- metadata +181 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: b062334f6fd120c3cbdd8f9dbab1d0a50119c6e1
|
4
|
+
data.tar.gz: f9b3e62818fe79276aec1cb54435de8c51767eb1
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 8aaab568dd9e153e2942534ad74341e5b6da4e87ed389dfb2c2bfa85cb1c5629d94382d8dcec02b4b888bb61a728b4fa4c8ecc1a293144ae4274ba686db7f896
|
7
|
+
data.tar.gz: af5f17435c0992adbe6bb142cd3db5f20f29aef8a8bd54a7646084a9e1a1b9c8c4451c462d6003cfa7034ba26a0c1fe32978562e3f8873974e426b6fdef3411d
|
data/.gitignore
ADDED
data/.rspec
ADDED
data/.travis.yml
ADDED
data/CODE_OF_CONDUCT.md
ADDED
@@ -0,0 +1,49 @@
|
|
1
|
+
# Contributor Code of Conduct
|
2
|
+
|
3
|
+
As contributors and maintainers of this project, and in the interest of
|
4
|
+
fostering an open and welcoming community, we pledge to respect all people who
|
5
|
+
contribute through reporting issues, posting feature requests, updating
|
6
|
+
documentation, submitting pull requests or patches, and other activities.
|
7
|
+
|
8
|
+
We are committed to making participation in this project a harassment-free
|
9
|
+
experience for everyone, regardless of level of experience, gender, gender
|
10
|
+
identity and expression, sexual orientation, disability, personal appearance,
|
11
|
+
body size, race, ethnicity, age, religion, or nationality.
|
12
|
+
|
13
|
+
Examples of unacceptable behavior by participants include:
|
14
|
+
|
15
|
+
* The use of sexualized language or imagery
|
16
|
+
* Personal attacks
|
17
|
+
* Trolling or insulting/derogatory comments
|
18
|
+
* Public or private harassment
|
19
|
+
* Publishing other's private information, such as physical or electronic
|
20
|
+
addresses, without explicit permission
|
21
|
+
* Other unethical or unprofessional conduct
|
22
|
+
|
23
|
+
Project maintainers have the right and responsibility to remove, edit, or
|
24
|
+
reject comments, commits, code, wiki edits, issues, and other contributions
|
25
|
+
that are not aligned to this Code of Conduct, or to ban temporarily or
|
26
|
+
permanently any contributor for other behaviors that they deem inappropriate,
|
27
|
+
threatening, offensive, or harmful.
|
28
|
+
|
29
|
+
By adopting this Code of Conduct, project maintainers commit themselves to
|
30
|
+
fairly and consistently applying these principles to every aspect of managing
|
31
|
+
this project. Project maintainers who do not follow or enforce the Code of
|
32
|
+
Conduct may be permanently removed from the project team.
|
33
|
+
|
34
|
+
This code of conduct applies both within project spaces and in public spaces
|
35
|
+
when an individual is representing the project or its community.
|
36
|
+
|
37
|
+
Instances of abusive, harassing, or otherwise unacceptable behavior may be
|
38
|
+
reported by contacting a project maintainer at dwkoogt@gmail.com. All
|
39
|
+
complaints will be reviewed and investigated and will result in a response that
|
40
|
+
is deemed necessary and appropriate to the circumstances. Maintainers are
|
41
|
+
obligated to maintain confidentiality with regard to the reporter of an
|
42
|
+
incident.
|
43
|
+
|
44
|
+
This Code of Conduct is adapted from the [Contributor Covenant][homepage],
|
45
|
+
version 1.3.0, available at
|
46
|
+
[http://contributor-covenant.org/version/1/3/0/][version]
|
47
|
+
|
48
|
+
[homepage]: http://contributor-covenant.org
|
49
|
+
[version]: http://contributor-covenant.org/version/1/3/0/
|
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
The MIT License (MIT)
|
2
|
+
|
3
|
+
Copyright (c) 2017 Dong Wook Koo
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
13
|
+
all copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,41 @@
|
|
1
|
+
# PikaQue
|
2
|
+
|
3
|
+
Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/pika_que`. To experiment with that code, run `bin/console` for an interactive prompt.
|
4
|
+
|
5
|
+
TODO: Delete this and the text above, and describe your gem
|
6
|
+
|
7
|
+
## Installation
|
8
|
+
|
9
|
+
Add this line to your application's Gemfile:
|
10
|
+
|
11
|
+
```ruby
|
12
|
+
gem 'pika_que'
|
13
|
+
```
|
14
|
+
|
15
|
+
And then execute:
|
16
|
+
|
17
|
+
$ bundle
|
18
|
+
|
19
|
+
Or install it yourself as:
|
20
|
+
|
21
|
+
$ gem install pika_que
|
22
|
+
|
23
|
+
## Usage
|
24
|
+
|
25
|
+
TODO: Write usage instructions here
|
26
|
+
|
27
|
+
## Development
|
28
|
+
|
29
|
+
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment. Run `bundle exec pika_que` to use the gem in this directory, ignoring other installed copies of this gem.
|
30
|
+
|
31
|
+
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
32
|
+
|
33
|
+
## Contributing
|
34
|
+
|
35
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/pika_que. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
|
36
|
+
|
37
|
+
|
38
|
+
## License
|
39
|
+
|
40
|
+
The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
|
41
|
+
|
data/Rakefile
ADDED
data/bin/console
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require "bundler/setup"
|
4
|
+
require "pika_que"
|
5
|
+
|
6
|
+
# You can add fixtures and/or initialization code here to make experimenting
|
7
|
+
# with your gem easier. You can also use a different console, if you like.
|
8
|
+
|
9
|
+
# (If you use this, don't forget to add pry to your Gemfile!)
|
10
|
+
# require "pry"
|
11
|
+
# Pry.start
|
12
|
+
|
13
|
+
require "irb"
|
14
|
+
IRB.start
|
data/bin/setup
ADDED
data/examples/demo.rb
ADDED
@@ -0,0 +1,42 @@
|
|
1
|
+
# > bundle exec ruby examples/demo.rb
|
2
|
+
$: << File.expand_path('../examples', File.dirname(__FILE__))
|
3
|
+
require 'pika_que'
|
4
|
+
require 'pika_que/processor'
|
5
|
+
require 'pika_que/publisher'
|
6
|
+
|
7
|
+
require 'dev_worker'
|
8
|
+
require 'demo_worker'
|
9
|
+
require 'demo_reporter'
|
10
|
+
|
11
|
+
PikaQue.logger.level = ::Logger::DEBUG
|
12
|
+
|
13
|
+
PikaQue.middleware do |chain|
|
14
|
+
chain.add DemoReporter
|
15
|
+
end
|
16
|
+
|
17
|
+
workers = [DemoWorker,DevWorker]
|
18
|
+
|
19
|
+
begin
|
20
|
+
pro = PikaQue::Processor.new(workers: workers, concurrency: 2)
|
21
|
+
pro.start
|
22
|
+
rescue => e
|
23
|
+
puts e
|
24
|
+
puts e.backtrace.join("\n")
|
25
|
+
end
|
26
|
+
|
27
|
+
sleep 3
|
28
|
+
|
29
|
+
pub = PikaQue::Publisher.new()
|
30
|
+
300.times do |i|
|
31
|
+
pub.publish({ msg: "hello world #{i}" }, routing_key: 'pika-que-dev')
|
32
|
+
pub.publish({ msg: "hola mundo #{i}" }, routing_key: 'pika-que-demo')
|
33
|
+
# ph.publish({ msg: "hello world #{i} wait" }, routing_key: 'pika_que-dev', expiration: 10000)
|
34
|
+
end
|
35
|
+
|
36
|
+
sleep 3
|
37
|
+
|
38
|
+
pro.stop
|
39
|
+
|
40
|
+
puts "bye"
|
41
|
+
|
42
|
+
exit 1
|
@@ -0,0 +1,41 @@
|
|
1
|
+
# > bundle exec ruby examples/demo_delay.rb
|
2
|
+
require 'pika_que'
|
3
|
+
require 'pika_que/worker'
|
4
|
+
require 'pika_que/runner'
|
5
|
+
|
6
|
+
class DemoWorker
|
7
|
+
include PikaQue::Worker
|
8
|
+
from_queue "pika-que-demo"
|
9
|
+
|
10
|
+
def perform(msg)
|
11
|
+
logger.info msg["msg"]
|
12
|
+
ack!
|
13
|
+
end
|
14
|
+
|
15
|
+
end
|
16
|
+
|
17
|
+
PikaQue.logger.level = ::Logger::DEBUG
|
18
|
+
|
19
|
+
PikaQue.config.add_processor(PikaQue.config.delete(:delay_options))
|
20
|
+
PikaQue.config.add_processor(workers: [DemoWorker])
|
21
|
+
|
22
|
+
runner = PikaQue::Runner.new
|
23
|
+
|
24
|
+
begin
|
25
|
+
runner.run
|
26
|
+
rescue => e
|
27
|
+
puts e
|
28
|
+
puts e.backtrace.join("\n")
|
29
|
+
end
|
30
|
+
|
31
|
+
sleep 3
|
32
|
+
|
33
|
+
DemoWorker.enqueue_at({ msg: "delay message !!!" }, (Time.now + 180).to_i)
|
34
|
+
|
35
|
+
sleep 200
|
36
|
+
|
37
|
+
runner.stop
|
38
|
+
|
39
|
+
puts "bye"
|
40
|
+
|
41
|
+
exit 1
|
@@ -0,0 +1,29 @@
|
|
1
|
+
# > bundle exec ruby examples/demo_oneoff.rb
|
2
|
+
$: << File.expand_path('../examples', File.dirname(__FILE__))
|
3
|
+
require 'pika_que'
|
4
|
+
|
5
|
+
require 'dev_worker'
|
6
|
+
|
7
|
+
PikaQue.logger.level = ::Logger::DEBUG
|
8
|
+
|
9
|
+
begin
|
10
|
+
pro = DevWorker.new(concurrency: 2)
|
11
|
+
pro.start
|
12
|
+
rescue => e
|
13
|
+
puts e
|
14
|
+
puts e.backtrace.join("\n")
|
15
|
+
end
|
16
|
+
|
17
|
+
sleep 3
|
18
|
+
|
19
|
+
300.times do |i|
|
20
|
+
DevWorker.enqueue({ msg: "hello world #{i}" })
|
21
|
+
end
|
22
|
+
|
23
|
+
sleep 3
|
24
|
+
|
25
|
+
pro.stop
|
26
|
+
|
27
|
+
puts "bye"
|
28
|
+
|
29
|
+
exit 1
|
@@ -0,0 +1,52 @@
|
|
1
|
+
# > bundle exec ruby examples/demo_priority.rb
|
2
|
+
require 'pika_que'
|
3
|
+
require 'pika_que/worker'
|
4
|
+
|
5
|
+
PikaQue.logger.level = ::Logger::DEBUG
|
6
|
+
|
7
|
+
class HighPriorityWorker
|
8
|
+
include PikaQue::Worker
|
9
|
+
from_queue "pika-que-priority", :arguments => { :'x-max-priority' => 10 }, :priority => 10
|
10
|
+
|
11
|
+
def perform(msg)
|
12
|
+
logger.info msg["msg"]
|
13
|
+
ack!
|
14
|
+
end
|
15
|
+
|
16
|
+
end
|
17
|
+
|
18
|
+
class LowPriorityWorker
|
19
|
+
include PikaQue::Worker
|
20
|
+
from_queue "pika-que-priority", :arguments => { :'x-max-priority' => 10 }, :priority => 1
|
21
|
+
|
22
|
+
def perform(msg)
|
23
|
+
logger.info msg["msg"]
|
24
|
+
ack!
|
25
|
+
end
|
26
|
+
|
27
|
+
end
|
28
|
+
|
29
|
+
workers = [HighPriorityWorker,LowPriorityWorker]
|
30
|
+
|
31
|
+
begin
|
32
|
+
pro = PikaQue::Processor.new(workers: workers, concurrency: 10)
|
33
|
+
pro.start
|
34
|
+
rescue => e
|
35
|
+
puts e
|
36
|
+
puts e.backtrace.join("\n")
|
37
|
+
end
|
38
|
+
|
39
|
+
sleep 3
|
40
|
+
|
41
|
+
300.times do |i|
|
42
|
+
LowPriorityWorker.enqueue({ msg: "low priority #{i}" })
|
43
|
+
HighPriorityWorker.enqueue({ msg: "high priority #{i}" })
|
44
|
+
end
|
45
|
+
|
46
|
+
sleep 3
|
47
|
+
|
48
|
+
pro.stop
|
49
|
+
|
50
|
+
puts "bye"
|
51
|
+
|
52
|
+
exit 1
|
@@ -0,0 +1,19 @@
|
|
1
|
+
class DemoReporter
|
2
|
+
|
3
|
+
def initialize(opts = {})
|
4
|
+
STDOUT.sync = true
|
5
|
+
end
|
6
|
+
|
7
|
+
def call(worker, delivery_info, metadata, msg)
|
8
|
+
puts "entering middleware DemoReporter for msg: #{msg}"
|
9
|
+
begin
|
10
|
+
yield
|
11
|
+
rescue => e
|
12
|
+
puts "error caught in middleware DemoReporter for msg: #{msg}, error: #{e.message}"
|
13
|
+
raise e
|
14
|
+
ensure
|
15
|
+
puts "leaving middleware DemoReporter for msg: #{msg}"
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
# > bundle exec ruby examples/demo_retry.rb
|
2
|
+
require 'pika_que'
|
3
|
+
require 'pika_que/worker'
|
4
|
+
require 'pika_que/handlers/retry_handler'
|
5
|
+
|
6
|
+
class DemoWorker
|
7
|
+
include PikaQue::Worker
|
8
|
+
from_queue "pika-que-demo"
|
9
|
+
handle_with PikaQue::Handlers::RetryHandler, retry_mode: :const, retry_max_times: 3
|
10
|
+
|
11
|
+
def perform(msg)
|
12
|
+
logger.info msg["msg"]
|
13
|
+
raise "BOOM!"
|
14
|
+
ack!
|
15
|
+
end
|
16
|
+
|
17
|
+
end
|
18
|
+
|
19
|
+
PikaQue.logger.level = ::Logger::DEBUG
|
20
|
+
|
21
|
+
workers = [DemoWorker]
|
22
|
+
|
23
|
+
begin
|
24
|
+
pro = PikaQue::Processor.new(workers: workers)
|
25
|
+
pro.start
|
26
|
+
rescue => e
|
27
|
+
puts e
|
28
|
+
puts e.backtrace.join("\n")
|
29
|
+
end
|
30
|
+
|
31
|
+
sleep 3
|
32
|
+
|
33
|
+
DemoWorker.enqueue({ msg: "retry message" })
|
34
|
+
|
35
|
+
sleep 200
|
36
|
+
|
37
|
+
pro.stop
|
38
|
+
|
39
|
+
puts "bye"
|
40
|
+
|
41
|
+
exit 1
|
@@ -0,0 +1,17 @@
|
|
1
|
+
require 'pika_que/worker'
|
2
|
+
require 'pika_que/handlers/error_handler'
|
3
|
+
|
4
|
+
class DemoWorker
|
5
|
+
include PikaQue::Worker
|
6
|
+
from_queue "pika-que-demo"
|
7
|
+
handle_with PikaQue::Handlers::ErrorHandler
|
8
|
+
|
9
|
+
def perform(msg)
|
10
|
+
logger.info msg["msg"]
|
11
|
+
if rand(2) == 1
|
12
|
+
raise "BOOM!"
|
13
|
+
end
|
14
|
+
ack!
|
15
|
+
end
|
16
|
+
|
17
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
require 'pika_que/worker'
|
2
|
+
require 'pika_que/handlers/default_handler'
|
3
|
+
|
4
|
+
# explicitly setting default handler (not necessary)
|
5
|
+
|
6
|
+
class DevWorker
|
7
|
+
include PikaQue::Worker
|
8
|
+
from_queue "pika-que-dev"
|
9
|
+
handle_with PikaQue::Handlers::DefaultHandler
|
10
|
+
|
11
|
+
def perform(msg)
|
12
|
+
logger.info msg["msg"]
|
13
|
+
if rand(2) == 1
|
14
|
+
raise "BOOM!"
|
15
|
+
end
|
16
|
+
ack!
|
17
|
+
end
|
18
|
+
|
19
|
+
end
|