pika_que 0.1.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (54) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +9 -0
  3. data/.rspec +2 -0
  4. data/.travis.yml +4 -0
  5. data/CODE_OF_CONDUCT.md +49 -0
  6. data/Gemfile +4 -0
  7. data/LICENSE.txt +21 -0
  8. data/README.md +41 -0
  9. data/Rakefile +6 -0
  10. data/bin/console +14 -0
  11. data/bin/setup +8 -0
  12. data/examples/demo.rb +42 -0
  13. data/examples/demo_delay.rb +41 -0
  14. data/examples/demo_oneoff.rb +29 -0
  15. data/examples/demo_priority.rb +52 -0
  16. data/examples/demo_reporter.rb +19 -0
  17. data/examples/demo_retry.rb +41 -0
  18. data/examples/demo_worker.rb +17 -0
  19. data/examples/dev_worker.rb +19 -0
  20. data/exe/pika_que +8 -0
  21. data/lib/active_job/queue_adapters/pika_que_adapter.rb +42 -0
  22. data/lib/pika_que.rb +44 -0
  23. data/lib/pika_que/broker.rb +88 -0
  24. data/lib/pika_que/cli.rb +180 -0
  25. data/lib/pika_que/codecs/json.rb +22 -0
  26. data/lib/pika_que/codecs/noop.rb +20 -0
  27. data/lib/pika_que/codecs/rails.rb +22 -0
  28. data/lib/pika_que/configuration.rb +110 -0
  29. data/lib/pika_que/connection.rb +47 -0
  30. data/lib/pika_que/delay_worker.rb +55 -0
  31. data/lib/pika_que/errors.rb +5 -0
  32. data/lib/pika_que/handlers/default_handler.rb +31 -0
  33. data/lib/pika_que/handlers/delay_handler.rb +124 -0
  34. data/lib/pika_que/handlers/error_handler.rb +69 -0
  35. data/lib/pika_que/handlers/retry_handler.rb +186 -0
  36. data/lib/pika_que/launcher.rb +92 -0
  37. data/lib/pika_que/logging.rb +33 -0
  38. data/lib/pika_que/metrics.rb +26 -0
  39. data/lib/pika_que/metrics/log_metric.rb +23 -0
  40. data/lib/pika_que/metrics/null_metric.rb +14 -0
  41. data/lib/pika_que/middleware/active_record.rb +13 -0
  42. data/lib/pika_que/middleware/chain.rb +90 -0
  43. data/lib/pika_que/processor.rb +45 -0
  44. data/lib/pika_que/publisher.rb +23 -0
  45. data/lib/pika_que/rails.rb +62 -0
  46. data/lib/pika_que/reporters.rb +18 -0
  47. data/lib/pika_que/reporters/log_reporter.rb +13 -0
  48. data/lib/pika_que/runner.rb +24 -0
  49. data/lib/pika_que/subscriber.rb +80 -0
  50. data/lib/pika_que/util.rb +17 -0
  51. data/lib/pika_que/version.rb +3 -0
  52. data/lib/pika_que/worker.rb +99 -0
  53. data/pika_que.gemspec +37 -0
  54. metadata +181 -0
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: b062334f6fd120c3cbdd8f9dbab1d0a50119c6e1
4
+ data.tar.gz: f9b3e62818fe79276aec1cb54435de8c51767eb1
5
+ SHA512:
6
+ metadata.gz: 8aaab568dd9e153e2942534ad74341e5b6da4e87ed389dfb2c2bfa85cb1c5629d94382d8dcec02b4b888bb61a728b4fa4c8ecc1a293144ae4274ba686db7f896
7
+ data.tar.gz: af5f17435c0992adbe6bb142cd3db5f20f29aef8a8bd54a7646084a9e1a1b9c8c4451c462d6003cfa7034ba26a0c1fe32978562e3f8873974e426b6fdef3411d
@@ -0,0 +1,9 @@
1
+ /.bundle/
2
+ /.yardoc
3
+ /Gemfile.lock
4
+ /_yardoc/
5
+ /coverage/
6
+ /doc/
7
+ /pkg/
8
+ /spec/reports/
9
+ /tmp/
data/.rspec ADDED
@@ -0,0 +1,2 @@
1
+ --format documentation
2
+ --color
@@ -0,0 +1,4 @@
1
+ language: ruby
2
+ rvm:
3
+ - 2.2.6
4
+ before_install: gem install bundler -v 1.11.2
@@ -0,0 +1,49 @@
1
+ # Contributor Code of Conduct
2
+
3
+ As contributors and maintainers of this project, and in the interest of
4
+ fostering an open and welcoming community, we pledge to respect all people who
5
+ contribute through reporting issues, posting feature requests, updating
6
+ documentation, submitting pull requests or patches, and other activities.
7
+
8
+ We are committed to making participation in this project a harassment-free
9
+ experience for everyone, regardless of level of experience, gender, gender
10
+ identity and expression, sexual orientation, disability, personal appearance,
11
+ body size, race, ethnicity, age, religion, or nationality.
12
+
13
+ Examples of unacceptable behavior by participants include:
14
+
15
+ * The use of sexualized language or imagery
16
+ * Personal attacks
17
+ * Trolling or insulting/derogatory comments
18
+ * Public or private harassment
19
+ * Publishing other's private information, such as physical or electronic
20
+ addresses, without explicit permission
21
+ * Other unethical or unprofessional conduct
22
+
23
+ Project maintainers have the right and responsibility to remove, edit, or
24
+ reject comments, commits, code, wiki edits, issues, and other contributions
25
+ that are not aligned to this Code of Conduct, or to ban temporarily or
26
+ permanently any contributor for other behaviors that they deem inappropriate,
27
+ threatening, offensive, or harmful.
28
+
29
+ By adopting this Code of Conduct, project maintainers commit themselves to
30
+ fairly and consistently applying these principles to every aspect of managing
31
+ this project. Project maintainers who do not follow or enforce the Code of
32
+ Conduct may be permanently removed from the project team.
33
+
34
+ This code of conduct applies both within project spaces and in public spaces
35
+ when an individual is representing the project or its community.
36
+
37
+ Instances of abusive, harassing, or otherwise unacceptable behavior may be
38
+ reported by contacting a project maintainer at dwkoogt@gmail.com. All
39
+ complaints will be reviewed and investigated and will result in a response that
40
+ is deemed necessary and appropriate to the circumstances. Maintainers are
41
+ obligated to maintain confidentiality with regard to the reporter of an
42
+ incident.
43
+
44
+ This Code of Conduct is adapted from the [Contributor Covenant][homepage],
45
+ version 1.3.0, available at
46
+ [http://contributor-covenant.org/version/1/3/0/][version]
47
+
48
+ [homepage]: http://contributor-covenant.org
49
+ [version]: http://contributor-covenant.org/version/1/3/0/
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in pika_que.gemspec
4
+ gemspec
@@ -0,0 +1,21 @@
1
+ The MIT License (MIT)
2
+
3
+ Copyright (c) 2017 Dong Wook Koo
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in
13
+ all copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ THE SOFTWARE.
@@ -0,0 +1,41 @@
1
+ # PikaQue
2
+
3
+ Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/pika_que`. To experiment with that code, run `bin/console` for an interactive prompt.
4
+
5
+ TODO: Delete this and the text above, and describe your gem
6
+
7
+ ## Installation
8
+
9
+ Add this line to your application's Gemfile:
10
+
11
+ ```ruby
12
+ gem 'pika_que'
13
+ ```
14
+
15
+ And then execute:
16
+
17
+ $ bundle
18
+
19
+ Or install it yourself as:
20
+
21
+ $ gem install pika_que
22
+
23
+ ## Usage
24
+
25
+ TODO: Write usage instructions here
26
+
27
+ ## Development
28
+
29
+ After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment. Run `bundle exec pika_que` to use the gem in this directory, ignoring other installed copies of this gem.
30
+
31
+ To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
32
+
33
+ ## Contributing
34
+
35
+ Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/pika_que. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
36
+
37
+
38
+ ## License
39
+
40
+ The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
41
+
@@ -0,0 +1,6 @@
1
+ require "bundler/gem_tasks"
2
+ require "rspec/core/rake_task"
3
+
4
+ RSpec::Core::RakeTask.new(:spec)
5
+
6
+ task :default => :spec
@@ -0,0 +1,14 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require "bundler/setup"
4
+ require "pika_que"
5
+
6
+ # You can add fixtures and/or initialization code here to make experimenting
7
+ # with your gem easier. You can also use a different console, if you like.
8
+
9
+ # (If you use this, don't forget to add pry to your Gemfile!)
10
+ # require "pry"
11
+ # Pry.start
12
+
13
+ require "irb"
14
+ IRB.start
@@ -0,0 +1,8 @@
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+ IFS=$'\n\t'
4
+ set -vx
5
+
6
+ bundle install
7
+
8
+ # Do any other automated setup that you need to do here
@@ -0,0 +1,42 @@
1
+ # > bundle exec ruby examples/demo.rb
2
+ $: << File.expand_path('../examples', File.dirname(__FILE__))
3
+ require 'pika_que'
4
+ require 'pika_que/processor'
5
+ require 'pika_que/publisher'
6
+
7
+ require 'dev_worker'
8
+ require 'demo_worker'
9
+ require 'demo_reporter'
10
+
11
+ PikaQue.logger.level = ::Logger::DEBUG
12
+
13
+ PikaQue.middleware do |chain|
14
+ chain.add DemoReporter
15
+ end
16
+
17
+ workers = [DemoWorker,DevWorker]
18
+
19
+ begin
20
+ pro = PikaQue::Processor.new(workers: workers, concurrency: 2)
21
+ pro.start
22
+ rescue => e
23
+ puts e
24
+ puts e.backtrace.join("\n")
25
+ end
26
+
27
+ sleep 3
28
+
29
+ pub = PikaQue::Publisher.new()
30
+ 300.times do |i|
31
+ pub.publish({ msg: "hello world #{i}" }, routing_key: 'pika-que-dev')
32
+ pub.publish({ msg: "hola mundo #{i}" }, routing_key: 'pika-que-demo')
33
+ # ph.publish({ msg: "hello world #{i} wait" }, routing_key: 'pika_que-dev', expiration: 10000)
34
+ end
35
+
36
+ sleep 3
37
+
38
+ pro.stop
39
+
40
+ puts "bye"
41
+
42
+ exit 1
@@ -0,0 +1,41 @@
1
+ # > bundle exec ruby examples/demo_delay.rb
2
+ require 'pika_que'
3
+ require 'pika_que/worker'
4
+ require 'pika_que/runner'
5
+
6
+ class DemoWorker
7
+ include PikaQue::Worker
8
+ from_queue "pika-que-demo"
9
+
10
+ def perform(msg)
11
+ logger.info msg["msg"]
12
+ ack!
13
+ end
14
+
15
+ end
16
+
17
+ PikaQue.logger.level = ::Logger::DEBUG
18
+
19
+ PikaQue.config.add_processor(PikaQue.config.delete(:delay_options))
20
+ PikaQue.config.add_processor(workers: [DemoWorker])
21
+
22
+ runner = PikaQue::Runner.new
23
+
24
+ begin
25
+ runner.run
26
+ rescue => e
27
+ puts e
28
+ puts e.backtrace.join("\n")
29
+ end
30
+
31
+ sleep 3
32
+
33
+ DemoWorker.enqueue_at({ msg: "delay message !!!" }, (Time.now + 180).to_i)
34
+
35
+ sleep 200
36
+
37
+ runner.stop
38
+
39
+ puts "bye"
40
+
41
+ exit 1
@@ -0,0 +1,29 @@
1
+ # > bundle exec ruby examples/demo_oneoff.rb
2
+ $: << File.expand_path('../examples', File.dirname(__FILE__))
3
+ require 'pika_que'
4
+
5
+ require 'dev_worker'
6
+
7
+ PikaQue.logger.level = ::Logger::DEBUG
8
+
9
+ begin
10
+ pro = DevWorker.new(concurrency: 2)
11
+ pro.start
12
+ rescue => e
13
+ puts e
14
+ puts e.backtrace.join("\n")
15
+ end
16
+
17
+ sleep 3
18
+
19
+ 300.times do |i|
20
+ DevWorker.enqueue({ msg: "hello world #{i}" })
21
+ end
22
+
23
+ sleep 3
24
+
25
+ pro.stop
26
+
27
+ puts "bye"
28
+
29
+ exit 1
@@ -0,0 +1,52 @@
1
+ # > bundle exec ruby examples/demo_priority.rb
2
+ require 'pika_que'
3
+ require 'pika_que/worker'
4
+
5
+ PikaQue.logger.level = ::Logger::DEBUG
6
+
7
+ class HighPriorityWorker
8
+ include PikaQue::Worker
9
+ from_queue "pika-que-priority", :arguments => { :'x-max-priority' => 10 }, :priority => 10
10
+
11
+ def perform(msg)
12
+ logger.info msg["msg"]
13
+ ack!
14
+ end
15
+
16
+ end
17
+
18
+ class LowPriorityWorker
19
+ include PikaQue::Worker
20
+ from_queue "pika-que-priority", :arguments => { :'x-max-priority' => 10 }, :priority => 1
21
+
22
+ def perform(msg)
23
+ logger.info msg["msg"]
24
+ ack!
25
+ end
26
+
27
+ end
28
+
29
+ workers = [HighPriorityWorker,LowPriorityWorker]
30
+
31
+ begin
32
+ pro = PikaQue::Processor.new(workers: workers, concurrency: 10)
33
+ pro.start
34
+ rescue => e
35
+ puts e
36
+ puts e.backtrace.join("\n")
37
+ end
38
+
39
+ sleep 3
40
+
41
+ 300.times do |i|
42
+ LowPriorityWorker.enqueue({ msg: "low priority #{i}" })
43
+ HighPriorityWorker.enqueue({ msg: "high priority #{i}" })
44
+ end
45
+
46
+ sleep 3
47
+
48
+ pro.stop
49
+
50
+ puts "bye"
51
+
52
+ exit 1
@@ -0,0 +1,19 @@
1
+ class DemoReporter
2
+
3
+ def initialize(opts = {})
4
+ STDOUT.sync = true
5
+ end
6
+
7
+ def call(worker, delivery_info, metadata, msg)
8
+ puts "entering middleware DemoReporter for msg: #{msg}"
9
+ begin
10
+ yield
11
+ rescue => e
12
+ puts "error caught in middleware DemoReporter for msg: #{msg}, error: #{e.message}"
13
+ raise e
14
+ ensure
15
+ puts "leaving middleware DemoReporter for msg: #{msg}"
16
+ end
17
+ end
18
+
19
+ end
@@ -0,0 +1,41 @@
1
+ # > bundle exec ruby examples/demo_retry.rb
2
+ require 'pika_que'
3
+ require 'pika_que/worker'
4
+ require 'pika_que/handlers/retry_handler'
5
+
6
+ class DemoWorker
7
+ include PikaQue::Worker
8
+ from_queue "pika-que-demo"
9
+ handle_with PikaQue::Handlers::RetryHandler, retry_mode: :const, retry_max_times: 3
10
+
11
+ def perform(msg)
12
+ logger.info msg["msg"]
13
+ raise "BOOM!"
14
+ ack!
15
+ end
16
+
17
+ end
18
+
19
+ PikaQue.logger.level = ::Logger::DEBUG
20
+
21
+ workers = [DemoWorker]
22
+
23
+ begin
24
+ pro = PikaQue::Processor.new(workers: workers)
25
+ pro.start
26
+ rescue => e
27
+ puts e
28
+ puts e.backtrace.join("\n")
29
+ end
30
+
31
+ sleep 3
32
+
33
+ DemoWorker.enqueue({ msg: "retry message" })
34
+
35
+ sleep 200
36
+
37
+ pro.stop
38
+
39
+ puts "bye"
40
+
41
+ exit 1
@@ -0,0 +1,17 @@
1
+ require 'pika_que/worker'
2
+ require 'pika_que/handlers/error_handler'
3
+
4
+ class DemoWorker
5
+ include PikaQue::Worker
6
+ from_queue "pika-que-demo"
7
+ handle_with PikaQue::Handlers::ErrorHandler
8
+
9
+ def perform(msg)
10
+ logger.info msg["msg"]
11
+ if rand(2) == 1
12
+ raise "BOOM!"
13
+ end
14
+ ack!
15
+ end
16
+
17
+ end
@@ -0,0 +1,19 @@
1
+ require 'pika_que/worker'
2
+ require 'pika_que/handlers/default_handler'
3
+
4
+ # explicitly setting default handler (not necessary)
5
+
6
+ class DevWorker
7
+ include PikaQue::Worker
8
+ from_queue "pika-que-dev"
9
+ handle_with PikaQue::Handlers::DefaultHandler
10
+
11
+ def perform(msg)
12
+ logger.info msg["msg"]
13
+ if rand(2) == 1
14
+ raise "BOOM!"
15
+ end
16
+ ack!
17
+ end
18
+
19
+ end