sanger_warren 0.1.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '08237db3ed2d846148e3e1b127b737d660cb48f755489ced7fbda20d582fb3c2'
-  data.tar.gz: 41dc24f4262cc1b641e985058d7ef6b1cb294ef9e26c42775ac68bd21182c8e3
+  metadata.gz: 07d40b65d949f1c9fe0a1622faab1cfc6e64483ef498400daae60d7dd17d4897
+  data.tar.gz: badadcd8587f75dd72cc240df07947a39a222741043525ed204efb7738b13511
 SHA512:
-  metadata.gz: e60a34f8902db181b06978f0b554485c8f69148e9111b2a325ad6b37b13dc4abef93f5e8d010ad4e2d2d7dab4dcdfddfa51f6b5fe5866e037d9d437e85095af2
-  data.tar.gz: 98a52f7cdfaf2815f6b83a5166791bb83ab84aaa37e87ed9b4d68534fccb7a9efe14ec0c618467bd24f8c67deecd2124833245b0e3bc7b156ca48ff2a4762bb6
+  metadata.gz: 333c5cb6382e1ac5cdf475a66e541afcaa0ca5698105cf8e62d435c8bbf2171e59f165875597d507c75b2b8a4eff8504a96f7d6295cf24f05a5393ac55c01716
+  data.tar.gz: de3d17ae3bdeaf272ca52a3f5170187efb570709ca77f77a657221400d1fcf50706d8d48c6aa48118bfb3f67276925412baaf4fe7fb250ef64abe1da02134267

data/.github/workflows/ruby.yml

@@ -0,0 +1,39 @@
+name: Ruby
+
+on:
+  - push
+  - pull_request
+
+jobs:
+  test:
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        ruby: [ '2.6', '2.7', '3.0' ]
+    name: Ruby Test ${{ matrix.ruby }}
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+        bundler-cache: true # Runs bundle install and caches gems. See the ruby_test.yml
+                            # example if you need more control over bundler.
+    - name: Run tests
+      run: bundle exec rake
+  lint:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: 3.0
+        bundler-cache: true # Runs bundle install and caches gems. See the ruby_test.yml
+                            # example if you need more control over bundler.
+    - name: Run lint
+      run: bundle exec rubocop

data/.rubocop.yml

@@ -8,14 +8,20 @@
 # where the inspected file is and continue its way up to the root directory.
 #
 # See https://docs.rubocop.org/rubocop/configuration
+require:
+  - rubocop-rake
+  - rubocop-rspec
+
 AllCops:
   TargetRubyVersion: 2.6
+  NewCops: enable
 # Blocks are used to structure tests and are part of the shoulda dsl.
 # The standard BlockLength limits are too stringent for this purpose.
-# Block DSL is also used in routes
-# We should be careful keeping rake tasks here, as large tasks are still an issue.
-# Not sure if we need to split namespaces across several files though.
-# We exclude builders, as they are concerned with rendering xml
+# We also disable it for the gemspec, as while its using new, its more like
+# a DSL in nature
 Metrics/BlockLength:
-  ExcludedMethods:
+  Exclude:
+  - '*.gemspec'
+  IgnoredMethods:
    - describe
+   - context

data/.yardopts

@@ -0,0 +1,3 @@
+-
+LICENSE.txt
+CHANGELOG.md

data/CHANGELOG.md

@@ -5,6 +5,39 @@ Unreleased section to make new releases easy.
 
 ## [Unreleased]
 
-Initial release
+## [0.3.0] - 2021-06-04
+
+### Added
+
+- Added support for delay exchanges to process messages after a fixed delay
+- Increased documentation
+- Added Warren::Message::Simple for wrapping just routing key and payload.
+- Added optional worker_count to warren_consumers.yml to control number of worker threads
+
+### Removed
+
+- Warren::Handler::Test and Warren::Handler::Test::Channel no loner respond to
+  `add_exchange`. These methods were undocumented, and unused internally.
+
+## Changed
+
+- Messages must now implement `#headers`, although simply returning an empty
+  hash is sufficient.
+  See {Warren::Message::Simple#headers} for example
+- Subscriber templates now use the path 'app/warren/subscriber' rather than
+  'app/warren/subscribers' to correctly match class namespacing.
+- 3 consumer worker threads will be spun up by default
+
+## [0.2.0]
+
+### Added
+
+- Added railties to automatically initialize and configure Warren in rails apps.
+  You can remove the `config/initializers/warren.rb` and the `config.warren = config_for(:warren)`
+  line from `config/application.rb`
+- Added `warren config` CLI to help generate a warren.yml configuration file.
+- Added warren consumers managed through `warren consumer`
+
+## Initial release
 
 - Import of lib/warren from sequencescape

data/Gemfile

@@ -6,8 +6,13 @@ source 'https://rubygems.org'
 gemspec
 
 gem 'pry'
-gem 'rake', '~> 12.0'
+gem 'rake', '~> 13.0'
 gem 'rspec', '~> 3.0'
 gem 'rubocop'
+gem 'rubocop-rake'
+gem 'rubocop-rspec'
 gem 'simplecov'
 gem 'yard'
+
+# Used to provide instance_doubles for more robust testing
+gem 'activerecord', '~>5.2.0', require: false

data/Gemfile.lock

@@ -1,77 +1,109 @@
 PATH
   remote: .
   specs:
-    sanger_warren (0.1.0)
+    sanger_warren (0.3.0)
       bunny (~> 2.17.0)
       connection_pool (~> 2.2.0)
       multi_json (~> 1.0)
+      thor (~> 1.1.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
+    activemodel (5.2.6)
+      activesupport (= 5.2.6)
+    activerecord (5.2.6)
+      activemodel (= 5.2.6)
+      activesupport (= 5.2.6)
+      arel (>= 9.0)
+    activesupport (5.2.6)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 0.7, < 2)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
     amq-protocol (2.3.2)
-    ast (2.4.1)
+    arel (9.0.0)
+    ast (2.4.2)
     bunny (2.17.0)
       amq-protocol (~> 2.3, >= 2.3.1)
-    coderay (1.1.2)
-    connection_pool (2.2.3)
+    coderay (1.1.3)
+    concurrent-ruby (1.1.8)
+    connection_pool (2.2.5)
     diff-lcs (1.4.4)
-    docile (1.3.2)
-    method_source (0.9.2)
-    multi_json (1.13.1)
-    parallel (1.19.2)
-    parser (2.7.2.0)
+    docile (1.4.0)
+    i18n (1.8.10)
+      concurrent-ruby (~> 1.0)
+    method_source (1.0.0)
+    minitest (5.14.4)
+    multi_json (1.15.0)
+    parallel (1.20.1)
+    parser (3.0.1.1)
       ast (~> 2.4.1)
-    pry (0.12.2)
-      coderay (~> 1.1.0)
-      method_source (~> 0.9.0)
+    pry (0.14.1)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
     rainbow (3.0.0)
-    rake (12.3.3)
-    regexp_parser (1.8.2)
-    rexml (3.2.4)
-    rspec (3.9.0)
-      rspec-core (~> 3.9.0)
-      rspec-expectations (~> 3.9.0)
-      rspec-mocks (~> 3.9.0)
-    rspec-core (3.9.3)
-      rspec-support (~> 3.9.3)
-    rspec-expectations (3.9.2)
+    rake (13.0.3)
+    regexp_parser (2.1.1)
+    rexml (3.2.5)
+    rspec (3.10.0)
+      rspec-core (~> 3.10.0)
+      rspec-expectations (~> 3.10.0)
+      rspec-mocks (~> 3.10.0)
+    rspec-core (3.10.1)
+      rspec-support (~> 3.10.0)
+    rspec-expectations (3.10.1)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.9.0)
-    rspec-mocks (3.9.1)
+      rspec-support (~> 3.10.0)
+    rspec-mocks (3.10.2)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.9.0)
-    rspec-support (3.9.3)
-    rubocop (1.0.0)
+      rspec-support (~> 3.10.0)
+    rspec-support (3.10.2)
+    rubocop (1.15.0)
       parallel (~> 1.10)
-      parser (>= 2.7.1.5)
+      parser (>= 3.0.0.0)
       rainbow (>= 2.2.2, < 4.0)
-      regexp_parser (>= 1.8)
+      regexp_parser (>= 1.8, < 3.0)
       rexml
-      rubocop-ast (>= 0.6.0)
+      rubocop-ast (>= 1.5.0, < 2.0)
       ruby-progressbar (~> 1.7)
-      unicode-display_width (>= 1.4.0, < 2.0)
-    rubocop-ast (1.0.0)
-      parser (>= 2.7.1.5)
-    ruby-progressbar (1.10.1)
-    simplecov (0.19.0)
+      unicode-display_width (>= 1.4.0, < 3.0)
+    rubocop-ast (1.5.0)
+      parser (>= 3.0.1.1)
+    rubocop-rake (0.5.1)
+      rubocop
+    rubocop-rspec (2.3.0)
+      rubocop (~> 1.0)
+      rubocop-ast (>= 1.1.0)
+    ruby-progressbar (1.11.0)
+    simplecov (0.21.2)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
+      simplecov_json_formatter (~> 0.1)
     simplecov-html (0.12.3)
-    unicode-display_width (1.7.0)
-    yard (0.9.25)
+    simplecov_json_formatter (0.1.3)
+    thor (1.1.0)
+    thread_safe (0.3.6)
+    tzinfo (1.2.9)
+      thread_safe (~> 0.1)
+    unicode-display_width (2.0.0)
+    yard (0.9.26)
 
 PLATFORMS
   ruby
+  x86_64-darwin-19
 
 DEPENDENCIES
+  activerecord (~> 5.2.0)
   pry
-  rake (~> 12.0)
+  rake (~> 13.0)
   rspec (~> 3.0)
   rubocop
+  rubocop-rake
+  rubocop-rspec
   sanger_warren!
   simplecov
   yard
 
 BUNDLED WITH
-   2.1.4
+   2.2.8

data/README.md

@@ -14,68 +14,158 @@ gem 'sanger-warren'
 
 And then execute:
 
-    $ bundle install
+```bash
+bundle install
+```
 
 Or install it yourself as:
 
-    $ gem install sanger-warren
+```bash
+gem install sanger_warren
+```
 
 ## Usage
 
-If using with a Rails app, here's an example to get you started:
+### Basic setup
+
+If using with a Rails app, you can simply run `bundle exec warren config` to
+help generate a warren config file. Warren will automatically be initialize
+on Rails start-up.
+
+In rails 5 you will need to add the following to your `config/application.rb`
+to ensure the auto-loader can find the subscribers.
 
 ```ruby
-# config/initializers/warren.rb
-require 'warren'
+config.autoload_paths += %W{#{Rails.root}/app}
+```
+
+### Handler types
 
-Warren.setup(Rails.application.config.warren.deep_symbolize_keys.slice(:type, :config))
+In development mode, warren is usually configured to log to the console only. If
+you wish to enable broadcast mode, the easiest way is via an environmental
+variable, WARREN_TYPE.
+
+```bash
+WARREN_TYPE=broadcast bundle exec rails s
 ```
 
+### Broadcasting a message
+
+To broadcast a message, simply push the message onto the message handler
+
 ```ruby
-# config/application.rb
-# ...
-config.warren = config_for(:warren)
-# ...
+    Warren.handler << message
 ```
 
-```yaml
-# config/warren.yml
-development:
-  type: log
-  config: # Useful to allow easy switching to broadcast in development
-    routing_key_prefix: 'dev'
-    server:
-      host: localhost
-      port: 5672
-      username: guest
-      password: guest
-      vhost: /
-      frame_max: 0
-      heartbeat: 30
-    exchange: exchange_to_use
-test:
-  type: test
-production: # In practice keep this out of your source control
-  type: broadcast
-  config: # Useful to allow easy switching to broadcast
-    routing_key_prefix: 'production'
-    server:
-      host: localhost # Or a remote host
-      port: 5672
-      username: ...
-      password: ...
-      vhost: /
-      frame_max: 0
-      heartbeat: 30
-    exchange: exchange_to_use
+Message should be an object that responds to routing_key and payload. The handler
+will automatically prefix the routing key with the 'routing_key_prefix' configured
+in the warren.yml file. By default this is usually the name of your environment.
+
+If you are sending out multiple messages, you can check out a channel from the
+connection pool, and use that instead.
+
+```ruby
+    Warren.handler.with_channel do |channel|
+        channel << message_a
+        channel << message_b
+    end
+```
+
+### Setting up a consumer
+
+A command line interface exists to assist with setting up consumers. It be be
+invoked with:
+
+```bash
+bundle exec warren consumer add
 ```
 
+This will guide you through configuration, and template out a subscriber class.
+Subscribers receive the message payload, and metadata information and process
+them in their #process method.
+
+For more information about optional command line arguments you can supply to
+the cli use:
+
+```bash
+bundle exec warren consumer add --help
+```
+
+#### Opinionated defaults
+
+The cli makes some opinionated assumptions to simplify setup process:
+
+- Consumers will use a subscriber class named Warren::Subscribed::CamelCasedConsumerName
+- All consumers will dead-letter to the fanout exchange consumer-name.dead_letters
+- consumer-name.dead_letters will be bound to a queue of the same name
+- All queues and exchanges are durable
+
+These options can be over-ridden in the warren_consumers.yml file if necessary.
+If you wish to completely disable dead-letter queue configuration, such as when
+using policies, then you can set dead_letters to false.
+
+#### Delayed messaged
+
+Warren uses a delay exchange / queue approach for delaying the redelivery of messages.
+We don't currently support the
+[delayed message exchange plugin](https://github.com/rabbitmq/rabbitmq-delayed-message-exchange)
+
+The way this works:
+
+- Consumers with a delay option will automatically register an exchange and a queue
+- By default both of these will be named '<consumer_name>.delay' although you can
+  override this by updating `warren_consumers.yml`
+- The '<consumer_name>.delay' queue will have a time-to-live (ttl) set, after
+  which any messages on the queue will be dead-lettered
+- The dead-letter exchange for this queue is configured to an empty string `''`
+  which corresponds to the 'default exchange', and the routing key is set to
+  match the original queue name.
+- This exchange is a little bit special, and automatically routes messages to
+  any queue on the host where the queue name matches the message routing key.
+
+This approach avoids repeat delivery of the message to any other queues bound
+to the original exchange, and avoids the need for further exchanges or bindings.
+
+### Running consumers
+
+To run all configure consumers use:
+
+```bash
+bundle exec warren consumer start
+```
+
+You can also run only a subset of consumers:
+
+```bash
+bundle exec warren consumer start --consumers=consumer_name other_consumer
+```
+
+If you are testing in development, don't forget to set the WARREN_TYPE
+environmental variable if you want to pull messages off an actual queue.
+
+```bash
+WARREN_TYPE=broadcast bundle exec warren consumer start
+```
+
+## Testing
+
+Warren provides useful helpers to assist with testing. Helper documentation, and
+some testing examples can be found in the documentation for the
+{Warren::Handler::Test test helper}, or online at
+[rubydoc.info](https://rubydoc.info/gems/sanger_warren/Warren/Handler/Test)
+
 ## Development
 
-After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+After checking out the repo, run `bin/setup` to install dependencies. Then, run
+`rake spec` to run the tests. You can also run `bin/console` for an interactive
+prompt that will allow you to experiment.
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+To install this gem onto your local machine, run `bundle exec rake install`. To
+release a new version, update the version number in `version.rb`, and then run
+`bundle exec rake release`, which will create a git tag for the version, push
+git commits and tags, and push the `.gem` file to
+[rubygems.org](https://rubygems.org).
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/Warren.
+Bug reports and pull requests are welcome on [GitHub](https://github.com/sanger/warren)