cloudenvoy 0.1.0.dev → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dd2ec9d1e57064ce857c167a6ec7059790daa1f6c55dfab0131ba29bc133b2db
-  data.tar.gz: c15723602a03899edd1abe13bab834609af75ce5dc6eb8487640bfcd1aa56a2f
+  metadata.gz: 788d59dcdb276ba209ea7502f1d03765ab0beac646089ac5891ea67967a96a25
+  data.tar.gz: 819add5f2b7d7b3b37fc6f1f32d008af05fddd1409d585e0a88c7249f0427f57
 SHA512:
-  metadata.gz: 2f6a835095c6b26d0601a9e244a1b9f4c1262f5ef0f42c7ae9a3b5b8e3d04a9618b8fd8cfef06b419cabb1e11a85d64986b56dbac07b3b1f20f40623a26877d6
-  data.tar.gz: 6de2c1aa9543cd354e6a304c63f7ceebd97920d68c4f5241f7b478e7887067558f19d24360c1546f8b49e94238ce2eae4e61a227c4d13f94c9a1464ded34a880
+  metadata.gz: 3df6bb839c332feea426ec26ac09f46c951f8997159a53d32ce5ac3a11549648842f747ae1b2a0e962f399d8d12a11e6a64501d72134c0347485d39ec07d0ca8
+  data.tar.gz: cd320825e2b8c842e2746c5b7acbfbb82751f743c9bae38a4e997f411a3aa44b6d1e658bf8c8a7afba108ecc7fcc3a5cca4bd2c4b9711a9bfeadd1d88e512c31

data/.github/workflows/test.yml

@@ -0,0 +1,37 @@
+name: Test
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        ruby:
+          - '2.5.x'
+          - '2.6.x'
+        appraisal:
+          - 'rails-5.2'
+          - 'rails-6.0'
+    steps:
+      - name: Setup System
+        run: sudo apt-get install libsqlite3-dev
+      - uses: actions/checkout@v2
+      - uses: zhulik/redis-action@1.1.0
+      - name: Set up Ruby 2.6
+        uses: actions/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+      - name: Build and test with Rake
+        env:
+            APPRAISAL_CONTEXT: ${{ matrix.appraisal }}
+        run: |
+          gem install bundler
+          bundle install --jobs 4 --retry 3
+          bundle exec rubocop
+          bundle exec appraisal ${APPRAISAL_CONTEXT} bundle
+          bundle exec appraisal ${APPRAISAL_CONTEXT} rspec

data/.gitignore

@@ -3,9 +3,12 @@
 /_yardoc/
 /coverage/
 /doc/
+/examples/rails/log/*.log
+/examples/rails/tmp/
 /pkg/
 /spec/reports/
 /tmp/
+/log/*.log
 
 # rspec failure tracking
 .rspec_status

data/.rubocop.yml

@@ -32,6 +32,7 @@ RSpec/ScatteredSetup:
 Metrics/BlockLength:
   Exclude:
     - cloudenvoy.gemspec
+    - lib/tasks/**/*
     - 'spec/**/*'
 
 Style/Documentation:

data/Appraisals

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+appraise 'rails-5.2' do
+  gem 'rails', '5.2'
+end
+
+appraise 'rails-6.0' do
+  gem 'rails', '6.0'
+end

data/CHANGELOG.md

@@ -0,0 +1,4 @@
+# Changelog
+
+## [v0.1.0](https://github.com/keypup-io/cloudenvoy/tree/v0.1.0) (2020-09-16)
+Initial release

data/Gemfile.lock

@@ -1,19 +1,189 @@
 PATH
   remote: .
   specs:
-    cloudenvoy (0.1.0.dev)
+    cloudenvoy (0.1.0)
+      activesupport
+      google-cloud-pubsub (~> 2.0)
+      jwt
+      retriable
 
 GEM
   remote: https://rubygems.org/
   specs:
+    actioncable (6.0.3.2)
+      actionpack (= 6.0.3.2)
+      nio4r (~> 2.0)
+      websocket-driver (>= 0.6.1)
+    actionmailbox (6.0.3.2)
+      actionpack (= 6.0.3.2)
+      activejob (= 6.0.3.2)
+      activerecord (= 6.0.3.2)
+      activestorage (= 6.0.3.2)
+      activesupport (= 6.0.3.2)
+      mail (>= 2.7.1)
+    actionmailer (6.0.3.2)
+      actionpack (= 6.0.3.2)
+      actionview (= 6.0.3.2)
+      activejob (= 6.0.3.2)
+      mail (~> 2.5, >= 2.5.4)
+      rails-dom-testing (~> 2.0)
+    actionpack (6.0.3.2)
+      actionview (= 6.0.3.2)
+      activesupport (= 6.0.3.2)
+      rack (~> 2.0, >= 2.0.8)
+      rack-test (>= 0.6.3)
+      rails-dom-testing (~> 2.0)
+      rails-html-sanitizer (~> 1.0, >= 1.2.0)
+    actiontext (6.0.3.2)
+      actionpack (= 6.0.3.2)
+      activerecord (= 6.0.3.2)
+      activestorage (= 6.0.3.2)
+      activesupport (= 6.0.3.2)
+      nokogiri (>= 1.8.5)
+    actionview (6.0.3.2)
+      activesupport (= 6.0.3.2)
+      builder (~> 3.1)
+      erubi (~> 1.4)
+      rails-dom-testing (~> 2.0)
+      rails-html-sanitizer (~> 1.1, >= 1.2.0)
+    activejob (6.0.3.2)
+      activesupport (= 6.0.3.2)
+      globalid (>= 0.3.6)
+    activemodel (6.0.3.2)
+      activesupport (= 6.0.3.2)
+    activerecord (6.0.3.2)
+      activemodel (= 6.0.3.2)
+      activesupport (= 6.0.3.2)
+    activestorage (6.0.3.2)
+      actionpack (= 6.0.3.2)
+      activejob (= 6.0.3.2)
+      activerecord (= 6.0.3.2)
+      marcel (~> 0.3.1)
+    activesupport (6.0.3.2)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 0.7, < 2)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+      zeitwerk (~> 2.2, >= 2.2.2)
+    addressable (2.7.0)
+      public_suffix (>= 2.0.2, < 5.0)
+    appraisal (2.3.0)
+      bundler
+      rake
+      thor (>= 0.14.0)
     ast (2.4.1)
+    builder (3.2.4)
+    concurrent-ruby (1.1.6)
+    crack (0.4.3)
+      safe_yaml (~> 1.0.0)
+    crass (1.0.6)
     diff-lcs (1.4.2)
+    erubi (1.9.0)
+    faraday (1.0.1)
+      multipart-post (>= 1.2, < 3)
+    gapic-common (0.3.4)
+      google-protobuf (~> 3.12, >= 3.12.2)
+      googleapis-common-protos (>= 1.3.9, < 2.0)
+      googleapis-common-protos-types (>= 1.0.4, < 2.0)
+      googleauth (~> 0.9)
+      grpc (~> 1.25)
+    globalid (0.4.2)
+      activesupport (>= 4.2.0)
+    google-cloud-core (1.5.0)
+      google-cloud-env (~> 1.0)
+      google-cloud-errors (~> 1.0)
+    google-cloud-env (1.3.3)
+      faraday (>= 0.17.3, < 2.0)
+    google-cloud-errors (1.0.1)
+    google-cloud-pubsub (2.0.0)
+      concurrent-ruby (~> 1.1)
+      google-cloud-core (~> 1.5)
+      google-cloud-pubsub-v1 (~> 0.0)
+    google-cloud-pubsub-v1 (0.1.2)
+      gapic-common (~> 0.3)
+      google-cloud-errors (~> 1.0)
+      grpc-google-iam-v1 (>= 0.6.10, < 2.0)
+    google-protobuf (3.13.0-universal-darwin)
+    googleapis-common-protos (1.3.10)
+      google-protobuf (~> 3.11)
+      googleapis-common-protos-types (>= 1.0.5, < 2.0)
+      grpc (~> 1.27)
+    googleapis-common-protos-types (1.0.5)
+      google-protobuf (~> 3.11)
+    googleauth (0.13.1)
+      faraday (>= 0.17.3, < 2.0)
+      jwt (>= 1.4, < 3.0)
+      memoist (~> 0.16)
+      multi_json (~> 1.11)
+      os (>= 0.9, < 2.0)
+      signet (~> 0.14)
+    grpc (1.32.0-universal-darwin)
+      google-protobuf (~> 3.13)
+      googleapis-common-protos-types (~> 1.0)
+    grpc-google-iam-v1 (0.6.10)
+      google-protobuf (~> 3.11)
+      googleapis-common-protos (>= 1.3.10, < 2.0)
+      grpc (~> 1.27)
+    hashdiff (1.0.1)
+    i18n (1.8.5)
+      concurrent-ruby (~> 1.0)
     jaro_winkler (1.5.4)
+    jwt (2.2.2)
+    loofah (2.6.0)
+      crass (~> 1.0.2)
+      nokogiri (>= 1.5.9)
+    mail (2.7.1)
+      mini_mime (>= 0.1.1)
+    marcel (0.3.3)
+      mimemagic (~> 0.3.2)
+    memoist (0.16.2)
+    method_source (1.0.0)
+    mimemagic (0.3.5)
+    mini_mime (1.0.2)
+    mini_portile2 (2.4.0)
+    minitest (5.14.1)
+    multi_json (1.15.0)
+    multipart-post (2.1.1)
+    nio4r (2.5.2)
+    nokogiri (1.10.10)
+      mini_portile2 (~> 2.4.0)
+    os (1.1.1)
     parallel (1.19.2)
     parser (2.7.1.4)
       ast (~> 2.4.1)
+    public_suffix (4.0.5)
+    rack (2.2.3)
+    rack-test (1.1.0)
+      rack (>= 1.0, < 3)
+    rails (6.0.3.2)
+      actioncable (= 6.0.3.2)
+      actionmailbox (= 6.0.3.2)
+      actionmailer (= 6.0.3.2)
+      actionpack (= 6.0.3.2)
+      actiontext (= 6.0.3.2)
+      actionview (= 6.0.3.2)
+      activejob (= 6.0.3.2)
+      activemodel (= 6.0.3.2)
+      activerecord (= 6.0.3.2)
+      activestorage (= 6.0.3.2)
+      activesupport (= 6.0.3.2)
+      bundler (>= 1.3.0)
+      railties (= 6.0.3.2)
+      sprockets-rails (>= 2.0.0)
+    rails-dom-testing (2.0.3)
+      activesupport (>= 4.2.0)
+      nokogiri (>= 1.6)
+    rails-html-sanitizer (1.3.0)
+      loofah (~> 2.3)
+    railties (6.0.3.2)
+      actionpack (= 6.0.3.2)
+      activesupport (= 6.0.3.2)
+      method_source
+      rake (>= 0.8.7)
+      thor (>= 0.20.3, < 2.0)
     rainbow (3.0.0)
     rake (13.0.1)
+    retriable (3.1.2)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
       rspec-expectations (~> 3.9.0)
@@ -26,6 +196,14 @@ GEM
     rspec-mocks (3.9.1)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.9.0)
+    rspec-rails (4.0.1)
+      actionpack (>= 4.2)
+      activesupport (>= 4.2)
+      railties (>= 4.2)
+      rspec-core (~> 3.9)
+      rspec-expectations (~> 3.9)
+      rspec-mocks (~> 3.9)
+      rspec-support (~> 3.9)
     rspec-support (3.9.3)
     rubocop (0.76.0)
       jaro_winkler (~> 1.5.1)
@@ -37,17 +215,50 @@ GEM
     rubocop-rspec (1.37.0)
       rubocop (>= 0.68.1)
     ruby-progressbar (1.10.1)
+    safe_yaml (1.0.5)
+    signet (0.14.0)
+      addressable (~> 2.3)
+      faraday (>= 0.17.3, < 2.0)
+      jwt (>= 1.5, < 3.0)
+      multi_json (~> 1.10)
+    sprockets (4.0.2)
+      concurrent-ruby (~> 1.0)
+      rack (> 1, < 3)
+    sprockets-rails (3.2.1)
+      actionpack (>= 4.0)
+      activesupport (>= 4.0)
+      sprockets (>= 3.0.0)
+    sqlite3 (1.4.2)
+    thor (1.0.1)
+    thread_safe (0.3.6)
+    timecop (0.9.1)
+    tzinfo (1.2.7)
+      thread_safe (~> 0.1)
     unicode-display_width (1.6.1)
+    webmock (3.8.3)
+      addressable (>= 2.3.6)
+      crack (>= 0.3.2)
+      hashdiff (>= 0.4.0, < 2.0.0)
+    websocket-driver (0.7.3)
+      websocket-extensions (>= 0.1.0)
+    websocket-extensions (0.1.5)
+    zeitwerk (2.4.0)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
+  appraisal
   cloudenvoy!
+  rails
   rake (>= 12.3.3)
   rspec (~> 3.0)
+  rspec-rails
   rubocop (= 0.76.0)
   rubocop-rspec (= 1.37.0)
+  sqlite3
+  timecop
+  webmock
 
 BUNDLED WITH
    2.1.4

data/README.md

@@ -1,8 +1,33 @@
+![Build Status](https://github.com/keypup-io/cloudenvoy/workflows/Test/badge.svg) [![Gem Version](https://badge.fury.io/rb/cloudenvoy.svg)](https://badge.fury.io/rb/cloudenvoy)
+
+**Note**: this gem is currently in alpha stage and has not been tested in production yet.
+
 # Cloudenvoy
 
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/cloudenvoy`. To experiment with that code, run `bin/console` for an interactive prompt.
+Cross-application messaging framework for GCP Pub/Sub.
+
+Cloudenvoy provides an easy to use interface to GCP Pub/Sub. Using Cloudenvoy you can simplify cross-application event messaging by using a publish/subscribe approach. Pub/Sub is particularly suited for micro-service architectures where a great number of components need to be aware of other components' activities. In these architectures using point to point communication via API can quickly become messy and hard to maintain due to the number of interconnections to maintain.
+
+Pub/Sub solves that event distribution problem by allowing developers to define topics, publishers and subscribers to distribute and process event messages. Cloudenvoy furthers simplifies the process of setting up Pub/Sub by giving developers an object-oriented way of managing publishers and subscribers.
+
+Cloudenvoy works with the local pub/sub emulator as well, meaning that you can work offline without access to GCP.
+
+## Summary
 
-TODO: Delete this and the text above, and describe your gem
+1. [Installation](#installation)
+2. [Get started with Rails](#get-started-with-rails)
+3. [Configuring Cloudenvoy](#configuring-cloudenvoy)
+    1. [Pub/Sub authentication & permissions](#pubsub-authentication--permissions)
+    2. [Cloudenvoy initializer](#cloudenvoy-initializer)
+4. [Creating topics and subscriptions](#creating-topics-and-subscriptions)
+    1. [Sending messages](#sending-messages)
+    2. [Publisher implementation layout](#publisher-implementation-layout)
+5. [Receiving messages](#receiving-messages)
+6. [Error Handling](#error-handling)
+7. [Testing](#testing)
+    1. [Test helper setup](#test-helper-setup)
+    2. [In-memory queues](#in-memory-queues)
+    3. [Unit tests](#unit-tests)
 
 ## Installation
 
@@ -20,9 +45,547 @@ Or install it yourself as:
 
     $ gem install cloudenvoy
 
-## Usage
+## Get started with Rails
+
+Cloudenvoy is pre-integrated with Rails. Follow the steps below to get started.
+
+Install the pub/sub local emulator
+```bash
+gcloud components install pubsub-emulator
+gcloud components update
+```
+
+Add the following initializer
+```ruby
+# config/initializers/cloudenvoy.rb
+
+Cloudenvoy.configure do |config|
+  #
+  # GCP Configuration
+  #
+  config.gcp_project_id = 'some-project'
+  config.gcp_sub_prefix = 'my-app'
+
+  #
+  # Adapt the server port to be the one used by your Rails web process
+  #
+  config.processor_host = 'http://localhost:3000'
+
+  #
+  # If you do not have any Rails secret_key_base defined, uncomment the following
+  # This secret is used to authenticate messages sent to the processing endpoint
+  # of your application.
+  #
+  # config.secret = 'some-long-token'
+end
+```
+
+Define a publisher:
+```ruby
+# app/publishers/dummy_publisher.rb
+
+class DummyPublisher
+  include Cloudenvoy::Publisher
+
+  cloudenvoy_options topic: 'test-msgs'
+
+  # Format the message payload. The payload can be a hash
+  # or a string.
+  def payload(msg)
+    {
+      type: 'message',
+      content: msg
+    }
+  end
+end
+```
+
+Define a subscriber:
+```ruby
+# app/subscribers/dummy_subscriber.rb
+
+class HelloSubscriber
+  include Cloudenvoy::Subscriber
+
+  cloudenvoy_options topics: ['test-msgs']
+
+  # Do something with the message
+  def process(message)
+    logger.info("Received message #{message.payload.dig('content')}")
+  end
+end
+```
+
+Launch the pub/sub emulator:
+```bash
+gcloud beta emulators pubsub start
+```
+
+Use cloudenvoy to setup your topic and subscription
+```bash
+bundle exec rake cloudenvoy:setup
+```
+
+Launch Rails
+```bash
+rails s -p 3000
+```
+
+Open a Rails console and send a message
+```ruby
+  DummyPublisher.publish('Hello pub/sub')
+```
+
+Your Rails logs should display the following:
+```log
+Started POST "/cloudenvoy/receive?token=1234" for 66.102.6.140 at 2020-09-16 11:12:47 +0200
+Processing by Cloudenvoy::SubscriberController#receive as JSON
+  Parameters: {"message"=>{"attributes"=>{"kind"=>"hello"}, "data"=>"eyJ0eXBlIjoibWVzc2FnZSIsImNvbnRlbnQiOiJIZWxsbyBmcmllbmQifQ==", "messageId"=>"1501653492745522", "message_id"=>"1501653492745522", "publishTime"=>"2020-09-16T09:12:45.214Z", "publish_time"=>"2020-09-16T09:12:45.214Z"}, "subscription"=>"projects/keypup-dev/subscriptions/my-app.hello_subscriber.test-msgs", "token"=>"eyJhbGciOiJIUzI1NiJ9.eyJpYXQiOjE2MDAyNDc0OTh9.5SbVsDCZLcyoFXseCpPuvE7KY7WXqIQtO6ceoFXcrdw", "subscriber"=>{"message"=>{"attributes"=>{"kind"=>"hello"}, "data"=>"eyJ0eXBlIjoibWVzc2FnZSIsImNvbnRlbnQiOiJIZWxsbyBmcmllbmQifQ==", "messageId"=>"1501653492745522", "message_id"=>"1501653492745522", "publishTime"=>"2020-09-16T09:12:45.214Z", "publish_time"=>"2020-09-16T09:12:45.214Z"}, "subscription"=>"projects/keypup-dev/subscriptions/my-app.hello_subscriber.test-msgs"}}
+[Cloudenvoy][HelloSubscriber][1501653492745522] Processing message... -- {:id=>"1501653492745522", :metadata=>{}, :topic=>"test-msgs"}
+[Cloudenvoy][HelloSubscriber][1501653492745522] Received message Hello pub/sub -- {:id=>"1501653492745522", :metadata=>{}, :topic=>"test-msgs"}
+[Cloudenvoy][HelloSubscriber][1501653492745522] Processing done after 0.001s -- {:id=>"1501653492745522", :metadata=>{}, :topic=>"test-msgs", :duration=>0.001}
+Completed 204 No Content in 1ms (ActiveRecord: 0.0ms | Allocations: 500)
+```
+
+Hurray! Your published message was immediately processed by the subscriber.
+
+## Configuring Cloudenvoy
+
+### Pub/Sub authentication & permissions
+
+The Google Cloud library authenticates via the Google Cloud SDK by default. If you do not have it setup then we recommend you [install it](https://cloud.google.com/sdk/docs/quickstarts).
+
+Other options are available such as using a service account. You can see all authentication options in the [Google Cloud Authentication guide](https://github.com/googleapis/google-cloud-ruby/blob/master/google-cloud-bigquery/AUTHENTICATION.md).
+
+In order to function properly Cloudenvoy requires the authenticated account to have the following IAM permissions:
+- `pubsub.subscriptions.create`
+- `pubsub.subscriptions.get`
+- `pubsub.topics.create`
+- `pubsub.topics.get`
+- `pubsub.topics.publish`
+
+To get started quickly you can add the `roles/pubsub.admin` role to your account via the [IAM Console](https://console.cloud.google.com/iam-admin/iam). This is not required if your account is a project admin account.
+
+### Cloudenvoy initializer
+
+The gem can be configured through an initializer. See below all the available configuration options.
+
+```ruby
+# config/initializers/cloudenvoy.rb
+
+Cloudenvoy.configure do |config|
+  #
+  # If you do not have any Rails secret_key_base defined, uncomment the following.
+  # This secret is used to authenticate messages sent to the processing endpoint
+  # of your application.
+  #
+  # Default with Rails: Rails.application.credentials.secret_key_base
+  #
+  # config.secret = 'some-long-token'
+
+  #
+  # GCP Configuration
+  #
+  config.gcp_project_id = 'some-project'
+
+  #
+  # Specify the namespace for your subscriptions
+  #
+  # The gem attempts to keep GCP subscriptions organized by
+  # properly namespacing them. Each subscription has the following
+  # format:
+  # > projects/<gcp_project_id>/subscriptions/<gcp_sub_prefix>.<subscriber_class>.<topic>
+  #
+  config.gcp_sub_prefix = 'my-app'
+
+  #
+  # Specify the publicly accessible host for your application
+  #
+  # > E.g. in development, using the pub/sub local emulator
+  # config.processor_host = 'http://localhost:3000'
+  #
+  # > E.g. in development, using `config.mode = :production` and ngrok
+  # config.processor_host = 'https://111111.ngrok.io'
+  #
+  config.processor_host = 'https://app.mydomain.com'
+
+  #
+  # Specify the mode of operation:
+  # - :development => messages will be pushed to the local pub/sub emulator
+  # - :production => messages will be pushed to Google Cloud Pub/Sub. Requires a publicly accessible domain.
+  #
+  # Defaults to :development unless CLOUDENVOY_ENV or RAILS_ENV or RACK_ENV is set to something else.
+  #
+  # config.mode = Rails.env.production? || Rails.env.my_other_env? ? :production : :development
+
+  #
+  # Specify the logger to use
+  #
+  # Default with Rails: Rails.logger
+  # Default without Rails: Logger.new(STDOUT)
+  #
+  # config.logger = MyLogger.new(STDOUT)
+end
+```
+
+### Creating topics and subscriptions
+
+Topics and subscriptions can be created using the provided Rake tasks:
+```bash
+# Setup publishers (topics) and subscribers (subscriptions) in one go
+bundle exec rake cloudenvoy:setup
+
+# Or set them up individually
+bundle exec rake cloudenvoy:setup_publishers
+bundle exec rake cloudenvoy:setup_subscribers
+```
+
+For non-rails applications you can run the following in a console to setup your publishers and subscribers:
+```ruby
+DummyPublisher.setup
+DummySubscriber.setup
+```
+
+## Publishing messages
+
+### Sending messages
+
+Cloudenvoy provides a helper method to publish arbitrary messages to any topic.
+```ruby
+Cloudenvoy.publish('my-topic', { 'some' => 'payload' }, { 'optional' => 'message attribute' })
+```
+
+This helper is useful for sending basic messages however it is not the preferred way of sending messages as you will quickly clutter your application with message formatting logic over time.
+
+Cloudenvoy provides an object-oriented way of sending messages allowing developers to separate their core business logic from any kind of message formatting logic. These are called `Publishers`.
+
+The example below shows you how to publish new users to a topic using Cloudenvoy publishers:
+```ruby
+# app/publishers/user_publisher.rb
+
+# The publisher is responsible for configuring and formatting
+# the pub/sub message.
+class UserPublisher
+  include Cloudenvoy::Publisher
+
+  cloudenvoy_options topic: 'system-users'
+
+  # Publishers must at least implement the `payload` method,
+  # which specifies how the message should be formatted.
+  def payload(user)
+    {
+      id: user.id,
+      name: user.name,
+      email: user.email
+    }
+  end
+end
+```
+
+Then in your user model you can do the following:
+```ruby
+# app/users/user_publisher.rb
+
+class User < ApplicationRecord
+  after_create :publish_user
+
+  private
+
+  # Publish users after they have been created
+  def publish_user
+    UserPublisher.publish(self)
+  end
+end
+```
+
+### Publisher implementation layout
+
+A full publisher implementation looks like this:
+```ruby
+class MyPublisher
+  include Cloudenvoy::Publisher
+
+  # The topic option defines the default topic messages will be
+  # sent to. The publishing topic can be overriden on a per message
+  # basis. See the #topic method below.
+  cloudenvoy_options topic: 'my-topic'
+
+  # Evaluate the topic at runtime based on publishing arguments.
+  # Returning `nil` makes the publisher use the default topic
+  # defined via cloudenvoy_options.
+  #
+  # Note: runtime topics do not get created by the rake tasks. You
+  # must create them manually at this stage.
+  def topic(arg1, arg2)
+    arg1 == 'other' ? 'some-other-topic' : nil
+  end
+
+  # Attach pub/sub attributes to the message. Pub/sub attributes
+  # can be used for message filtering.
+  def metadata(arg1, arg2)
+    { reference: "#{arg1}_#{arg2}" }
+  end
+
+  # Publishers must at least implement the `payload` method,
+  # which specifies how arguments should be transformed into
+  # a message payload (Hash or String).
+  def payload(arg1, arg2)
+    {
+      foo: arg1,
+      bar: arg2
+    }
+  end
+
+  # This hook is invoked when the message fails to be formatted and published.
+  # If something wrong happens in the methods above, this hook will be triggered.
+  def on_error(error)
+    logger.error("Oops! Something wrong happened!")
+  end
+end
+```
+
+## Receiving messages
+
+After you have subscribed to a topic, Pub/Sub sends messages to your application via webhook on the `/cloudenvoy/receive` endpoint. Cloudenvoy then automatically dispatches the message to the right subscriber for processing.
+
+Following up on the previous user publishing example, you might define the following subscriber in another Rails application:
+
+```ruby
+# app/subscribers/user_subscriber.rb
+
+class UserSubscriber
+  include Cloudenvoy::Subscriber
+
+  # Subscribers can subscribe to multiple topics
+  cloudenvoy_options topics: ['system-users']
+
+  # Create the user locally if it does not exist already
+  #
+  # A message has the following attributes:
+  #   id: the pub/sub message id
+  #   payload: the content of the message (String or Hash)
+  #   metadata: the pub/sub message attributes
+  #   sub_uri: the pub/sub subscription URI
+  #   topic: the topic the message comes from
+  #
+  def process(message)
+    payload = message.payload
+
+    User.create_or_find_by(system_id: payload['id']) do |u|
+      u.first_name = payload['name']
+      u.email = payload['email']
+    end
+  end
+
+  # This hook will be invoked if the message processing fails
+  def on_error(error)
+    logger.error("The following error happened: #{error}")
+  end
+end
+```
+
+## Logging
+There are several options available to configure logging and logging context.
+
+### Configuring a logger
+Cloudenvoy uses `Rails.logger` if Rails is available and falls back on a plain ruby logger `Logger.new(STDOUT)` if not.
+
+It is also possible to configure your own logger. For example you can setup Cloudenvoy with [semantic_logger](http://rocketjob.github.io/semantic_logger) by doing the following in your initializer:
+```ruby
+# config/initializers/cloudenvoy.rb
+
+Cloudenvoy.configure do |config|
+  config.logger = SemanticLogger[Cloudenvoy]
+end
+```
+
+### Logging context
+Cloudenvoy provides publisher/subscriber contextual information to the `logger` methods.
+
+For example:
+```ruby
+# app/subscribers/dummy_subscriber.rb
+
+class DummySubscriber
+  include Cloudenvoy::Subscriber
+
+  cloudenvoy_options topics: ['my-topic']
+
+  def process(message)
+    logger.info("Subscriber processed with #{message.inspect}. This is working!")
+  end
+end
+```
+
+Will generate the following log with context `{:id=>..., :metadata=>..., :topic=>...}`
+```log
+[Cloudenvoy][DummySubscriber][1501678353930997] Subscriber processed with ###. This is working! -- {:id=>"1501678353930997", :metadata=>{"some"=>"meta"}, :topic=>"my-topic"}
+```
+
+The way contextual information is displayed depends on the logger itself. For example with [semantic_logger](http://rocketjob.github.io/semantic_logger) contextual information might not appear in the log message but show up as payload data on the log entry itself (e.g. using the fluentd adapter).
+
+Contextual information can be customised globally and locally using a log context_processor. By default the loggers are configured this way:
+```ruby
+# Publishers
+Cloudenvoy::PublisherLogger.log_context_processor = ->(publisher) { publisher.message&.to_h&.slice(:id, :metadata, :topic) || {} }
+
+# Subscribers
+Cloudenvoy::SubscriberLogger.log_context_processor = ->(subscriber) { subscriber.message.to_h.slice(:id, :metadata, :topic) }
+```
+
+You can decide to add a global identifier for your publisher logs using the following:
+```ruby
+# config/initializers/cloudenvoy.rb
+
+Cloudenvoy::PublisherLogger.log_context_processor = lambda { |publisher|
+  publisher.message.to_h.slice(:id, :metadata, :topic).merge(app: 'my-app')
+}
+```
+
+You could also decide to log all available context - including the message payload - for specific subscribers only:
+```ruby
+# app/subscribers/full_context_subscriber.rb
+
+class FullContextSubscriber
+  include Cloudenvoy::Subscriber
+
+  cloudenvoy_options topics: ['my-topic'], log_context_processor: ->(s) { s.message.to_h }
+
+  def process(message)
+    logger.info("This log entry will have full context!")
+  end
+end
+```
+
+See the [Cloudenvoy::Publisher](lib/cloudenvoy/publisher.rb), [Cloudenvoy::Subscriber](lib/cloudenvoy/subscriber.rb) and [Cloudenvoy::Message](lib/cloudenvoy/message.rb) for more information on attributes available to be logged in your `log_context_processor` proc.
+
+## Error Handling
+
+Message failures will return an HTTP error to Pub/Sub and trigger a retry at a later time. By default Pub/Sub will retry sending the message until the acknowledgment deadline expires. A number of retries can be explicitly configured by setting up a dead-letter queue.
+
+### HTTP Error codes
+
+When Cloudenvoy fails to process a message it returns the following HTTP error code to Pub/Sub, based on the actual reason:
 
-TODO: Write usage instructions here
+| Code | Description |
+|------|-------------|
+| 204 | The message was processed successfully |
+| 404 | The message subscriber does not exist.  |
+| 422 | An error occured during the processing of the message (`process` method) |
+
+### Error callbacks
+
+Publishers and subscribers can implement the `on_error(error)` callback to do things when a message fails to be published or received:
+
+E.g.
+```ruby
+# app/publisher/handle_error_publisher.rb
+
+class HandleErrorPublisher
+  include Cloudenvoy::Publisher
+
+  cloudenvoy_options topic: 'my-topic'
+
+  def payload(arg)
+    raise(ArgumentError)
+  end
+
+  # The runtime error is passed as an argument.
+  def on_error(error)
+    logger.error("The following error occured: #{error}")
+  end
+end
+```
+
+## Testing
+Cloudenvoy provides several options to test your publishers and subscribers.
+
+### Test helper setup
+Require `cloudenvoy/testing` in your `rails_helper.rb` (Rspec Rails) or `spec_helper.rb` (Rspec) or test unit helper file then enable one of the two modes:
+
+```ruby
+require 'cloudenvoy/testing'
+
+# Mode 1 (default): Push messages to GCP Pub/Sub (env != development)
+Cloudenvoy::Testing.enable!
+
+# Mode 2: Push message to in-memory queues. You will be responsible for clearing the
+# topic queues using `Cloudenvoy::Testing.clear_all` or `Cloudenvoy::Testing.clear('my-topic')`
+Cloudenvoy::Testing.fake!
+```
+
+You can query the current testing mode with:
+```ruby
+Cloudenvoy::Testing.enabled?
+Cloudenvoy::Testing.fake?
+```
+
+Each testing mode accepts a block argument to temporarily switch to it:
+```ruby
+# Enable fake mode for all tests
+Cloudenvoy::Testing.fake!
+
+# Enable real mode temporarily for a given test
+Cloudenvoy.enable! do
+   MyPublisher.publish(1,2)
+end
+```
+
+Note that extension middlewares - if any has been registered - run in test mode. You can disable middlewares in your tests by adding the following to your test helper:
+```ruby
+# Remove all middlewares
+Cloudenvoy.configure do |c|
+  c.publisher_middleware.clear
+  c.subscriber_middleware.clear
+end
+
+# Remove all specific middlewares
+Cloudenvoy.configure do |c|
+  c.publisher_middleware.remove(MyMiddleware::Publisher)
+  c.subscriber_middleware.remove(MyMiddleware::Subscriber)
+end
+```
+
+### In-memory queues
+The `fake!` modes uses in-memory queues for topics, which can be queried and controlled using the following methods:
+
+```ruby
+# Clear all messages across all topics
+Cloudenvoy::Testing.clear_all
+
+# Remove all messages in a given topic
+Cloudenvoy::Testing.clear('my-top')
+
+# Get all messages for a given topic
+Cloudenvoy::Testing.queue('my-top')
+```
+
+### Unit tests
+Below are examples of rspec tests. It is assumed that `Cloudenvoy::Testing.fake!` has been set in the test helper.
+
+**Example 1**: Testing publishers
+```ruby
+describe 'message publishing'
+  subject(:publish_message) { MyPublisher.publish(1,2) }
+
+  let(:queue) { Cloudenvoy::Testing.queue('my-topic') }
+
+  it { expect { publish_message }.to change(queue, :size).by(1) }
+  it { is_expected.to have_attributes(payload: { 'foo' => 'bar' }) }
+end
+```
+
+**Example 2**: Testing subscribers
+```ruby
+describe 'message processing'
+  subject { VerifyDataViaApiSubscriber.new(message: message).execute } }
+
+  let(:message) { Cloudenvoy::Message.new(payload: { 'some' => 'payload' }) }
+
+  before { expect(MyApi).to receive(:fetch).and_return([]) }
+  it { is_expected.to be_truthy }
+end
+```
 
 ## Development
 
@@ -32,8 +595,7 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/alachaum/cloudenvoy. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](https://github.com/alachaum/cloudenvoy/blob/master/CODE_OF_CONDUCT.md).
-
+Bug reports and pull requests are welcome on GitHub at https://github.com/keypup-io/cloudenvoy. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](https://github.com/keypup-io/cloudenvoy/blob/master/CODE_OF_CONDUCT.md).
 
 ## License
 
@@ -41,4 +603,4 @@ The gem is available as open source under the terms of the [MIT License](https:/
 
 ## Code of Conduct
 
-Everyone interacting in the Cloudenvoy project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/alachaum/cloudenvoy/blob/master/CODE_OF_CONDUCT.md).
+Everyone interacting in the Cloudenvoy project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/keypup-io/cloudenvoy/blob/master/CODE_OF_CONDUCT.md).