magic_pipe 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: a16af40ae46abcba5bafeca16db82382e91bd4ee
+  data.tar.gz: 31eb6c6e23ce061727e467879869a749adf7af08
+SHA512:
+  metadata.gz: 4cfebdd5448a995c0a93b0f10b1ac5f950e9ed9c2d94c1eeed99020da205d1943d520d2c24fa91279e48eb779f27cf5f2de7fb5ad469277fb2f083675abc66c2
+  data.tar.gz: 82cfe543de0788e2e3cf758450396de4707fa7255b0bc90564465203b3d6f439e23120cc85a38c20ac1c72ca23edee4fa3d738399c3f314f778a71dd406ecff1

data/.gitignore

@@ -0,0 +1,11 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.travis.yml

@@ -0,0 +1,8 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.4.3
+  - 2.3.6
+  - 2.2.9
+script:
+  - bundle exec rspec

data/CHANGELOG.md

@@ -0,0 +1,10 @@
+# MagicPipe Changelog
+
+## v0.1.0
+
+Initial release.
+
+Codecs: json, message_pack, jaml.
+Loaders: simple_active_record.
+Senders: sync and async (Sidekiq).
+Transports: debug, log, https, sqs, multi.

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in magic_pipe.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,92 @@
+PATH
+  remote: .
+  specs:
+    magic_pipe (0.1.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.5.2)
+      public_suffix (>= 2.0.2, < 4.0)
+    aws-partitions (1.56.0)
+    aws-sdk-core (3.14.0)
+      aws-partitions (~> 1.0)
+      aws-sigv4 (~> 1.0)
+      jmespath (~> 1.0)
+    aws-sdk-sqs (1.3.0)
+      aws-sdk-core (~> 3)
+      aws-sigv4 (~> 1.0)
+    aws-sigv4 (1.0.2)
+    coderay (1.1.2)
+    concurrent-ruby (1.0.5)
+    connection_pool (2.2.1)
+    crack (0.4.3)
+      safe_yaml (~> 1.0.0)
+    diff-lcs (1.3)
+    ethon (0.11.0)
+      ffi (>= 1.3.0)
+    faraday (0.14.0)
+      multipart-post (>= 1.2, < 3)
+    ffi (1.9.18)
+    hashdiff (0.3.7)
+    jmespath (1.3.1)
+    method_source (0.9.0)
+    msgpack (1.2.2)
+    multipart-post (2.0.0)
+    oj (3.3.10)
+    pry (0.11.3)
+      coderay (~> 1.1.0)
+      method_source (~> 0.9.0)
+    public_suffix (3.0.1)
+    rack (2.0.3)
+    rack-protection (2.0.0)
+      rack
+    rake (10.5.0)
+    redis (4.0.1)
+    rspec (3.7.0)
+      rspec-core (~> 3.7.0)
+      rspec-expectations (~> 3.7.0)
+      rspec-mocks (~> 3.7.0)
+    rspec-core (3.7.0)
+      rspec-support (~> 3.7.0)
+    rspec-expectations (3.7.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.7.0)
+    rspec-mocks (3.7.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.7.0)
+    rspec-support (3.7.0)
+    safe_yaml (1.0.4)
+    sidekiq (5.0.5)
+      concurrent-ruby (~> 1.0)
+      connection_pool (~> 2.2, >= 2.2.0)
+      rack-protection (>= 1.5.0)
+      redis (>= 3.3.4, < 5)
+    timecop (0.9.1)
+    typhoeus (1.3.0)
+      ethon (>= 0.9.0)
+    webmock (3.3.0)
+      addressable (>= 2.3.6)
+      crack (>= 0.3.2)
+      hashdiff
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  aws-sdk-sqs (~> 1.3)
+  bundler (~> 1.16)
+  faraday
+  magic_pipe!
+  msgpack
+  oj
+  pry (~> 0.11)
+  rake (~> 10.0)
+  rspec (~> 3.0)
+  sidekiq
+  timecop (~> 0.9)
+  typhoeus
+  webmock (~> 3.3)
+
+BUNDLED WITH
+   1.16.1

data/LICENSE.txt

@@ -0,0 +1,19 @@
+Copyright (c) 2018 Tommaso Pavese
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,241 @@
+# MagicPipe
+
+[![Build Status](https://travis-ci.org/tompave/magic_pipe.svg?branch=master)](https://travis-ci.org/tompave/magic_pipe) [![Gem Version](https://badge.fury.io/rb/magic_pipe.svg)](https://badge.fury.io/rb/magic_pipe)
+
+MagicPipe is a Ruby library to push data to remote destinations on multiple topics.
+
+It provides client adapters for several popular message busses, and it's meant to facilitate publishing messages and streaming data, in different formats and to different backends.
+
+## Content
+
+* [Design, concepts and internals](#design-concepts-and-internals)
+  - [The moving parts](#the-moving-parts)
+    + [Codecs](#codecs)
+    + [Transports](#transports)
+    + [Senders](#senders)
+    + [Loaders](#loaders)
+  - [Gluing everything together](#gluing-everything-together)
+  - [Multiple pipes](#multiple-pipes)
+  - [The message payloads](#the-message-payloads)
+* [Usage](#usage)
+  - [Configuration](#configuration)
+* [Dependencies](#dependencies)
+* [Use cases](#use-cases)
+* [Installation](#installation)
+
+## Design, concepts and internals
+
+Its design principles are:
+
+* It should be plug and play, with minimal configuration -- it's an opinionated library.
+* It should support different message formats and backends with a consistent interface.
+* It should allow for multiple backends to be targeted at the same time.
+* It should be extendable and customizable.
+
+To achieve these goals, MagicPipe adopts a modular design with interchangeable parts.
+
+### The moving parts
+
+The four main units of work are codecs, transports, senders and loaders. MagicPipe provides a set of classes out of the box, but users of the library can configure their own custom classes that implement the correct interface.
+
+#### Codecs
+
+Codecs accept a Ruby object and produce an encoded output. The input must respond to `as_json` and return a Hash. The provided codecs are:
+
+* Yaml
+* JSON
+* MessagePack
+* Thrift (work in progress)
+
+#### Transports
+
+Transports are adapters for the different backends, and take care of establishing connections and submitting the payloads. The provided transports are:
+
+* Debug
+* Log
+* HTTPS
+* SQS
+* Kafka (work in progress)
+* DynamoDB (work in progress)
+* Multi (allows to target different transports at the same time)
+
+#### Senders
+
+Senders glue things together and implement a processing strategy. The provided senders are:
+
+* Sync
+* Async (Sidekiq)
+
+#### Loaders
+
+Loaders are used with the Async sender to serialize Ruby objects into something that can be passed to Sidekiq, and then to rebuild the original Objects inside the Sidekiq workers. The provided loaders are:
+
+* SimpleActiveRecord: it takes `ActiveRecord::Base` instances and an optional wrapper (e.g. an `ActiveModel::Serializer`) and turns them into class references (Strings) and a record ID. Then, when the Sidekiq jobs execute, it loads the record from the DB and wraps it in the serializer, if present.
+
+### Gluing everything together
+
+A MagicPipe client encapsulates the configured parts. For example, clients with a single and multiple transports can be represented like this:
+
+```
+Client
+├ Configuration
+└ Sender
+  ├ Codec
+  ├ (Loader)
+  └ Transport
+
+Client
+├ Configuration
+└ Sender
+  ├ Codec
+  ├ (Loader)
+  └ Transports
+    ├ Transport A
+    ├ Transport B
+    └ Transport C
+```
+
+A client can only have a single codec and sender, but can have multiple transports to submit data to multiple backends. This is particularly efficient because it allows to reduce the number of DB queries to re-load the data, when using the async sender.
+
+### Multiple pipes
+
+Multiple clients with different configurations can be used together in the same process.
+
+The main use case is to support different codecs (message formats). Some applications may in fact need to emit messages with different formats to different backends, for example JSON to both SQS and a remote HTTPS endpoint, and Thrift to Kafka.
+
+Another use case is to use different Sidekiq queues (and worker pools) for different topics, which can be accomplished by using different MagicPipe clients for different types of objects.
+
+### The message payloads
+
+MagicPipe wraps the payloads in message envelopes with extra metadata. These extra attributes are:
+
+* the message topic (string)
+* the producer name (string)
+* the submission timestamp, captured when `client#send_data` is invoked (integer)
+* the payload mime type, e.g. `application/json` (string)
+
+Some transports will additionally provide this metadata as message meta attributes. For example, the HTTPS transport will set them as custom HTTP request headers, and the SQS transport will set them as the SQS message custom attributes.
+
+## Usage
+
+Create and configure a MagicPipe client:
+(Temporary API! Still a work in progress)
+
+```ruby
+require "magic_pipe/senders/async"
+require "magic_pipe/transports/https"
+require "magic_pipe/transports/sqs"
+
+$magic_pipe = MagicPipe.build do |mp|
+  mp.sender = :async
+  mp.loader = :simple_active_record
+
+  mp.codec = :json
+  mp.transports = [:https, :sqs]
+
+  mp.sidekiq_options = {
+    queue: "magic_pipe"
+  }
+  mp.https_transport_options = {
+    url: "https://my.receiver.service/foo",
+    auth_token: "bar",
+  }
+  mp.sqs_transport_options = {
+    queue: "my_data_stream"
+  }
+
+  mp.logger = Rails.logger
+  mp.metrics_client = $statsd_client
+end
+```
+
+Then, to submit a message:
+
+```ruby
+$magic_pipe.send_data(
+  object: object,
+  topic: "my_topic",
+  wrapper: nil, # default
+  time: Time.now.utc # default
+)
+```
+
+A more concrete example, with an active record object:
+
+```ruby
+class Article < ActiveRecord::Base
+  after_commit :send_on_magic_pipe
+
+  private
+
+  def send_on_magic_pipe
+    $magic_pipe.send_data(
+      object: self,
+      topic: "articles",
+      wrapper: Serializers::InventoryArticleSerializer,
+      time: updated_at
+    )
+  end
+end
+```
+
+### Configuration
+
+The [`MagicPipe::Config`](https://github.com/tompave/magic_pipe/blob/master/lib/magic_pipe/config.rb) class lists all supported configuration options and their default values.
+
+Users of the library are strongly encouraged to set explicit values for these settings:
+
+* `client_name`: used internally to identify a specific magic pipe client instance, for example when handing over the message publication to Sidekiq.
+* `producer_name`: the name of your application, as it should appear in the metadata of the published messages.
+* `logger`: when used in Rails, this should be set to `Rails.logger`.
+* `metrics_client`: to collect operational metrics. Wrapped by [`MagicPipe::Metrics`](https://github.com/tompave/magic_pipe/blob/master/lib/magic_pipe/metrics.rb). It's mainly designed to work with [DataDog's StatsD](https://rubygems.org/gems/dogstatsd-ruby) out of the box, but any stats collector library would work, if you provide a thin adapter wrapper. If not configured, the default implementation will simply send the metrics to the logger.
+
+More specific configuration is then required for the different modules of the library, when used.
+
+#### Transport: SQS
+
+This transport requires credentials for the AWS API. The credentials need to be associated to an IAM user with full access to SQS, and need to be present in the system env:
+
+```
+export AWS_ACCESS_KEY_ID='foo'
+export AWS_SECRET_ACCESS_KEY='bar'
+export AWS_REGION='us-east-1'
+```
+
+## Dependencies
+
+Becasuse of MagicPipe's modular design, and in order to keep a small installation footprint, all of its dependencies are optional. Users of the library need to manually install the required dependencies in their projects.
+
+The Ruby gems MagicPipe's modules depend on are:
+
+* Senders:
+  - Async: `sidekiq`
+* Transports:
+  - SQS: `aws-sdk-sqs`
+  - HTTPS: `faraday`, `typhoeus`
+* Codecs:
+  - JSON: `oj` (optional, will fallback to `json` from the stdlib if `oj` is missing)
+  - MessagePack: `msgpack`
+
+## Use cases
+
+TODO
+
+* event driven architectures
+* streaming domain data to replicate it somewhere else
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'magic_pipe'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install magic_pipe

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+
+ENV['AWS_ACCESS_KEY_ID']      = ""
+ENV['AWS_SECRET_ACCESS_KEY']  = ""
+ENV['AWS_SDK_CONFIG_OPT_OUT'] = "true"
+ENV['AWS_REGION']             = "eu-west-1"
+
+require "bundler/setup"
+require "magic_pipe"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+require "pry"
+Pry.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here