pipeline.rb 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 197e078f9b2e93718eedbde53a8c453536c166a2
+  data.tar.gz: ff87a1819aacea18aa67db93c54c1364d14d388e
+SHA512:
+  metadata.gz: 90ab0ea4b657163aaf74ffd388377ac730e073292a56ccfab08fe9ea70f60d1e091dc6ed5205729b107bc22593e4725f0e2ce2ea4173231fb4d77d9b2ecd848d
+  data.tar.gz: 8e176b651fc34d7d05e18d510214c9967f5e867bcd6c40997377abf5603f1a1392f0e1b79fd6284eb399235d7b361921fef458142979e969e118b1e317b74a31

data/Gemfile

@@ -0,0 +1,6 @@
+gemspec
+
+group(:spec) do
+  gem 'rspec-core'
+  gem 'rspec-expectations'
+end

data/Gemfile.lock

@@ -0,0 +1,25 @@
+PATH
+  remote: .
+  specs:
+    pipeline.rb (0.0.1)
+      amq-client
+
+GEM
+  specs:
+    amq-client (1.0.4)
+      amq-protocol (>= 1.9.0)
+      eventmachine
+    amq-protocol (1.9.2)
+    diff-lcs (1.2.5)
+    eventmachine (1.0.3)
+    rspec-core (2.14.7)
+    rspec-expectations (2.14.4)
+      diff-lcs (>= 1.1.3, < 2.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  pipeline.rb!
+  rspec-core
+  rspec-expectations

data/LINCENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 @botanicus
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,141 @@
+# About
+
+Plugin pipeline infrastructure connected **through RabbitMQ**. The plugins are just **normal Ruby gems**.
+
+```
+Instance(s) of plugin 1 -> RabbitMQ broker -> Instance(s) of plugin 2
+```
+
+It uses `amq.topic` exchange, so **you can use globs** to consume from within a plugin or a custom script. So if you want to see all the messages going through the system, simply create a queue a bind it to `amq.topic` with routing key `#`. Need all the events? `#event#`. Easy as that.
+
+# Why?
+
+It's been proved over and over that splitting app into services and is the way to go, because:
+
+* It's **easier to maintain** such app.
+* **Rewriting components** is easy.
+* It's **easy to scale** by adding more consumers of given service.
+* It's **easy to inspect** what's going on.
+* It's much **easier to split work** on the project between multiple developers.
+* You can **restart services without losing any data**, as they are kept in RabbitMQ.
+* You don't have to introduce any code dependencies: just consume given queue and publish to another one.
+
+# Settings
+
+Pipeline.rb uses JSON for configuration.
+
+The cool thing is it supports global and local configuration. That means for instance if you want to configure a database, you're probably going to use the same database drivers on either development machine or on production server.
+
+So first let's create a global configuration file `config/database.json`:
+
+```json
+{
+  "adapter":  "mysql",
+  "username": "mysql",
+  "database": "example"
+}
+```
+
+And now the local `config/database.local.json`:
+
+```json
+{
+  "password": "ae28cd87adb5c385117f89e9bd452d18"
+}
+```
+
+Don't forget to ignore the local settings `.gitignore`:
+
+```
+config/*.local.json
+```
+
+# HOWTO
+
+Pipeline.rb expects you to provide AMQP configuration, so it knows how to connect to RabbitMQ:
+
+`config/amqp.json`:
+
+```json
+{
+  "vhost": "example",
+  "user":  "example",
+}
+```
+
+`config/amqp.local.json`:
+
+```json
+{
+  "password": "ae28cd87adb5c385117f89e9bd452d18"
+}
+```
+
+Don't forget to ignore the local settings `.gitignore`:
+
+```
+config/*.local.json
+```
+
+Shop for plugins and add them to your `Gemfile`:
+
+```ruby
+source 'https://rubygems.org'
+
+# The app uses pipeline.rb itself.
+gem 'pipeline.rb'
+
+# Tasks.
+gem 'nake'
+
+# Existing pipeline.rb plugins.
+group(:plugins) do
+  gem 'mail_queue'
+end
+```
+
+```ruby
+#!/usr/bin/env bundle exec ruby
+
+require 'nake/runner'
+
+# Load all the plugins.
+Bundler.require(:plugins)
+
+# Tasks: declare.
+require 'pipeline/tasks'
+
+Nake.run
+```
+
+Now run `./tasks.rb declare` to declare all the required queues.
+
+And finally create your app:
+
+```ruby
+require 'mail_queue'
+
+# This is already loaded,
+# but just to make it clear
+# that the app is actually
+# just another plugin.
+require 'pipeline/plugin'
+
+class App < Pipeline::Plugin
+  def run
+    EM.add_timer(0.5) do
+      client.publish("Hello World!", 'emails.random')
+    end
+  end
+end
+```
+
+And finally run it all:
+
+1. Start RabbitMQ.
+2. Start `bin/mail_queue.rb` to send your emails.
+3. Run your app.
+
+# Deployment
+
+TODO: Show example Upstart scripts.

data/example/Gemfile

@@ -0,0 +1,12 @@
+source 'https://rubygems.org'
+
+# Local Machine
+# bundle config local.pipeline-mail_queue ~/Dropbox/Code/pipeline-plugins/mail_queue
+# bundle config local.pipeline.rb ~/Dropbox/Code/sources/pipeline.rb
+# bundle config local.nake ~/Dropbox/Code/sources/nake
+gem 'pipeline.rb', github: 'botanicus/pipeline.rb', branch: 'master'
+gem 'nake', github: 'botanicus/nake', branch: 'master'
+
+group(:plugins) do
+  gem 'pipeline-mail_queue', github: 'botanicus/pipeline-mail_queue', branch: 'master'
+end

data/example/Gemfile.lock

@@ -0,0 +1,52 @@
+GIT
+  remote: git://github.com/botanicus/nake.git
+  revision: 6361daacec1caa6af9e12e7e52e0fcb6cf1b91ac
+  branch: master
+  specs:
+    nake (0.0.9.5)
+      term-ansicolor
+
+GIT
+  remote: git://github.com/botanicus/pipeline-mail_queue.git
+  revision: 5f860a6a01de0a85f181e918f1432c05a20b5b37
+  branch: master
+  specs:
+    pipeline-mail_queue (0.0.1)
+      mail
+      pipeline.rb
+
+GIT
+  remote: git://github.com/botanicus/pipeline.rb.git
+  revision: 345259bc7bc48b8e3a5cc6e8e2594aaefe2bfa96
+  branch: master
+  specs:
+    pipeline.rb (0.0.1)
+      amq-client
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    amq-client (1.0.4)
+      amq-protocol (>= 1.9.0)
+      eventmachine
+    amq-protocol (1.9.2)
+    eventmachine (1.0.3)
+    mail (2.5.4)
+      mime-types (~> 1.16)
+      treetop (~> 1.4.8)
+    mime-types (1.25.1)
+    polyglot (0.3.3)
+    term-ansicolor (1.2.2)
+      tins (~> 0.8)
+    tins (0.13.1)
+    treetop (1.4.15)
+      polyglot
+      polyglot (>= 0.3.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  nake!
+  pipeline-mail_queue!
+  pipeline.rb!

data/example/README.md

@@ -0,0 +1,38 @@
+# About
+
+# Setup
+
+## Set Up and Run RabbitMQ
+
+```bash
+rabbitmqctl add_user example ae28cd87adb5c385117f89e9bd452d18
+rabbitmqctl add_vhost example
+rabbitmqctl set_permissions -p example example '.*' '.*' '.*'
+```
+
+## Fill In Your Gmail Details
+
+`config/smtp.local.json`
+
+```json
+{
+  "user_name": "you@gmail.com",
+  "password": "password",
+}
+```
+
+## Run Bundler
+
+`bundle install`
+
+## Declare Queues
+
+`./tasks.rb declare`
+
+## Run `mail_queue` Consumer
+
+`bundle exec mail_queue.rb`
+
+## Run The App
+
+`./app.rb`

data/example/app.rb

@@ -0,0 +1,29 @@
+#!/usr/bin/env bundle exec ruby
+
+require 'mail_queue'
+
+# This is already loaded,
+# but just to make it clear
+# that the app is actually
+# just another plugin.
+require 'pipeline/plugin'
+
+$receiver_email = (ARGV.shift || abort("Usage: #{$0} [receiver_email"))
+
+class App < Pipeline::Plugin
+  def run
+    email = Mail.new do
+      from    'james@101ideas.cz'
+      to      $receiver_email
+      subject "Hello World!"
+      body    "Hello World!"
+    end
+
+    EM.add_periodic_timer(1.5) do
+      puts "~ Publishing #{email.inspect}"
+      client.publish(email.to_s, 'emails.random')
+    end
+  end
+end
+
+App.run(Dir.pwd)

data/example/config/amqp.json

@@ -0,0 +1,5 @@
+{
+  "vhost":    "example",
+  "user":     "example",
+  "password": "ae28cd87adb5c385117f89e9bd452d18"
+}

data/example/config/smtp.json

@@ -0,0 +1,9 @@
+{
+  "address": "smtp.gmail.com",
+  "port": 587,
+  "domain": "101ideas.cz",
+  "user_name": "TODO: Override in config/smtp.local.json",
+  "password": "TODO: Override in config/smtp.local.json",
+  "authentication": "plain",
+  "enable_starttls_auto": true
+}

data/example/config/smtp.local.json

@@ -0,0 +1,4 @@
+{
+  "user_name": "james@101ideas.cz",
+  "password": "3ff5s0sgd4dg1gdsFffhsds5s2s58sf6"
+}

data/example/tasks.rb

@@ -0,0 +1,11 @@
+#!/usr/bin/env bundle exec ruby
+
+require 'nake/runner'
+
+# Load all the plugins.
+Bundler.require(:plugins)
+
+# Tasks: declare.
+require 'pipeline/tasks'
+
+Nake.run

data/lib/pipeline/client.rb

@@ -0,0 +1,98 @@
+require 'eventmachine'
+require 'amq/client'
+require 'json'
+
+# We aren't handling TCP connection lost, since this is running on the same machine.
+# It means, however, that if we restart the broker, we have to restart all the services manually.
+module Pipeline
+  class Client
+    def self.boot(config)
+      client = self.new
+
+      # Next tick, so we can use it with Thin.
+      EM.next_tick do
+        client.connect(config.merge(adapter: 'eventmachine'))
+
+        # Set up signals.
+        ['INT', 'TERM'].each do |signal|
+          Signal.trap(signal) do
+            puts "~ Received #{signal} signal, terminating."
+            client.disconnect { EM.stop }
+          end
+        end
+      end
+
+      client
+    end
+
+    def initialize
+      @on_open_callbacks = Array.new
+    end
+
+    attr_reader :connection, :channel, :exchange, :on_open_callbacks
+    def connect(opts)
+      @connection = AMQ::Client.connect(opts)
+      @channel = AMQ::Client::Channel.new(@connection, 1)
+
+      @connection.on_open do
+        puts "~ Connected to RabbitMQ."
+
+        @channel.open do
+          self.on_open_callbacks.each do |callback|
+            callback.call
+          end
+        end
+      end
+    end
+
+    def exchange
+      @exchange ||= AMQ::Client::Exchange.new(@connection, @channel, 'amq.topic')
+    end
+
+    def declare_queue(name, routing_key)
+      queue = AMQ::Client::Queue.new(@connection, @channel, name)
+
+      self.on_open do
+        queue.declare(false, true, false, true) do
+          # puts "~ Queue #{queue.name.inspect} is ready"
+        end
+
+        queue.bind(self.exchange.name, routing_key) do
+          puts "~ Queue #{queue.name} is now bound to #{self.exchange.name} with routing key #{routing_key}"
+        end
+      end
+
+      queue
+    end
+
+    def consumer(name, routing_key = name, &block)
+      queue = self.declare_queue(name, routing_key)
+
+      queue.consume(true) do |consume_ok|
+        puts "Subscribed for messages routed to #{queue.name}, consumer tag is #{consume_ok.consumer_tag}, using no-ack mode"
+
+        queue.on_delivery do |basic_deliver, header, payload|
+          block.call(payload, header, basic_deliver)
+        end
+      end
+    end
+
+    # This runs after the channel is open.
+    # TODO: Why amq-client doesn't support adding multiple callbacks?
+    def on_open(&block)
+      if @channel.status == :opening
+        self.on_open_callbacks << block
+      else
+        block.call
+      end
+    end
+
+    def publish(*args)
+      self.exchange.publish(*args)
+    end
+
+    def disconnect(&block)
+      @connection.disconnect(&block)
+    end
+  end
+end

data/lib/pipeline/plugin.rb

@@ -0,0 +1,63 @@
+require 'json'
+require 'pathname'
+require 'pipeline/client'
+
+module Pipeline
+  class Plugin
+    def self.plugins
+      @@plugins ||= Array.new
+    end
+
+    def self.inherited(subclass)
+      Pipeline::Plugin.plugins << subclass
+    end
+
+    def self.run(root)
+      plugin = self.new(root)
+
+      Dir.chdir(plugin.root.to_s) do
+        EM.run do
+          # First runs next tick from Client.boot.
+          EM.next_tick do
+            plugin.run!
+          end
+        end
+      end
+    end
+
+    attr_reader :root
+    def initialize(root)
+      @root = Pathname.new(root)
+      @config = Hash.new
+
+      self.client
+    end
+
+    def config(key)
+      @config[key] ||= begin
+        path = self.root.join("config/#{key}.json")
+        data = JSON.parse(path.read)
+
+        path = self.root.join("config/#{key}.local.json")
+        data.merge!(JSON.parse(path.read)) if path.exist?
+
+        data.reduce(Hash.new) do |buffer, (key, value)|
+          buffer.merge!(key.to_sym => value)
+        end
+      end
+    end
+
+    def client
+      @client ||= Pipeline::Client.boot(self.config('amqp'))
+    end
+
+    def run!
+      self.client.on_open do
+        EM.add_timer(0.1) do # Oh my, yeah, whatever, it's for now only, dammit!
+          puts "~ Running #{self.class}#run."
+          self.run
+        end
+      end
+    end
+  end
+end

data/lib/pipeline/tasks.rb

@@ -0,0 +1,16 @@
+require 'nake'
+
+Nake::Task.new(:queues) do |task|
+  task.define do
+    Pipeline::Plugin.plugins.each do |plugin|
+      puts "#{plugin}: #{plugin::QUEUES.inspect rescue []}"
+    end
+  end
+end
+
+
+Nake::Task.new(:declare) do |task|
+  task.define do
+    abort "Still on the TODO list.\nFor now, just run all the consumers and everything gets declared at runtime."
+  end
+end

data/lib/pipeline/version.rb

@@ -0,0 +1,3 @@
+module Pipeline
+  VERSION ||= '0.0.1'
+end

data/pipeline.rb.gemspec

@@ -0,0 +1,20 @@
+#!/usr/bin/env gem build
+
+$LOAD_PATH.unshift(File.expand_path('../lib', __FILE__))
+
+require 'pipeline/version'
+
+Gem::Specification.new do |s|
+  s.name = 'pipeline.rb'
+  s.version = Pipeline::VERSION
+  s.authors = ['@botanicus']
+  s.homepage = 'http://github.com/botanicus/pipeline.rb'
+  s.summary = "Plugin pipeline infrastructure connected through RabbitMQ."
+  s.description = "#{s.summary}. Asynchronous and robust. Also contains configuration system."
+  s.email = 'james@101ideas.cz'
+  s.files = Dir.glob('**/*')
+  s.license = 'MIT'
+  s.require_paths = ['lib']
+
+  s.add_dependency 'amq-client'
+end

data/spec/data/app/config/amqp.json

@@ -0,0 +1,5 @@
+{
+  "vhost":    "example",
+  "user":     "example",
+  "password": "ae28cd87adb5c385117f89e9bd452d18"
+}

data/spec/data/app/config/smtp.json

@@ -0,0 +1,8 @@
+{
+  "address": "smtp.gmail.com",
+  "port": 587,
+  "domain": "101ideas.cz",
+  "user_name": "user.global",
+  "authentication": "plain",
+  "enable_starttls_auto": true
+}

data/spec/data/app/config/smtp.local.json

@@ -0,0 +1,4 @@
+{
+  "user_name": "user.local",
+  "password": "password.local"
+}

data/spec/data/app/config/test.json

@@ -0,0 +1,3 @@
+{
+  "local": false
+}

data/spec/plugin_spec.rb

@@ -0,0 +1,42 @@
+require 'spec_helper'
+
+require 'pipeline/plugin'
+
+describe Pipeline::Plugin do
+  describe '#new(root)' do
+    it "takes one argument which is the root directory" do
+      expect { described_class.new(File.join(Dir.pwd, 'spec', 'data', 'app')) }.not_to raise_error
+    end
+  end
+
+  describe '#root' do
+    subject { described_class.new(File.join(Dir.pwd, 'spec', 'data', 'app')) }
+
+    it "returns path to the root directory" do
+      subject.root.should be_kind_of(Pathname)
+      subject.root.to_s.should match('pipeline.rb/spec/data/app')
+    end
+  end
+
+  describe '#config' do
+    subject { described_class.new(File.join(Dir.pwd, 'spec', 'data', 'app')) }
+
+    let(:config) { subject.config('smtp') }
+
+    it "contains the global settings" do
+      config[:address].should eql('smtp.gmail.com')
+    end
+
+    it "contains the global settings" do
+      config[:password].should eql('password.local')
+    end
+
+    it "overrides the global settings by the local settings" do
+      config[:user_name].should eql('user.local')
+    end
+
+    it "should not require local settings" do
+      subject.config('test')[:local].should be_false
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,3 @@
+RSpec.configure do |config|
+  config.mock_framework = 'gimme a break will ya?'
+end

metadata

@@ -0,0 +1,81 @@
+--- !ruby/object:Gem::Specification
+name: pipeline.rb
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- '@botanicus'
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-01-13 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: amq-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Plugin pipeline infrastructure connected through RabbitMQ.. Asynchronous
+  and robust. Also contains configuration system.
+email: james@101ideas.cz
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- example/app.rb
+- example/config/amqp.json
+- example/config/smtp.json
+- example/config/smtp.local.json
+- example/Gemfile
+- example/Gemfile.lock
+- example/README.md
+- example/tasks.rb
+- Gemfile
+- Gemfile.lock
+- lib/pipeline/client.rb
+- lib/pipeline/plugin.rb
+- lib/pipeline/tasks.rb
+- lib/pipeline/version.rb
+- LINCENSE
+- pipeline.rb.gemspec
+- README.md
+- spec/data/app/config/amqp.json
+- spec/data/app/config/smtp.json
+- spec/data/app/config/smtp.local.json
+- spec/data/app/config/test.json
+- spec/plugin_spec.rb
+- spec/spec_helper.rb
+homepage: http://github.com/botanicus/pipeline.rb
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: Plugin pipeline infrastructure connected through RabbitMQ.
+test_files: []