eventhub-processor2 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 341f8d37ae52dd08f16ba0db17f49cc795b0f502
+  data.tar.gz: c6585473340abe1d26ef115518c951749b079bc8
+SHA512:
+  metadata.gz: 1a0e68f65673ef2b6feb271a02161320c747b9b419c3f130c5449a9f3bb7ee54f5b81bc1de4af52fbe9b27f5d9c422f7507d6403c9b0c33c53fe2909164004e4
+  data.tar.gz: f7fbabfe99aa117d1c70001286420e674849709ea255248dee67cb3f4fad5a7e95c563a781b60a0a26927ff36903807446baccc301dbaecc915b414db9adc860

data/.gitignore

@@ -0,0 +1,5 @@
+
+coverage
+.rspec_status
+logs
+example/data

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--require spec_helper

data/.travis.yml

@@ -0,0 +1,36 @@
+sudo: required
+
+language: ruby
+
+services:
+- docker
+
+os:
+- linux
+
+rvm:
+- 2.4.2
+- 2.4.1
+- 2.3.1
+- 2.1.5
+
+env:
+  - secure: R0rBtWClmnJKO+zyAncF6+1qDpZ6JQR+Z6VXzUmWw8MuxUXv732XTNYulY/tTeDW876GofAsOF9QfX/jhF+2ubV8SzM6rKTZS09feATuOWRPtECEaBOPcb0JGYouBNg/Lv4onuXb8KS/9U093sRqiJMfib82iY7QK9bnDPsYoQXtMnI7KBeQNrTV0H4nby3lvbjcpMDhmyg+anOmCP5yepZeS53Qq+C5GHguYivisvhSPmCfONrT303XxFLDo/aOkMDlceeZBDhtcpm5Px3GIwybSI9xRtAnpT73oL1Piazda6mRN7VHoT7YAHKqcJNZDB48HYuTj3Pt3oVVw0v78Cf25caBqXttXflXfB8U3qfVt+94uF0icH34NAnpW9l1B8GXHi6CTr27Hg2ZatWkQCN1rCpcCaW0eM/NoAG6snwTIwr1VvjLWpSW5wwhQgHPyMl+MRlTmCn/UM+eQSwPVhflKpMS1+ah6cbhDrQQF1jDt3dRaLqSnVE4APrafdT90SA3SCG2MGOE+pp+DX528gEsRB34UdX2V67dlwXeRu1eU5vZBR7jZrqwuwTAgSvxEF0eJWC0kgvEfouzuVHhVmmSAcNNHlg2kd0/oyQELyXTsklJghwAupcy4X/GEAyfLW9GAxItjATyV7XyfjmaBmdzHl/5JWeXicoyZ/4+8iE=
+
+before_install:
+- cd docker && docker build -t processor-rabbitmq .
+- docker run -d -p 5672:5672 -p 15672:15672 --name processor-rabbitmq processor-rabbitmq
+- cd ..
+- curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64
+  > ./cc-test-reporter
+- chmod +x ./cc-test-reporter
+- gem install bundler --version '~> 1.16.0'
+
+before_script:
+- "./cc-test-reporter before-build"
+
+script:
+- bundle exec rspec
+
+after_script:
+- "./cc-test-reporter after-build --exit-code $TRAVIS_TEST_RESULT"

data/Gemfile

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+
+git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in eventhub-processor2.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,76 @@
+PATH
+  remote: .
+  specs:
+    eventhub-processor2 (1.0.0)
+      bunny (~> 2.7)
+      celluloid (~> 0.17)
+      eventhub-components (~> 0.2)
+      uuidtools (~> 2.1)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    amq-protocol (2.2.0)
+    bunny (2.7.2)
+      amq-protocol (>= 2.2.0)
+    celluloid (0.17.3)
+      celluloid-essentials
+      celluloid-extras
+      celluloid-fsm
+      celluloid-pool
+      celluloid-supervision
+      timers (>= 4.1.1)
+    celluloid-essentials (0.20.5)
+      timers (>= 4.1.1)
+    celluloid-extras (0.20.5)
+      timers (>= 4.1.1)
+    celluloid-fsm (0.20.5)
+      timers (>= 4.1.1)
+    celluloid-pool (0.20.5)
+      timers (>= 4.1.1)
+    celluloid-supervision (0.20.6)
+      timers (>= 4.1.1)
+    diff-lcs (1.3)
+    docile (1.1.5)
+    eventhub-components (0.2.0)
+      logstash-logger (~> 0.25)
+    hitimes (1.2.6)
+    json (2.1.0)
+    logstash-event (1.2.02)
+    logstash-logger (0.25.1)
+      logstash-event (~> 1.2)
+    rake (12.3.0)
+    rspec (3.7.0)
+      rspec-core (~> 3.7.0)
+      rspec-expectations (~> 3.7.0)
+      rspec-mocks (~> 3.7.0)
+    rspec-core (3.7.0)
+      rspec-support (~> 3.7.0)
+    rspec-expectations (3.7.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.7.0)
+    rspec-mocks (3.7.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.7.0)
+    rspec-support (3.7.0)
+    simplecov (0.15.1)
+      docile (~> 1.1.0)
+      json (>= 1.8, < 3)
+      simplecov-html (~> 0.10.0)
+    simplecov-html (0.10.2)
+    timers (4.1.2)
+      hitimes
+    uuidtools (2.1.5)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.16)
+  eventhub-processor2!
+  rake (~> 12.2)
+  rspec (~> 3.7)
+  simplecov (~> 0.15)
+
+BUNDLED WITH
+   1.16.0

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2017 thomis
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,38 @@
+[![Dependency Status](https://gemnasium.com/badges/github.com/thomis/eventhub-processor2.svg)](https://gemnasium.com/github.com/thomis/eventhub-processor2)
+[![Maintainability](https://api.codeclimate.com/v1/badges/9112358562f0614e0e02/maintainability)](https://codeclimate.com/github/thomis/eventhub-processor2/maintainability)
+[![Test Coverage](https://api.codeclimate.com/v1/badges/9112358562f0614e0e02/test_coverage)](https://codeclimate.com/github/thomis/eventhub-processor2/test_coverage)
+[![Build Status](https://travis-ci.org/thomis/eventhub-processor2.svg?branch=master)](https://travis-ci.org/thomis/eventhub-processor2)
+
+# EventHub::Processor2
+
+Next generation gem to build ruby based eventhub processor.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'eventhub-processor2'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install eventhub-processor2
+
+
+## Usage
+
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/thomis/eventhub-processor2.

data/Rakefile

@@ -0,0 +1,6 @@
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'eventhub/processor2'
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require 'irb'
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/docker/Dockerfile

@@ -0,0 +1,6 @@
+FROM rabbitmq:3-management
+ADD definitions.json /etc/rabbitmq/
+ADD rabbitmq.config /etc/rabbitmq/
+RUN chown rabbitmq:rabbitmq /etc/rabbitmq/rabbitmq.config /etc/rabbitmq/definitions.json
+
+EXPOSE 5672 15672

data/docker/README.md

@@ -0,0 +1,14 @@
+## Docker Container
+
+eventhub-processor2 interacts with RabbitMQ. For processor development and rspec test's is required to have a running RabbitMQ environment.
+Please follow these steps to build and run a docker container with a predefined virtual host, exchanges, and queues.
+
+* Have latest docker community edition installed (https://www.docker.com)
+* cd into the docker folder
+* docker build -t processor-rabbitmq .
+* docker run -d -p 5672:5672 -p 15672:15672 --name processor-rabbitmq processor-rabbitmq
+
+Is the docker container running fine?
+Go to http://localhost:15672 and login to RabbitMQ Management Console with guest/guest
+
+Additional information for RabbitMQ docker image: https://hub.docker.com/r/_/rabbitmq/

data/docker/definitions.json

@@ -0,0 +1,87 @@
+{
+  "rabbit_version": "3.6.14",
+  "users": [
+    {
+      "name": "guest",
+      "password_hash": "Xvy2EUToV+QoQDNOypgOwwnnjrN2PYSR2s29TAy0BrSsP1i+",
+      "hashing_algorithm": "rabbit_password_hashing_sha256",
+      "tags": "administrator"
+    }
+  ],
+  "vhosts": [
+    {
+      "name": "event_hub"
+    }
+  ],
+  "permissions": [
+    {
+      "user": "guest",
+      "vhost": "event_hub",
+      "configure": ".*",
+      "write": ".*",
+      "read": ".*"
+    }
+  ],
+  "parameters": [],
+  "global_parameters": [
+    {
+      "name": "cluster_name",
+      "value": "rabbit@54d839ff16c7"
+    }
+  ],
+  "policies": [],
+  "queues": [
+    {
+      "name": "event_hub.inbound",
+      "vhost": "event_hub",
+      "durable": true,
+      "auto_delete": false,
+      "arguments": {}
+    },
+    {
+      "name": "example",
+      "vhost": "event_hub",
+      "durable": true,
+      "auto_delete": false,
+      "arguments": {}
+    }
+  ],
+  "exchanges": [
+    {
+      "name": "event_hub.inbound",
+      "vhost": "event_hub",
+      "type": "direct",
+      "durable": true,
+      "auto_delete": false,
+      "internal": false,
+      "arguments": {}
+    },
+    {
+      "name": "example",
+      "vhost": "event_hub",
+      "type": "direct",
+      "durable": true,
+      "auto_delete": false,
+      "internal": false,
+      "arguments": {}
+    }
+  ],
+  "bindings": [
+    {
+      "source": "event_hub.inbound",
+      "vhost": "event_hub",
+      "destination": "event_hub.inbound",
+      "destination_type": "queue",
+      "routing_key": "",
+      "arguments": {}
+    },
+    {
+      "source": "example",
+      "vhost": "event_hub",
+      "destination": "example",
+      "destination_type": "queue",
+      "routing_key": "",
+      "arguments": {}
+    }
+  ]
+}

data/docker/rabbitmq.config

@@ -0,0 +1,16 @@
+[
+  { rabbit, [
+    { loopback_users, [ ] },
+    { tcp_listeners, [ 5672 ] },
+    { ssl_listeners, [ ] },
+    { default_vhost, <<"event_hub">> },
+    { hipe_compile, false }
+  ] },
+  { rabbitmq_management, [
+    { load_definitions, "/etc/rabbitmq/definitions.json"},
+    { listener, [
+      { port, 15672 },
+      { ssl, false }
+    ]}
+  ]}
+].

data/eventhub-processor2.gemspec

@@ -0,0 +1,34 @@
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'eventhub/version'
+
+note = 'Next generation gem to build ruby based eventhub processor'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'eventhub-processor2'
+  spec.version       = EventHub::VERSION
+  spec.authors       = ['Steiner, Thomas']
+  spec.email         = ['thomas.steiner@ikey.ch']
+
+  spec.summary       = note
+  spec.description   = note
+  spec.homepage      = 'https://github.com/thomis/eventhub-processor2'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'celluloid', '~> 0.17'
+  spec.add_dependency 'bunny', '~> 2.7'
+  spec.add_dependency 'eventhub-components', '~> 0.2'
+  spec.add_dependency 'uuidtools', '~> 2.1'
+
+  spec.add_development_dependency 'bundler', '~> 1.16'
+  spec.add_development_dependency 'rake', '~> 12.2'
+  spec.add_development_dependency 'rspec', '~> 3.7'
+  spec.add_development_dependency 'simplecov', '~> 0.15'
+end

data/example/README.md

@@ -0,0 +1,19 @@
+## Example Application
+
+### Components
+* example.rb - is listening to the example queue, reads id from message and deletes file with id in data folder
+* publisher.rb - creates a file with name id.json (id=guid) and publishes a json messages with id as content to the example exchange
+* crasher.rb - randomly restarts example processes or docker container (processor-rabbitmq)
+
+### How to use
+* Make sure docker container (process-rabbitmq) is running
+* Start 1 or more example processes (I did 3)
+* Start 1 or more publisher processes ( I did 1)
+* Start one crasher.rb
+
+### What is the goal
+* See how the components work under various conditions. Feel free to manually interact (Exp. kill -9 PID)
+* There should be no files left in the data folder when all example messages are consumed
+
+### Note
+It can happen that a file gets deleted before message is acknowledged. This message will be processed again and will just log a warning about missing file. Due to the nature of 2 independent processes it can not garanteed that both process transaction are all done or not done at all, but with message acknowledgement and and publisher confirms we can mitigate risk of lost messages.

data/example/config/example.json

@@ -0,0 +1,19 @@
+{
+  "development": {
+    "server": {
+      "user": "guest",
+      "password": "guest",
+      "host": "localhost",
+      "vhost": "event_hub",
+      "port": 5672,
+      "tls": false
+    },
+    "processor": {
+      "heartbeat_cycle_in_s": 300,
+      "watchdog_cycle_in_s": 15,
+      "listener_queues": [
+        "example"
+      ]
+    }
+  }
+}

data/example/crasher.rb

@@ -0,0 +1,57 @@
+# MyProcess
+class MyProcess
+  attr_reader :id
+  def initialize(id)
+    @id = id
+  end
+
+  def restart
+    puts "Sending Signal HUP to process [#{@id}]"
+    Process.kill('HUP', @id)
+  rescue Errno::ESRCH
+  end
+
+  def self.all
+    processes = []
+    data = `ps | grep example.rb | grep ruby`
+    data.lines[0..-2].each do |line|
+      processes << MyProcess.new(line.split(' ')[0].to_i)
+    end
+
+    puts "Found ids: #{processes.map{ |pr| pr.id}.join(', ')}"
+    processes
+  end
+end
+
+# Docker
+class Docker
+  def initialize(name, time = 10)
+    @name = name
+    @time = time
+  end
+
+  def restart
+    puts "Restart (#{@time}) [#{@name}]"
+    `docker restart -t #{@time} #{@name}`
+  end
+end
+
+def items
+  a = []
+  a << Docker.new('processor-rabbitmq')
+  a << Docker.new('processor-rabbitmq', 0)
+  a << MyProcess.all
+  a.flatten!
+end
+
+run = true
+Signal.trap('INT') { run = false }
+
+while run
+  to_sleep = rand(3600)
+  puts "Waiting [#{to_sleep}]..."
+  sleep to_sleep
+  items.sample.restart
+end
+
+puts 'Done'

data/example/example.rb

@@ -0,0 +1,25 @@
+require_relative '../lib/eventhub'
+
+module EventHub
+  # Demo class
+  class Example < Processor2
+    def handle_message(message, args = {})
+      id = message.body['id']
+      name = "data/#{id}.json"
+
+      begin
+        File.delete(name)
+      rescue => ex
+        EventHub.logger.warn("File [#{name}]: #{ex}")
+      end
+
+      { body: { id: id, message: 'has been done'}}
+    end
+
+    def version
+      '10.1.1'
+    end
+  end
+end
+
+EventHub::Example.new.start

data/example/publisher.rb

@@ -0,0 +1,95 @@
+require 'bunny'
+require 'celluloid/current'
+require 'json'
+require 'securerandom'
+
+Celluloid.logger = nil
+Celluloid.exception_handler { |ex| puts "Exception occured: #{ex}" }
+
+# Publisher
+class Publisher
+  include Celluloid
+
+  def initialize
+    async.start
+  end
+
+  def start
+    connect
+    count = 1
+    loop do
+      do_the_work
+
+      sleep 0.001
+      print '.'
+      puts '' if (count % 80).zero?
+      count += 1
+    end
+  ensure
+    @connection.close if @connection
+  end
+
+  private
+
+  def connect
+    @connection = Bunny.new(vhost: 'event_hub',
+                            automatic_recovery: false,
+                            logger: Logger.new('/dev/null'))
+    @connection.start
+    @channel = @connection.create_channel
+    @channel.confirm_select
+    @exchange = @channel.direct('example', durable: true)
+  end
+
+  def do_the_work
+    id = SecureRandom.uuid
+    data = { body: { id: id } }.to_json
+
+    file = File.open("data/#{id}.json", 'w')
+    file.write(data)
+    file.close
+
+    @exchange.publish(data, persistent: true)
+    success = @channel.wait_for_confirms
+
+    raise 'Published message not confirmed' unless success
+  end
+end
+
+# Application
+class Application
+  def initialize
+    @run = true
+    @config = Celluloid::Supervision::Configuration.define(
+      [
+        { type: Publisher, as: :publisher }
+      ]
+    )
+
+    @config.injection!(:before_restart, proc do
+      puts 'Restarting in 5 seconds...'
+      sleep 5
+    end)
+  end
+
+  def start
+    puts 'Publisher has been started'
+    @config.deploy
+    main_event_loop
+    cleanup
+    puts 'Publisher has been stopped'
+  end
+
+  private
+
+  def main_event_loop
+    Signal.trap(:INT) { @run = false }
+    sleep 0.5 while @run
+  end
+
+  def cleanup
+    Celluloid.shutdown
+  end
+end
+
+Application.new.start