rflow-components-amqp 0.0.2

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*

data/Gemfile

@@ -0,0 +1,3 @@
+source "http://rubygems.org"
+
+gemspec

data/Rakefile

@@ -0,0 +1,15 @@
+require 'bundler'
+require 'rspec/core/rake_task'
+require 'rdoc/task'
+Bundler::GemHelper.install_tasks
+
+RSpec::Core::RakeTask.new(:spec) do |t|
+  t.verbose = true
+  t.rspec_opts = '--tty --color'
+end
+
+Rake::RDocTask.new do |rd|
+  rd.main = "README"
+  rd.rdoc_files.include("README", "lib/**/*.rb")
+  rd.rdoc_dir = File.join('doc', 'html')
+end

data/lib/rflow/components/amqp/extensions.rb

@@ -0,0 +1,27 @@
+class RFlow
+  module Components
+    module AMQP
+
+      # The set of extensions to add capability to AMQP data types
+      module Extensions
+
+        module AMQPMessageExtension
+          def self.extended(base_data)
+            base_data.data_object ||= {'header' => {}, 'payload' => ''}
+          end
+
+          # Default accessors
+           ['header', 'payload'].each do |name|
+             define_method name do |*args|
+               data_object[name]
+             end
+             define_method :"#{name}=" do |*args|
+               data_object[name] = args.first
+             end
+           end
+        end
+
+      end
+    end
+  end
+end

data/lib/rflow/components/amqp/subscriber.rb

@@ -0,0 +1,113 @@
+require 'eventmachine'
+require 'amqp'
+require 'rflow'
+
+class RFlow
+  module Components
+    module AMQP
+
+      # By default will use and exclusive, non-durable, auto-deleting,
+      # randomly named queue for subscribing to topic messages with an
+      # empty-string pattern
+      class Subscriber < RFlow::Component
+        output_port :amqp_port
+        output_port :raw_port
+
+        attr_accessor :config, :queue_config
+
+        DEFAULT_CONFIG =       {
+          'server' => '127.0.0.1',
+          'port' => 5672,
+          'username' => 'guest',
+          'password' => 'guest',
+          'vhost' => '/',
+
+          'queue_name' => 'asdf',
+          'queue_passive' => false,
+          'queue_durable' => false,
+          'queue_exclusive' => true,
+          'queue_auto_delete' => true,
+
+          'binding_pattern' => '',
+        }
+
+        def configure!(config)
+          @config = DEFAULT_CONFIG.merge config
+          @config['port'] = @config['port'].to_i
+
+          ['durable', 'passive', 'exclusive', 'auto_delete'].each do |queue_bool_opt|
+            @config["queue_#{queue_bool_opt}"] = to_boolean(@config["queue_#{queue_bool_opt}"])
+          end
+
+          # Convert the queue parameters into AMQP-friendly sym-keyed
+          # Hash that can be passed directly to underlying AMQP gem
+          # methods
+          @queue_config = {}
+          @config.each do |key, value|
+            md = /queue_(.*)/.match(key.to_s)
+            unless md.nil?
+              @queue_config[md[1].to_sym] = value
+            end
+          end
+        end
+
+
+        def run!
+          ::AMQP.connect(:host => @config['server'], :port => @config['port'], :vhost => @config['vhost'], :username => @config['username'], :password => @config['password']) do |conn|
+            @amqp_connection = conn
+
+            ::AMQP::Channel.new(@amqp_connection) do |channel|
+              @amqp_channel = channel
+              @amqp_exchange = @amqp_channel.topic
+
+              ::AMQP::Queue.new(@amqp_channel, @config['queue_name'], @queue_config) do |queue|
+                @amqp_queue = queue
+                @amqp_queue.bind(@amqp_exchange, :routing_key => @config['binding_pattern']).subscribe(:ack => true) do |header, payload|
+                  RFlow.logger.debug "AMQP message received"
+                  processing_event = RFlow::Message::ProcessingEvent.new(instance_uuid, Time.now.utc)
+
+                  amqp_message = RFlow::Message.new('RFlow::Message::Data::AMQP::Message')
+                  header.to_hash.each {|k,v| amqp_message.data.header[k.to_s] = v}
+                  amqp_message.data.payload = payload
+
+                  # TODO: Optimize out if not connected
+                  raw_message = RFlow::Message.new('RFlow::Message::Data::Raw')
+                  raw_message.data.raw = payload
+
+                  processing_event.completed_at = Time.now.utc
+                  amqp_message.provenance << processing_event
+                  raw_message.provenance << processing_event
+
+                  amqp_port.send_message amqp_message
+                  raw_port.send_message raw_message
+
+                  header.ack
+                end
+              end
+
+            end
+          end
+
+#          EM.add_timer(2) do
+#            EM.add_periodic_timer(0) do
+#              @amqp_exchange.publish Array.new(rand(1000)) { rand(256) }.pack('c*'), :routing_key => ""
+#            end
+#          end
+        end
+
+        def to_boolean(string)
+          case string
+          when /^true$/i, '1', true
+            true
+          when /^false/i, '0', false
+            false
+          else
+            raise ArgumentError, "'#{string}' cannot be coerced to a boolean value"
+          end
+        end
+
+      end
+
+    end
+  end
+end

data/lib/rflow/components/amqp/version.rb

@@ -0,0 +1,7 @@
+class RFlow
+  module Components
+    module AMQP
+      VERSION = "0.0.2"
+    end
+  end
+end

data/lib/rflow/components/amqp.rb

@@ -0,0 +1,25 @@
+require 'rflow/components/amqp/extensions'
+require 'rflow/components/amqp/subscriber'
+
+class RFlow
+  module Components
+    module AMQP
+      # Load the schemas
+      SCHEMA_DIRECTORY = ::File.expand_path(::File.join(::File.dirname(__FILE__), '..', '..', '..', 'schema'))
+
+      SCHEMA_FILES = {
+        'amqp_message.avsc'  => 'RFlow::Message::Data::AMQP::Message',
+      }
+
+      SCHEMA_FILES.each do |file_name, data_type_name|
+        schema_string = ::File.read(::File.join(SCHEMA_DIRECTORY, file_name))
+        RFlow::Configuration.add_available_data_type data_type_name, 'avro', schema_string
+      end
+
+      # Load the data extensions
+      RFlow::Configuration.add_available_data_extension('RFlow::Message::Data::AMQP::Message',
+                                                        RFlow::Components::AMQP::Extensions::AMQPMessageExtension)
+
+    end
+  end
+end

data/lib/rflow-components-amqp.rb

@@ -0,0 +1,2 @@
+require 'rflow'
+require 'rflow/components/amqp'

data/rflow-components-amqp.gemspec

@@ -0,0 +1,28 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "rflow/components/amqp/version"
+
+Gem::Specification.new do |s|
+  s.name        = "rflow-components-amqp"
+  s.version     = RFlow::Components::AMQP::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.required_ruby_version = '~> 1.9'
+  s.authors     = ["Michael L. Artz"]
+  s.email       = ["michael.artz@redjack.com"]
+  s.homepage    = "https://github.com/redjack/rflow-components-amqp"
+  s.summary     = %q{AMQP publisher and subscriber component for the RFlow FBP framework}
+  s.description = %q{AMQP publisher and subscriber component for the RFlow FBP framework.  Also includes the necessary AMQP::Message message type}
+
+  s.rubyforge_project = "rflow-components-amqp"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_dependency 'rflow', '~> 0.0.1'
+  s.add_dependency 'amqp', '>= 0.8.0.rc12'
+
+  s.add_development_dependency 'rspec', '~> 2.5.0'
+  s.add_development_dependency 'rake', '~> 0.8.7'
+end

data/schema/amqp_message.avsc

@@ -0,0 +1,15 @@
+{
+    "type": "record",
+    "name": "Message",
+    "namespace": "org.rflow.message.data.amqp",
+    "aliases": [],
+    "fields": [
+        {"name": "client_ip",    "type": ["string", "null"]},
+        {"name": "client_port",  "type": ["int", "null"]},
+        {"name": "server_ip",    "type": ["string", "null"]},
+        {"name": "server_port",  "type": ["int", "null"]},
+
+        {"name": "header",      "type": {"type": "map", "values": ["string", "int", "null"]}},
+        {"name": "payload",     "type": "bytes"}
+    ]
+}

data/spec/extensions_spec.rb

@@ -0,0 +1,47 @@
+require 'spec_helper.rb'
+
+# describe RFlow::Components::File::Extensions::FileExtension do
+#   before(:each) do
+#     @schema_string = RFlow::Configuration.available_data_types['RFlow::Message::Data::File']['avro']
+#   end
+#
+#   it "should add the extension to RFlow::Configuration" do
+#     RFlow::Configuration.available_data_extensions['RFlow::Message::Data::File'].should include(described_class)
+#   end
+#
+#   it "should set the defaults" do
+#     file = RFlow::Message.new('RFlow::Message::Data::File')
+#
+#     file.data.path.should == '/'
+#     file.data.size.should == 0
+#     file.data.content.should == ''
+#     file.data.creation_timestamp.should == nil
+#     file.data.modification_timestamp.should == nil
+#     file.data.accessed_timestamp.should == nil
+#   end
+#
+#   it "should correctly use integers or strings for size field" do
+#     file = RFlow::Message.new('RFlow::Message::Data::File')
+#
+#     file.data.size.should == 0
+#     file.data.size = 10
+#     file.data.size.should == 10
+#     file.data.size = '20'
+#     file.data.size == 20
+#   end
+#
+#   it "should correctly use Time or xmlschema strings for timestamp fields" do
+#     file = RFlow::Message.new('RFlow::Message::Data::File')
+#
+#     file.data.creation_timestamp.should == nil
+#     now = Time.now
+#
+#     file.data.creation_timestamp = now
+#     file.data.creation_timestamp.should == Time.xmlschema(now.xmlschema(9))
+#
+#     file.data.creation_timestamp = now.xmlschema
+#     file.data.creation_timestamp.should == Time.xmlschema(now.xmlschema)
+#   end
+#
+#
+# end

data/spec/schema_spec.rb

@@ -0,0 +1,29 @@
+require 'spec_helper.rb'
+
+describe 'RFlow::Message::Data::AMQP::Message Avro Schema' do
+  before(:each) do
+    @schema_string = RFlow::Configuration.available_data_types['RFlow::Message::Data::AMQP::Message']['avro']
+  end
+
+  it "should encode and decode an object" do
+    amqp_message = {
+      'header'  => {
+        'content_type'  => 'application/octet-stream',
+        'delivery_mode' => 2,
+        'priority'      => 0,
+      },
+      'payload' =>'PAYLOAD',
+    }
+
+    expect {encode_avro(@schema_string, amqp_message)}.to_not raise_error
+    avro_encoded_amqp_message = encode_avro(@schema_string, amqp_message)
+
+    expect {decode_avro(@schema_string, avro_encoded_amqp_message)}.to_not raise_error
+    decoded_amqp_message = decode_avro(@schema_string, avro_encoded_amqp_message)
+
+    decoded_amqp_message['payload'].should == amqp_message['payload']
+    decoded_amqp_message['header'].should == amqp_message['header']
+
+  end
+end
+

data/spec/spec_helper.rb

@@ -0,0 +1,21 @@
+require File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib', 'rflow-components-amqp'))
+
+require 'logger'
+
+RFlow.logger = Logger.new STDOUT
+
+def decode_avro(schema_string, serialized_object)
+  schema = Avro::Schema.parse(schema_string)
+  serialized_object.force_encoding 'BINARY'
+  sio = StringIO.new(serialized_object)
+  Avro::IO::DatumReader.new(schema, schema).read Avro::IO::BinaryDecoder.new(sio)
+end
+
+def encode_avro(schema_string, object)
+  encoded_string = ''
+  encoded_string.force_encoding 'BINARY'
+  schema = Avro::Schema.parse(schema_string)
+  sio = StringIO.new(encoded_string)
+  Avro::IO::DatumWriter.new(schema).write object, Avro::IO::BinaryEncoder.new(sio)
+  encoded_string
+end

metadata

@@ -0,0 +1,126 @@
+--- !ruby/object:Gem::Specification
+name: rflow-components-amqp
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+  prerelease: 
+platform: ruby
+authors:
+- Michael L. Artz
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-03-17 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rflow
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.0.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.0.1
+- !ruby/object:Gem::Dependency
+  name: amqp
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.8.0.rc12
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.8.0.rc12
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.5.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.5.0
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.8.7
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.8.7
+description: AMQP publisher and subscriber component for the RFlow FBP framework.  Also
+  includes the necessary AMQP::Message message type
+email:
+- michael.artz@redjack.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- Rakefile
+- lib/rflow-components-amqp.rb
+- lib/rflow/components/amqp.rb
+- lib/rflow/components/amqp/extensions.rb
+- lib/rflow/components/amqp/subscriber.rb
+- lib/rflow/components/amqp/version.rb
+- rflow-components-amqp.gemspec
+- schema/amqp_message.avsc
+- spec/extensions_spec.rb
+- spec/schema_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/redjack/rflow-components-amqp
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ~>
+    - !ruby/object:Gem::Version
+      version: '1.9'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: rflow-components-amqp
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: AMQP publisher and subscriber component for the RFlow FBP framework
+test_files:
+- spec/extensions_spec.rb
+- spec/schema_spec.rb
+- spec/spec_helper.rb