rflow 0.0.5

data/lib/rflow/configuration/uuid_keyed.rb

@@ -0,0 +1,18 @@
+require 'uuidtools'
+
+class RFlow
+  class Configuration
+    module UUIDKeyed
+      def self.included(base)
+        base.class_eval do
+          set_primary_key 'uuid'
+          before_create :generate_uuid
+          
+          def generate_uuid
+            self.uuid = UUIDTools::UUID.random_create.to_s
+          end 
+        end
+      end
+    end
+  end
+end

data/lib/rflow/connection.rb

@@ -0,0 +1,59 @@
+require 'rflow/message'
+
+class RFlow
+  class Connection
+    attr_accessor :instance_uuid, :name, :configuration, :recv_callback
+
+    # Attribute that holds the 
+    attr_accessor :recv_callback
+    
+    def initialize(connection_instance_uuid, connection_name=nil, connection_configuration={})
+      @instance_uuid = connection_instance_uuid
+      @name = connection_name
+      @configuration = connection_configuration
+    end
+
+
+    # Subclass and implement to be able to handle future 'recv'
+    # methods.  Will only be called in the context of a running
+    # EventMachine reactor
+    def connect_input!
+      raise NotImplementedError, "Raw connections do not support connect_input.  Please subclass and define a connect_output method."
+    end
+
+
+    # Subclass and implement to be able to handle future 'send'
+    # methods.  Will only be called in the context of a running
+    # EventMachine reactor
+    def connect_output!
+      raise NotImplementedError, "Raw connections do not support connect_output.  Please subclass and define a connect_output method."
+    end
+
+    
+    # Subclass and implement to handle outgoing messages.  The message
+    # will be a RFlow::Message object and the subclasses are expected
+    # to marshal it up into something that will be unmarshalled on the
+    # other side
+    def send_message(message)
+      raise NotImplementedError, "Raw connections do not support send_message.  Please subclass and define a send_message method."
+    end
+
+    # Parent component will set this attribute if it expects to
+    # recieve messages.  Connection subclass should call it
+    # (recv_callback.call(message)) when it gets a new message, which
+    # will be transmitted back to the parent component's
+    # process_message method.  Sublcass is responsible for
+    # deserializing whatever was on the wire into a RFlow::Message object
+    def recv_callback
+      @recv_callback ||= Proc.new {|message|}
+    end
+    
+  end # class Connection
+
+  class Disconnection < Connection
+    def send_message(message)
+      RFlow.logger.debug "Attempting to send without a connection, doing nothing"
+    end
+  end
+  
+end # class RFlow

data/lib/rflow/connections.rb

@@ -0,0 +1,2 @@
+# Load the core connection types
+require 'rflow/connections/zmq_connection'

data/lib/rflow/connections/zmq_connection.rb

@@ -0,0 +1,101 @@
+#require 'ffi'
+#require 'ffi-rzmq'
+require 'em-zeromq-mri'
+
+require 'rflow/connection'
+require 'rflow/message'
+
+class RFlow
+  module Connections
+    class ZMQConnection < RFlow::Connection
+
+      class << self
+        attr_accessor :zmq_context
+
+        def create_zmq_context
+          RFlow.logger.debug "Creating a new ZeroMQ context"
+          unless EM.reactor_running?
+            raise RuntimeError, "EventMachine reactor is not running when attempting to create a ZeroMQ context" 
+          end
+          EM::ZeroMQ::Context.new(1)
+        end
+        
+        # Returns the current ZeroMQ context object or creates it if
+        # it does not exist.  Assumes that we are within a running
+        # EventMachine reactor
+        def zmq_context
+          @zmq_context ||= create_zmq_context
+        end
+      end
+
+      attr_accessor :socket
+
+      REQUIRED_OPTION_SUFFIXES = ['_socket_type', '_address', '_responsibility']
+
+      def self.configuration_errors(configuration)
+        # TODO: Normalize/validate configuration
+        missing_config_elements = []
+
+        ['input', 'output'].each do |direction_prefix|
+          REQUIRED_OPTION_SUFFIXES.each do |option_suffix|
+            config_element = "#{direction_prefix}#{option_suffix}"
+            unless configuration.include? config_element
+              missing_config_elements << config_element
+            end
+          end
+        end
+
+        missing_config_elements
+      end
+
+      
+      def initialize(connection_instance_uuid, connection_name, connection_configuration)
+        configuration_errors = self.class.configuration_errors(connection_configuration)
+        unless configuration_errors.empty?
+          raise ArgumentError, "#{self.class.to_s}: configuration missing elements: #{configuration_errors.join ', '}"
+        end
+
+        super
+      end
+
+      
+      def connect_input!
+        RFlow.logger.debug "Connecting input #{instance_uuid} with #{configuration.find_all {|k, v| k.to_s =~ /input/}}"
+        self.socket = self.class.zmq_context.send(configuration['input_responsibility'],
+                                                  ZMQ.const_get(configuration['input_socket_type'].to_sym),
+                                                  configuration['input_address'],
+                                                  self)
+      end
+
+
+      def connect_output!
+        RFlow.logger.debug "Connecting output #{instance_uuid} with #{configuration.find_all {|k, v| k.to_s =~ /output/}}"
+        self.socket = self.class.zmq_context.send(configuration['output_responsibility'].to_s,
+                                                  ZMQ.const_get(configuration['output_socket_type'].to_sym),
+                                                  configuration['output_address'].to_s,
+                                                  self)
+      end
+
+
+      def on_readable(socket, message_parts)
+        message = RFlow::Message.from_avro(message_parts.last.copy_out_string)
+        RFlow.logger.debug "#{name}: Received message of type '#{message_parts.first.copy_out_string}'"
+        recv_callback.call(message)
+      end
+
+      # TODO: fix this tight loop of retries
+      def send_message(message)
+        RFlow.logger.debug "#{name}: Sending message of type '#{message.data_type_name.to_s}'"
+
+        begin
+          socket.send_msg(message.data_type_name.to_s, message.to_avro)
+          RFlow.logger.debug "#{name}: Successfully sent message of type '#{message.data_type_name.to_s}'"
+        rescue Exception => e
+          RFlow.logger.debug "Exception #{e.class}: #{e.message}, retrying send"
+          retry
+        end
+      end
+      
+    end
+  end
+end

data/lib/rflow/message.rb

@@ -0,0 +1,191 @@
+require 'stringio'
+require 'time'
+
+require 'avro'
+
+require 'rflow/configuration'
+
+class RFlow
+
+  # TODO: reduce reliance/expectation on avro serialization in method
+  # names and such.
+  class Message
+
+    class << self
+      def avro_message_schema; @avro_message_schema ||= Avro::Schema.parse(File.read(File.join(File.dirname(__FILE__), '..', '..', 'schema', 'message.avsc'))); end
+      
+      def avro_reader;  @avro_reader  ||= Avro::IO::DatumReader.new(avro_message_schema, avro_message_schema); end
+      def avro_writer;  @avro_writer  ||= Avro::IO::DatumWriter.new(avro_message_schema); end
+      def avro_decoder(io_object); Avro::IO::BinaryDecoder.new(io_object); end
+      def avro_encoder(io_object); Avro::IO::BinaryEncoder.new(io_object); end
+
+      # Take in an Avro serialization of a message and return a new
+      # Message object.  Assumes the org.rflow.Message Avro schema.
+      def from_avro(avro_serialized_message_byte_string)
+        avro_serialized_message_byte_stringio = StringIO.new(avro_serialized_message_byte_string)
+        message_hash = avro_reader.read avro_decoder(avro_serialized_message_byte_stringio)
+        Message.new(message_hash['data_type_name'], message_hash['provenance'],
+                    message_hash['data_serialization_type'], message_hash['data_schema'],
+                    message_hash['data'])
+      end
+    end
+    
+    
+    # Serialize the current message object to Avro using the
+    # org.rflow.Message Avro schema.  Note that we have to manually
+    # set the encoding for Ruby 1.9, otherwise the stringio would use
+    # UTF-8 by default, which would not work correctly, as a serialize
+    # avro string is BINARY, not UTF-8
+    def to_avro
+      avro_serialized_message_bytes = ''
+      avro_serialized_message_bytes.force_encoding 'BINARY'
+      avro_serialized_message_bytes_stringio = StringIO.new(avro_serialized_message_bytes, 'w')
+
+      deserialized_avro_object = {
+        'data_type_name' => self.data_type_name.to_s,
+        'provenance' => self.provenance.map(&:to_hash),
+        'data_serialization_type' => self.data.serialization_type.to_s,
+        'data_schema' => self.data.schema_string,
+        'data' => self.data.to_avro
+      }
+
+      self.class.avro_writer.write deserialized_avro_object, self.class.avro_encoder(avro_serialized_message_bytes_stringio)
+      avro_serialized_message_bytes
+    end
+    
+
+    attr_reader :data_type_name
+    attr_accessor :processing_event
+    attr_accessor :provenance
+    attr_reader :data, :data_extensions
+
+    
+    def initialize(data_type_name, provenance=[], data_serialization_type='avro', data_schema_string=nil, serialized_data_object=nil)
+      # Default the values, in case someone puts in a nil instead of
+      # the default
+      @data_type_name = data_type_name.to_s
+
+      # Turn the provenance array of Hashes into an array of
+      # ProcessingEvents for easier access and time validation.  TODO:
+      # do this lazily so as not to create/destroy objects that are
+      # never used
+      @provenance = (provenance || []).map do |processing_event_hash_or_object|
+        if processing_event_hash_or_object.is_a? ProcessingEvent
+          processing_event_hash_or_object
+        else
+          ProcessingEvent.new(processing_event_hash_or_object['component_instance_uuid'],
+                              processing_event_hash_or_object['started_at'],
+                              processing_event_hash_or_object['completed_at'],
+                              processing_event_hash_or_object['context'])
+        end
+      end
+      
+      # TODO: Make this better.  This check is technically
+      # unnecessary, as we are able to completely desrialize the
+      # message without needing to resort to the registered schema.
+      registered_data_schema_string = RFlow::Configuration.available_data_types[@data_type_name][data_serialization_type.to_s]
+      unless registered_data_schema_string
+        error_message = "Data type '#{@data_type_name}' with serialization_type '#{data_serialization_type}' not found"
+        RFlow.logger.error error_message
+        raise ArgumentError, error_message
+      end
+
+      # TODO: think about registering the schemas automatically if not
+      # found in Configuration
+      if data_schema_string && (registered_data_schema_string != data_schema_string)
+        error_message = "Passed schema ('#{data_schema_string}') does not equal registered schema ('#{registered_data_schema_string}') for data type '#{@data_type_name}' with serialization_type '#{data_serialization_type}'"
+        RFlow.logger.error error_message
+        raise ArgumentError, error_message
+      end
+      
+      @data = Data.new(registered_data_schema_string, data_serialization_type.to_s, serialized_data_object)
+
+      # Get the extensions and apply them to the data object to add capability
+      @data_extensions = RFlow::Configuration.available_data_extensions[@data_type_name]
+      @data_extensions.each do |data_extension|
+        RFlow.logger.debug "Extending '#{data_type_name}' with extension '#{data_extension}'"
+        @data.extend data_extension
+      end
+    end
+
+
+    class ProcessingEvent
+      attr_accessor :component_instance_uuid, :started_at, :completed_at, :context
+
+      def initialize(component_instance_uuid, started_at=nil, completed_at=nil, context=nil)
+        @component_instance_uuid = component_instance_uuid
+        @started_at = case started_at
+                      when String then Time.xmlschema(started_at) 
+                      when Time   then started_at
+                      else; nil; end
+        @completed_at = case completed_at
+                        when String then Time.xmlschema(completed_at) 
+                        when Time   then completed_at
+                        else; nil; end
+        @context = context
+      end
+      
+      def to_hash
+        {
+          'component_instance_uuid' => component_instance_uuid.to_s,
+          'started_at'   => started_at   ? started_at.xmlschema(6)   : nil,
+          'completed_at' => completed_at ? completed_at.xmlschema(6) : nil,
+          'context'      => context      ? context.to_s              : nil,
+        }
+      end
+    end
+    
+    # Should proxy most methods to data_object that we can serialize
+    # to avro using the schema.  Extensions should use 'extended' hook
+    # to apply immediate changes.
+    class Data
+      attr_reader :schema_string, :schema, :serialization_type
+      attr_accessor :data_object
+
+      def initialize(schema_string, serialization_type='avro', serialized_data_object=nil)
+        unless serialization_type == 'avro'
+          error_message = "Only Avro serialization_type supported at the moment"
+          RFlow.logger.error error_message
+          raise ArgumentError, error_message
+        end
+
+        @schema_string = schema_string
+        @serialization_type = serialization_type
+
+        begin
+          @schema = Avro::Schema.parse(schema_string)
+        rescue Exception => e
+          error_message = "Invalid schema '#{@schema_string}': #{e}: #{e.message}"
+          RFlow.logger.error error_message
+          raise ArgumentError, error_message
+        end
+        
+        if serialized_data_object
+          serialized_data_object.force_encoding 'BINARY'
+          avro_decoder = Avro::IO::BinaryDecoder.new StringIO.new(serialized_data_object)
+          @data_object = Avro::IO::DatumReader.new(schema, schema).read avro_decoder
+        end
+      end
+
+      def valid?
+        Avro::Schema.validate @schema, @data_object
+      end
+      
+      def to_avro
+        serialized_data_object_bytes = ''
+        serialized_data_object_bytes.force_encoding 'BINARY'
+        serialized_data_object_bytes_stringio = StringIO.new(serialized_data_object_bytes)
+        Avro::IO::DatumWriter.new(@schema).write @data_object, Avro::IO::BinaryEncoder.new(serialized_data_object_bytes_stringio)
+        serialized_data_object_bytes
+      end
+
+      # Proxy methods down to the underlying data_object, probably a
+      # Hash.  Hopefully an extension will provide any additional
+      # functionality so this won't be called unless needed
+      def method_missing(method_sym, *args, &block)
+        @data_object.send(method_sym, *args, &block)
+      end
+    end
+      
+  end
+end

data/lib/rflow/port.rb

@@ -0,0 +1,4 @@
+class RFlow
+  class Port
+  end # class Port
+end # class RFlow

data/lib/rflow/util.rb

@@ -0,0 +1,19 @@
+require 'uuidtools'
+
+class RFlow
+  module Util
+    # Generate a UUID based on either the SHA1 of a seed string (v5) with a
+    # 'zero' UUID namespace, or using a purely random generation
+    # (v4) if no seed string is present
+    def generate_uuid_string(seed=nil)
+      uuid = if seed
+               UUIDTools::UUID.sha1_create(UUIDTools::UUID.parse_int(0), seed)
+             else
+               UUIDTools::UUID.random_create
+             end
+      uuid.to_s
+    end
+
+
+  end
+end

data/lib/rflow/version.rb

@@ -0,0 +1,3 @@
+class RFlow
+  VERSION = "0.0.5"
+end # class RFlow

data/rflow.gemspec

@@ -0,0 +1,42 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "rflow/version"
+
+Gem::Specification.new do |s|
+  s.name        = "rflow"
+  s.version     = RFlow::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.required_ruby_version = '~> 1.9'
+  s.authors     = ["Michael L. Artz"]
+  s.email       = ["michael.artz@redjack.com"]
+  s.homepage    = ""
+  s.summary     = %q{A Ruby-based workflow framework}
+  s.description = %q{A Ruby-based workflow framework that utilizes ZeroMQ for component connections and Avro for serialization}
+
+  s.rubyforge_project = "rflow"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_dependency 'uuidtools', '~> 2.1'
+  s.add_dependency 'log4r', '~> 1.1'
+  
+  s.add_dependency 'sqlite3', '~> 1.3'
+  s.add_dependency 'activerecord', '~> 3.0'
+    
+  s.add_dependency 'avro', '>= 1.5.1'
+  s.add_dependency 'ffi', '~> 1.0'
+  s.add_dependency 'ffi-rzmq' , '~> 0.8'
+
+  s.add_dependency 'eventmachine', '>= 1.0.0.beta3'
+  # MRI-only because of the FFI memory leak.  TODO: remove when ffi fixed
+  s.add_dependency 'em-zeromq-mri', '~> 0.2'
+  # Remove this when we break it out into its own gem
+  s.add_dependency 'eventmachine_httpserver', '~> 0.2'
+  
+  s.add_development_dependency 'rspec', '~> 2.6'
+  s.add_development_dependency 'rake', '>= 0.8.7'
+  #s.add_development_dependency 'rcov', '= 0.9.9' # Not 1.9.2 compatible
+end