rflow 1.0.0a1 → 1.0.0a2

data/spec/rflow_message_data_raw.rb

@@ -1,26 +0,0 @@
-require 'spec_helper.rb'
-
-require 'rflow/components/raw'
-
-describe 'RFlow::Message::Data::Raw Avro Schema' do
-  before(:each) do
-    @schema_string = RFlow::Configuration.available_data_types['RFlow::Message::Data::Raw']['avro']
-  end
-
-  it "should load the schema" do
-    @schema_string.should_not == nil
-  end
-
-  it "should encode and decode an object" do
-    raw = {'raw' => 'rawdata'}
-
-    expect {encode_avro(@schema_string, raw)}.to_not raise_error
-    avro_encoded_raw = encode_avro(@schema_string, raw)
-
-    expect {decode_avro(@schema_string, avro_encoded_raw)}.to_not raise_error
-    decoded_raw = decode_avro(@schema_string, avro_encoded_raw)
-
-    decoded_raw.should == raw
-  end
-
-end

data/spec/rflow_message_data_spec.rb

@@ -1,60 +0,0 @@
-require 'spec_helper.rb'
-
-require 'rflow/message'
-
-describe RFlow::Message::Data do
-  before(:all) do
-    @string = 'this is a string to be serialized'
-    @invalid_avro_schema_string = 'invalid schema'
-    @valid_avro_string_schema_string = '{"type": "string"}'
-    @avro_serialized_string = encode_avro(@valid_avro_string_schema_string, @string)
-  end
-
-  context "if created without a schema" do
-    it "should throw an exception" do
-      expect {RFlow::Message::Data.new()}.to raise_error(ArgumentError)
-    end
-  end
-
-  context "if created with an invalid schema for the serialization" do
-    it "should throw and exception" do
-      expect {RFlow::Message::Data.new(@invalid_avro_schema_string)}.to raise_error(ArgumentError)
-      expect {RFlow::Message::Data.new(@invalid_avro_schema_string, 'avro')}.to raise_error(ArgumentError)
-      expect {RFlow::Message::Data.new(@invalid_avro_schema_string, 'avro')}.to raise_error(ArgumentError)
-    end
-  end
-
-  context "if created with a valid avro schema and serialization" do
-  end
-
-  context "if created with a valid avro schema" do
-    it "should instantiate correctly" do
-      expect {RFlow::Message::Data.new(@valid_avro_string_schema_string, 'avro')}.to_not raise_error
-      expect {RFlow::Message::Data.new(@valid_avro_string_schema_string, 'avro')}.to_not raise_error
-    end
-
-    context "if created with a non-avro data serialization" do
-      it "should throw an exception" do
-        expect {RFlow::Message::Data.new(@valid_avro_string_schema_string, 'unknown')}.to raise_error(ArgumentError)
-        expect {RFlow::Message::Data.new(@valid_avro_string_schema_string, :unknown)}.to raise_error(ArgumentError)
-        expect {RFlow::Message::Data.new(@valid_avro_string_schema_string, 'xml')}.to raise_error(ArgumentError)
-        expect {RFlow::Message::Data.new(@valid_avro_string_schema_string, :xml)}.to raise_error(ArgumentError)
-      end
-    end
-
-    context "if created with an avro serialization" do
-      it "should instantiate correctly" do
-        expect {RFlow::Message::Data.new(@valid_avro_string_schema_string, 'avro')}.to_not raise_error
-        expect {RFlow::Message::Data.new(@valid_avro_string_schema_string, 'avro')}.to_not raise_error
-      end
-
-      context "if created with a serialized data object" do
-        it "should instantiate correctly" do
-          expect {RFlow::Message::Data.new(@valid_avro_string_schema_string, 'avro', @avro_serialized_string)}.to_not raise_error
-          message = RFlow::Message::Data.new(@valid_avro_string_schema_string, 'avro', @avro_serialized_string)
-          p message
-        end
-      end
-    end
-  end
-end

data/spec/rflow_message_spec.rb

@@ -1,182 +0,0 @@
-require 'spec_helper.rb'
-
-require 'digest/md5'
-
-require 'rflow/message'
-
-describe RFlow::Message do
-
-  context "if created with an unknown data type" do
-    it "should throw an exception" do
-      expect {RFlow::Message.new('non_existant_data_type')}.to raise_error(ArgumentError)
-    end
-  end
-
-
-  context "if created with a known data type" do
-    before(:all) do
-      @avro_string_schema_string = '{"type": "string"}'
-      RFlow::Configuration.add_available_data_type(:string_type, 'avro', @avro_string_schema_string)
-    end
-
-    it "should instantiate correctly" do
-      expect {RFlow::Message.new('string_type')}.to_not raise_error
-    end
-
-    context "if created with empty provenance" do
-      context "if created with an unknown data serialization" do
-        it "should throw an exception" do
-          expect {RFlow::Message.new('string_type', [], 'unknown')}.to raise_error(ArgumentError)
-          expect {RFlow::Message.new('string_type', [], :unknown)}.to raise_error(ArgumentError)
-        end
-      end
-
-      context "if created with a known data serialization" do
-        it "should instantiate correctly" do
-          expect {RFlow::Message.new('string_type', [], 'avro')}.to_not raise_error
-          expect {RFlow::Message.new('string_type', [], 'avro')}.to_not raise_error
-        end
-
-        context "if created with a mismatched schema" do
-        end
-
-        context "if created with a matched schema" do
-        end
-
-
-        context "if created with a nil schema" do
-          context "if created with a serialized data object" do
-            before(:all) do
-              @string = 'this is a string to be serialized'
-              @avro_serialized_string = encode_avro(@avro_string_schema_string, @string)
-            end
-
-            it "should instantiate correctly" do
-              expect {RFlow::Message.new('string_type', [], 'avro', nil, @avro_serialized_string)}.to_not raise_error
-              message = RFlow::Message.new('string_type', [], 'avro', nil, @avro_serialized_string)
-            end
-          end
-        end
-      end
-    end
-
-    context "if created with invalid provenance" do
-      before(:all) do
-        @invalid_processing_event_hash = {'started_at' => 'bad time string'}
-        @invalid_provenance = [@invalid_processing_event_hash]
-      end
-
-      it "should throw an exception" do
-        expect {RFlow::Message.new('string_type', @invalid_provenance)}.to raise_error(ArgumentError)
-      end
-    end
-
-    context "if created with valid provenance" do
-      before(:all) do
-        @valid_xmlschema_time = '2001-01-01T01:01:01.000001Z'
-        @valid_processing_event_hash = {'component_instance_uuid' => 'uuid', 'started_at' => @valid_xmlschema_time}
-        @valid_processing_event = RFlow::Message::ProcessingEvent.new('uuid', @valid_xmlschema_time, @valid_xmlschema_time, 'context')
-        @valid_provenance = [
-                             RFlow::Message::ProcessingEvent.new('uuid'),
-                             @valid_processing_event_hash,
-                             @valid_processing_event,
-                            ]
-        @valid_provenance_hashes = [
-                                    {"component_instance_uuid"=>"uuid", "started_at"=>nil, "completed_at"=>nil, "context"=>nil},
-                                    {"component_instance_uuid"=>"uuid", "started_at"=>@valid_xmlschema_time, "completed_at"=>nil, "context"=>nil},
-                                    {"component_instance_uuid"=>"uuid", "started_at"=>@valid_xmlschema_time, "completed_at"=>@valid_xmlschema_time, "context"=>"context"},
-                                   ]
-      end
-
-      it "should instantiate correctly" do
-        p @valid_provenance
-        expect {RFlow::Message.new('string_type', @valid_provenance)}.to_not raise_error
-      end
-
-      it "should correctly set the provenance processing events" do
-        message = RFlow::Message.new('string_type', @valid_provenance)
-        message.provenance[1].component_instance_uuid.should == 'uuid'
-        message.provenance[1].started_at.should == Time.xmlschema(@valid_xmlschema_time)
-        message.provenance[1].completed_at.should == nil
-        message.provenance[1].context.should == nil
-      end
-
-      it "should to_hash its provenance correctly" do
-        message = RFlow::Message.new('string_type', @valid_provenance)
-        message.provenance.map(&:to_hash).should == @valid_provenance_hashes
-      end
-
-    end
-
-    context "if correctly created" do
-      it "should serialize and deserialized correctly to/from avro" do
-        message = RFlow::Message.new('string_type')
-        message.provenance << RFlow::Message::ProcessingEvent.new('UUID')
-        message.data.data_object = 'teh awesome'
-
-        processed_message = RFlow::Message.from_avro(message.to_avro)
-        message.data.to_avro.should == processed_message.data.to_avro
-        message.data.data_object.should == processed_message.data.data_object
-      end
-    end
-
-    context "if data extensions exist" do
-      it "should extend the data element with the extension" do
-        module ExtensionModule; def ext_method; end; end
-
-        message = RFlow::Message.new('string_type')
-        message.data.methods.should_not include(:ext_method)
-
-        RFlow::Configuration.add_available_data_extension('string_type', ExtensionModule)
-        message = RFlow::Message.new('string_type')
-        message.data.methods.should include(:ext_method)
-
-      end
-    end
-  end
-
-  it "should correctly handle large raw types" do
-    message = RFlow::Message.new('RFlow::Message::Data::Raw')
-    message.data.raw = Array.new(101) { rand(256) }.pack('c*')
-
-    message_avro = message.to_avro.force_encoding('BINARY')
-
-    processed_message = RFlow::Message.from_avro(message_avro)
-    processed_message_avro = processed_message.to_avro.force_encoding('BINARY')
-
-    @raw_schema = RFlow::Configuration.available_data_types['RFlow::Message::Data::Raw']['avro']
-
-    encode_avro(@raw_schema, message.data.data_object).should == message.data.to_avro
-    decode_avro(@raw_schema, message.data.to_avro).should == message.data.data_object
-
-    p message.data.raw
-    p message_avro
-    p message_avro.bytesize
-    p processed_message_avro
-    p processed_message_avro.bytesize
-
-    p message_avro.encoding
-    p message_avro.valid_encoding?
-
-    p processed_message_avro.encoding
-    p processed_message_avro.valid_encoding?
-
-    message_data_avro = message.data.to_avro.force_encoding('BINARY')
-    processed_message_data_avro = processed_message.data.to_avro.force_encoding('BINARY')
-
-    p message_data_avro.encoding
-    p message_data_avro.valid_encoding?
-    p message_data_avro
-    p processed_message_data_avro.encoding
-    p processed_message_data_avro.valid_encoding?
-    p processed_message_data_avro
-
-    Digest::MD5.hexdigest(message_avro).should == Digest::MD5.hexdigest(processed_message_avro)
-
-
-    message_data_avro.should == processed_message_data_avro
-    Digest::MD5.hexdigest(message_data_avro).should == Digest::MD5.hexdigest(processed_message_data_avro)
-    Digest::MD5.hexdigest(message.data.raw).should == Digest::MD5.hexdigest(processed_message.data.raw)
-  end
-
-end

data/spec/schema_spec.rb

@@ -1,28 +0,0 @@
-require 'spec_helper.rb'
-
-describe 'RFlow::Message::Data::Raw Avro Schema' do
-  before(:each) do
-    @schema_string = RFlow::Configuration.available_data_types['RFlow::Message::Data::Raw']['avro']
-  end
-
-  it "should encode and decode an object" do
-    raw = {
-      'raw' => Array.new(256) { rand(256) }.pack('c*')
-    }
-
-    expect {encode_avro(@schema_string, raw)}.to_not raise_error
-    avro_encoded_raw = encode_avro(@schema_string, raw)
-
-    expect {decode_avro(@schema_string, avro_encoded_raw)}.to_not raise_error
-    decoded_raw = decode_avro(@schema_string, avro_encoded_raw)
-
-    decoded_raw.should == raw
-
-    p decoded_raw['raw'].encoding
-    p raw['raw'].encoding
-
-    decoded_raw['raw'].should == raw['raw']
-
-  end
-
-end

data/temp.rb

@@ -1,295 +0,0 @@
-module RFlow
-  def self.run(config)
-    # Take in the config file
-    # Set a module-level config
-    # Set module-level attributes (logger)
-    # Create manager
-    # Start manager with parsed config elements
-  end
-
-  class Manager
-
-    def initialize(config)
-    end
-    # Find each component
-    # Instantiate (process management)
-  end
-end
-
-class SchemaRegistry
-  # maps data type names to schemas based on schema type
-  find_by_data_type_name
-end
-
-class MessageDataRegistry
-  def find(data_type_name)
-    # returns a data type class if registered, nil otherwise
-  end
-end
-
-class Message::Data
-  # contains the schema + data information
-  # subclasses can add extra functionality, otherwise will just have
-  # acces to standard messagedata stuffs (i.e. standard avro data types)
-  # delegates a lot to standard Avro types
-
-  # how does this get access to the registry at the class level?
-  class << self
-    attr_accessor :class_registry
-    attr_accessor :schema_registry
-  end
-
-  # Pointer to encapsulating message
-  attr_accessor :message
-
-  def initialize(data_type_name, serialized_data=nil, schema_name=nil, schema_type=nil, schema=nil, message=nil)
-    # schema_name ||= 'org.rflow.Messages.GenericStringMap'
-    # schema_type ||= 'avro'
-    # schema ||= 'default avro schema'
-
-    merge_options
-
-    # TODO: think about schema resolution and conflicts between passed
-    # data and schema registry
-    # Lookup schema based on data type name
-    registered_schema_name, registered_schema, registered_schema_type = self.class.schema_registry.find(data_type_name)
-    if registered_schema.nil? && schema
-      # If you were given a schema and didn't get one from the
-      # registry register the schema?
-      self.class.schema_registry.register(data_type_name, schema_name, schema_type, schema)
-    else
-
-    end
-
-  end
-
-  def self.create(data_type_name, data=nil, schema_name=nil, schema_type=nil, schema=nil)
-    # look for object in registry by data_type_name
-    # if object found, call new on that object
-    # otherwise, call new on the default object
-    message_class = self.class.data_class_registry.find(data_type_name)
-    if message_class.nil?
-      MessageData.new(data_type_name, data, schema_name, schema_type, schema)
-    else
-      message_class.create(data_type_name, data, schema_name, schema_type, schema)
-    end
-  end
-end
-
-module HTTPResponse
-end
-
-Message.new.extend(HTTPResponse)
-
-class HTTPRequest < RFlow::Message::Data
-  # used to add methods, defaults, and more to data object, if required
-
-  # Put this in the registry
-  AVRO_SCHEMA_NAME = 'org.rflow.http_request'
-  DATA_TYPE_NAME = "HTTPRequest"
-
-  # All subclasses must have the same initialize signature.  They need
-  # to figure out what to do when they get the extra parameters that
-  # might conflict with expectations.  Subclasses are usually meant to
-  # enable extra functionality on a given data type, so as long as it
-  # operates properly, it might not care (duck typing)
-  def initialize(data_type_name, data, schema_name, schema_type, schema)
-    super(DATA_TYPE_NAME, data, AVRO_SCHEMA_NAME)
-    # do nice stuff with data here
-  end
-
-  def self.create(data_type_name, data, schema_name, schema_type, schema)
-    # figure out if you are being called with incompatible arguments,
-    # i.e. schema stuff
-  end
-
-end
-
-class Message
-  # contains all definitions about what to do for a message
-  # has a default Avro schema for a data type
-
-  class << self
-    attr_accessor :data_class_registry
-  end
-
-
-  # Should load all the data stuff, perhaps to top level method on object
-
-  attr_accessor :data_type_name, :provenance, :origination_context, :data_type_schema, :data
-
-  def initialize(data_type_name, provenance=nil, origination_context=nil, data_type_schema=nil, data=nil)
-    if data
-      # Potentially register this data_type_name to the schema
-    else
-      # Lookup MessageData type in the MessageDataRegistry
-      # if found and a class, create a specific MessageData object
-      #   extend it with the module
-      # else, create generic MessageData object which will use
-      #   the schema registry, under the hood
-      # if found and a module, extend object with found module
-
-      message_data_class = self.class.data_class_registry.find(data_type_name)
-      if message_data_class && message_data_class.class.is_a? Class
-        message_data = message_data_class.new
-      else
-        message_data = Message::Data.new
-        message_data.extend message_data_class if message_data_class.is_a? Module
-      end
-    end
-  end
-
-end
-
-class Port
-  def read_message
-    parts = read_all_parts
-    parts.assemble
-    data_type_name = read_message_part
-    provenance = read_message_part
-    origination_context = read_message_part
-    data_type_schema = read_message_part
-    data = read_message_part
-
-    message = Message.new(data_type_name, provenance, origination_context, data_type_schema, data)
-
-    message
-  end
-end
-
-class PortCollection
-end
-
-class Logger
-end
-
-class Component
-  def self.input_port(port_def)
-    @@input_ports ||= PortCollection.new
-    if port_def.is_a? Array
-      port_name = port_def.first.to_sym
-      port_incidence = :array
-    else
-      port_name = port_def
-      port_incidence = :single
-    end
-    @@input_ports[port_name] = InputPort.new port_name, port_incidence
-  end
-
-  def self.output_port
-    # same as input port with different stuffs
-  end
-
-  STATES = [:initialized, :started, :configured, :running, :stopping, :stopped]
-  attr_accessor :state
-  attr_accessor :input_ports
-  attr_accessor :output_ports
-
-  attr_accessor :uuid
-  attr_accessor :name
-
-  CONFIG_DEFAULTS = {
-    :logger,
-    :working_directory_path,
-  }
-
-  def initialize(config, run_directory)
-    # configure component
-    config = {
-    }
-
-    # TODO: where is the management bus listener configured/started
-  end
-
-  def run
-    input_ports.ready do |port|
-      message = port.read_message
-      process_input(port, message)
-      # read from the port and think about things
-      out.send('stuff')
-      another_out.send('more stuff')
-    end
-    # listen to
-  end
-
-  def process_message(input_port, message)
-
-  end
-
-  def receive_message(port)
-    port.receive
-  end
-
-  def send_message(port, message)
-    port.send(message)
-  end
-
-end
-
-class HTTPServer < RFlow::Component
-  input_port :responses
-  output_port :requests
-
-  input_types "HTTP::Response"
-  output_types "HTTP::Request"
-
-
-end
-
-class PassThrough < RFlow::Component
-  input_port [:in]
-  input_port :another_in
-  output_port :out
-  output_port :another_out
-
-  output_types
-
-  def initialize(config, run_directory)
-    # This will initialize the ports
-    super
-    # Do stuff to initialize component.  Don't assume singleton
-  end
-
-
-  def process_message(input_port, data)
-    out.send(message)
-    another_out.send(message)
-
-
-  end
-
-  def process_data(input_port
-
-end
-
-
-class Transform < RFlow::Component
-
-end
-
-# Plugins:
-
-# MessageData subclass: rflow-data-http_request
-#  lib/rflow-data-http_request.rb
-require 'rflow'
-require 'lib/data_name'
-RFlow.available_data_types << data_name_object
-
-
-# Component: rflow-component-http_server
-#   lib/rflow-component-http_server
-require 'rflow'
-require 'lib/component_name'
-RFlow.available_components << component_class
-
-
-
-
-#   lib/component_name.rb ->
-# data_type_name => schema + registration: just register in the application
-
-
-
-# Server -> (HttpRequest -> Translate -> HTTPResponse) -> Server
-
-