rflow 1.0.0a1 → 1.0.0a2

data/spec/spec_helper.rb

@@ -2,11 +2,15 @@ require File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib', 'rflow')
 
 require 'fileutils'
 require 'log4r'
+require 'rspec/collection_matchers'
+
+I18n.enforce_available_locales = true
 
 RSpec.configure do |config|
   config.before(:all) do
     RFlow.logger = Log4r::Logger.new 'test'
     RFlow.logger.add Log4r::StdoutOutputter.new('test_stdout', :formatter => RFlow::Logger::LOG_PATTERN_FORMATTER)
+    RFlow.logger.level = 5
     @base_temp_directory_path = File.join(File.dirname(__FILE__), 'tmp')
   end
 
@@ -21,19 +25,11 @@ RSpec.configure do |config|
   end
 end
 
-
-def decode_avro(schema_string, serialized_object)
+def decode_avro(schema_string, bytes)
   schema = Avro::Schema.parse(schema_string)
-  serialized_object.force_encoding 'BINARY'
-  sio = StringIO.new(serialized_object)
-  Avro::IO::DatumReader.new(schema, schema).read Avro::IO::BinaryDecoder.new(sio)
+  RFlow::Avro.decode(Avro::IO::DatumReader.new(schema, schema), bytes)
 end
 
-def encode_avro(schema_string, object)
-  encoded_string = ''
-  encoded_string.force_encoding 'BINARY'
-  schema = Avro::Schema.parse(schema_string)
-  sio = StringIO.new(encoded_string)
-  Avro::IO::DatumWriter.new(schema).write object, Avro::IO::BinaryEncoder.new(sio)
-  encoded_string
+def encode_avro(schema_string, message)
+  RFlow::Avro.encode(Avro::IO::DatumWriter.new(Avro::Schema.parse(schema_string)), message)
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rflow
 version: !ruby/object:Gem::Version
-  version: 1.0.0a1
+  version: 1.0.0a2
 platform: ruby
 authors:
 - Michael L. Artz
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-01 00:00:00.000000000 Z
+date: 2014-06-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: uuidtools
@@ -114,14 +114,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.6'
+        version: '2.99'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.6'
+        version: '2.99'
+- !ruby/object:Gem::Dependency
+  name: rspec-collection_matchers
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.4
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.4
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -160,6 +174,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".rspec"
 - ".ruby-gemset"
 - ".ruby-version"
 - ".travis.yml"
@@ -176,11 +191,15 @@ files:
 - example/http_config.rb
 - example/http_extensions.rb
 - lib/rflow.rb
+- lib/rflow/child_process.rb
 - lib/rflow/component.rb
 - lib/rflow/component/port.rb
 - lib/rflow/components.rb
+- lib/rflow/components/clock.rb
+- lib/rflow/components/integer.rb
 - lib/rflow/components/raw.rb
-- lib/rflow/components/raw/extensions.rb
+- lib/rflow/components/replicate.rb
+- lib/rflow/components/ruby_proc_filter.rb
 - lib/rflow/configuration.rb
 - lib/rflow/configuration/component.rb
 - lib/rflow/configuration/connection.rb
@@ -197,30 +216,32 @@ files:
 - lib/rflow/connection.rb
 - lib/rflow/connections.rb
 - lib/rflow/connections/zmq_connection.rb
+- lib/rflow/daemon_process.rb
 - lib/rflow/logger.rb
 - lib/rflow/master.rb
 - lib/rflow/message.rb
 - lib/rflow/pid_file.rb
-- lib/rflow/port.rb
 - lib/rflow/shard.rb
-- lib/rflow/util.rb
 - lib/rflow/version.rb
 - rflow.gemspec
 - schema/message.avsc
 - schema/raw.avsc
+- schema/tick.avsc
 - spec/fixtures/config_ints.rb
 - spec/fixtures/config_shards.rb
 - spec/fixtures/extensions_ints.rb
-- spec/rflow_component_port_spec.rb
-- spec/rflow_configuration_ruby_dsl_spec.rb
-- spec/rflow_configuration_spec.rb
-- spec/rflow_message_data_raw.rb
-- spec/rflow_message_data_spec.rb
-- spec/rflow_message_spec.rb
+- spec/rflow/component/port_spec.rb
+- spec/rflow/components/clock_spec.rb
+- spec/rflow/configuration/ruby_dsl_spec.rb
+- spec/rflow/configuration_spec.rb
+- spec/rflow/forward_to_input_port_spec.rb
+- spec/rflow/forward_to_output_port_spec.rb
+- spec/rflow/logger_spec.rb
+- spec/rflow/message/data/raw_spec.rb
+- spec/rflow/message/data_spec.rb
+- spec/rflow/message_spec.rb
 - spec/rflow_spec.rb
-- spec/schema_spec.rb
 - spec/spec_helper.rb
-- temp.rb
 homepage: https://github.com/redjack/rflow
 licenses:
 - Apache-2.0
@@ -249,13 +270,16 @@ test_files:
 - spec/fixtures/config_ints.rb
 - spec/fixtures/config_shards.rb
 - spec/fixtures/extensions_ints.rb
-- spec/rflow_component_port_spec.rb
-- spec/rflow_configuration_ruby_dsl_spec.rb
-- spec/rflow_configuration_spec.rb
-- spec/rflow_message_data_raw.rb
-- spec/rflow_message_data_spec.rb
-- spec/rflow_message_spec.rb
+- spec/rflow/component/port_spec.rb
+- spec/rflow/components/clock_spec.rb
+- spec/rflow/configuration/ruby_dsl_spec.rb
+- spec/rflow/configuration_spec.rb
+- spec/rflow/forward_to_input_port_spec.rb
+- spec/rflow/forward_to_output_port_spec.rb
+- spec/rflow/logger_spec.rb
+- spec/rflow/message/data/raw_spec.rb
+- spec/rflow/message/data_spec.rb
+- spec/rflow/message_spec.rb
 - spec/rflow_spec.rb
-- spec/schema_spec.rb
 - spec/spec_helper.rb
 has_rdoc: 

data/lib/rflow/components/raw/extensions.rb

@@ -1,18 +0,0 @@
-class RFlow
-  module Components
-    module Raw
-
-      module Extensions
-
-        module RawExtension
-          def self.extended(base_data)
-            base_data.data_object ||= {'raw' => ''}
-          end
-
-          def raw; data_object['raw']; end
-          def raw=(new_raw); data_object['raw'] = new_raw; end
-        end
-      end
-    end
-  end
-end

data/lib/rflow/port.rb

@@ -1,4 +0,0 @@
-class RFlow
-  class Port
-  end # class Port
-end # class RFlow

data/lib/rflow/util.rb

@@ -1,19 +0,0 @@
-require 'uuidtools'
-
-class RFlow
-  module Util
-    # Generate a UUID based on either the SHA1 of a seed string (v5) with a
-    # 'zero' UUID namespace, or using a purely random generation
-    # (v4) if no seed string is present
-    def generate_uuid_string(seed=nil)
-      uuid = if seed
-               UUIDTools::UUID.sha1_create(UUIDTools::UUID.parse_int(0), seed)
-             else
-               UUIDTools::UUID.random_create
-             end
-      uuid.to_s
-    end
-
-
-  end
-end

data/spec/rflow_component_port_spec.rb

@@ -1,58 +0,0 @@
-require 'spec_helper.rb'
-
-describe RFlow::Component::Port do
-  it "should not be connected" do
-    described_class.new.connected?.should be_false
-  end
-end
-
-describe RFlow::Component::HashPort do
-  it "should not be connected" do
-    port_config = double('Port Config')
-    port_config.should_receive(:name).and_return('port')
-    port_config.should_receive(:uuid).and_return('1')
-
-    port = described_class.new(port_config)
-    port.connected?.should be_false
-  end
-end
-
-describe RFlow::Component::InputPort do
-  context ".connect!" do
-    it "should be connected" do
-      connection_double = double('connection')
-      connection_double.should_receive(:connect_input!)
-
-      port_config = double('Port Config')
-      port_config.should_receive(:name).and_return('port')
-      port_config.should_receive(:uuid).and_return('1')
-
-      port = described_class.new(port_config)
-      port.add_connection(nil, connection_double)
-
-      port.connected?.should be_false
-      port.connect!
-      port.connected?.should be_true
-    end
-  end
-end
-
-describe RFlow::Component::OutputPort do
-  context ".connect!" do
-    it "shouldbe connected" do
-      connection_double = double('connection')
-      connection_double.should_receive(:connect_output!)
-
-      port_config = double('Port Config')
-      port_config.should_receive(:name).and_return('port')
-      port_config.should_receive(:uuid).and_return('1')
-
-      port = described_class.new(port_config)
-      port.add_connection(nil, connection_double)
-
-      port.connected?.should be_false
-      port.connect!
-      port.connected?.should be_true
-    end
-  end
-end

data/spec/rflow_configuration_ruby_dsl_spec.rb

@@ -1,148 +0,0 @@
-require 'spec_helper.rb'
-require 'rflow/configuration'
-
-describe RFlow::Configuration::RubyDSL do
-  before(:each) do
-    ActiveRecord::Base.establish_connection adapter: "sqlite3", database: ":memory:"
-    RFlow::Configuration.migrate_database
-  end
-
-  it "should correctly process an empty DSL" do
-    described_class.configure { |c| }
-
-    config = RFlow::Configuration.new
-    RFlow::Configuration::Shard.count.should == 1
-    RFlow::Configuration::Component.count.should == 0
-    RFlow::Configuration::Port.count.should == 0
-    RFlow::Configuration::Connection.count.should == 0
-
-    puts config.to_s
-  end
-
-  it "should correctly process a component declaration" do
-    described_class.configure do |c|
-      c.component 'boom', 'town', 'opt1' => 'OPT1', 'opt2' => 'OPT2'
-    end
-
-    config = RFlow::Configuration.new
-    RFlow::Configuration::Shard.count.should == 1
-    RFlow::Configuration::Component.count.should == 1
-    RFlow::Configuration::Port.count.should == 0
-    RFlow::Configuration::Connection.count.should == 0
-
-    component = RFlow::Configuration::Component.all.first
-    component.name.should == 'boom'
-    component.specification.should == 'town'
-    component.options.should == {'opt1' => 'OPT1', 'opt2' => 'OPT2'}
-  end
-
-  it "should correctly process a connect declaration" do
-    described_class.configure do |c|
-      c.component 'first', 'First'
-      c.component 'second', 'Second'
-      c.connect 'first#out' => 'second#in'
-      c.connect 'first#out' => 'second#in[inkey]'
-      c.connect 'first#out[outkey]' => 'second#in'
-      c.connect 'first#out[outkey]' => 'second#in[inkey]'
-    end
-
-    config = RFlow::Configuration.new
-    RFlow::Configuration::Shard.count.should == 1
-    RFlow::Configuration::Component.count.should == 2
-    RFlow::Configuration::Port.count.should == 2
-    RFlow::Configuration::Connection.count.should == 4
-
-    first_component = RFlow::Configuration::Component.where(name: 'first').first
-    second_component = RFlow::Configuration::Component.where(name: 'second').first
-
-    first_component.specification.should == 'First'
-    first_component.input_ports.count.should == 0
-    first_component.output_ports.count.should == 1
-    first_component.output_ports.first.name.should == 'out'
-    first_connections = first_component.output_ports.first.connections.all
-    first_connections.count.should == 4
-    first_connections[0].input_port_key.should be_nil
-    first_connections[0].output_port_key.should be_nil
-    first_connections[1].input_port_key.should == 'inkey'
-    first_connections[1].output_port_key.should be_nil
-    first_connections[2].input_port_key.should be_nil
-    first_connections[2].output_port_key.should == 'outkey'
-    first_connections[3].input_port_key.should == 'inkey'
-    first_connections[3].output_port_key.should == 'outkey'
-
-    second_component.specification.should == 'Second'
-    second_component.input_ports.count.should == 1
-    second_component.output_ports.count.should == 0
-    second_component.input_ports.first.name.should == 'in'
-    second_connections = second_component.input_ports.first.connections.all
-    second_connections.count.should == 4
-
-    first_connections.should == second_connections
-
-    puts config.to_s
-  end
-
-  it "should correctly process shard declarations" do
-    described_class.configure do |c|
-      c.component 'first', 'First', :opt1 => 'opt1'
-
-      c.shard "s1", :process => 2 do |s|
-        s.component 'second', 'Second', :opt1 => 'opt1', "opt2" => "opt2"
-      end
-
-      c.shard "s2", :type => :process, :count => 10 do |s|
-        s.component 'third', 'Third'
-        s.component 'fourth', 'Fourth'
-      end
-
-      c.component 'fifth', 'Fifth'
-
-      c.connect 'first#out' => 'second#in'
-      c.connect 'second#out[outkey]' => 'third#in[inkey]'
-      c.connect 'second#out' => 'third#in2'
-      c.connect 'third#out' => 'fourth#in'
-      c.connect 'third#out' => 'fifth#in'
-    end
-
-    config = RFlow::Configuration.new
-    RFlow::Configuration::Shard.count.should == 3
-    RFlow::Configuration::Component.count.should == 5
-    RFlow::Configuration::Port.count.should == 8
-    RFlow::Configuration::Connection.count.should == 5
-
-    shards = RFlow::Configuration::Shard.all
-    shards.map(&:name).should == ['DEFAULT', 's1', 's2']
-    shards.first.components.all.map(&:name).should == ['first', 'fifth']
-    shards.second.components.all.map(&:name).should == ['second']
-    shards.third.components.all.map(&:name).should == ['third', 'fourth']
-
-    RFlow::Configuration::Port.all.map(&:name).should == ['out', 'in', 'out', 'in', 'in2', 'out', 'in', 'in']
-
-    RFlow::Configuration::Connection.all.map(&:name).should == ['first#out=>second#in',
-                                                                'second#out[outkey]=>third#in[inkey]',
-                                                                'second#out=>third#in2',
-                                                                'third#out=>fourth#in',
-                                                                'third#out=>fifth#in']
-
-    puts config.to_s
-  end
-
-  it "should not allow two components with the same name" do
-    expect do
-      described_class.configure do |c|
-        c.component 'first', 'First'
-        c.component 'first', 'First'
-      end
-    end.to raise_error
-  end
-
-  it "should not allow two shards with the same name" do
-    expect do
-      described_class.configure do |c|
-        c.shard("s1", :process => 2) { |s| }
-        c.shard("s1", :process => 2) { |s| }
-      end
-    end.to raise_error
-  end
-
-end

data/spec/rflow_configuration_spec.rb

@@ -1,73 +0,0 @@
-require 'spec_helper.rb'
-
-require 'rflow/configuration'
-
-
-describe RFlow::Configuration do
-  before(:each) do
-#    RFlow::Configuration.available_data_types.clear
-#    RFlow::Configuration.available_data_extensions.clear
-  end
-
-
-  describe '.add_available_data_type' do
-    context 'if passed a data_serialization that is not avro or xml' do
-      it "should throw an exception" do
-        expect do
-          RFlow::Configuration.add_available_data_type('A', 'boom', 'schema')
-        end.to raise_error(ArgumentError)
-      end
-
-      it "should not update the available_data_types" do
-        num_types = RFlow::Configuration.available_data_types.size
-        RFlow::Configuration.add_available_data_type('A', 'boom', 'schema') rescue nil
-        RFlow::Configuration.available_data_types.should have(num_types).items
-      end
-    end
-  end
-
-  describe "Data Extensions" do
-
-    describe ".add_available_data_extension" do
-      context 'if passed a non-module data extension' do
-        it "should throw an exception" do
-          expect do
-            RFlow::Configuration.add_available_data_extension('data_type', 'not a Module')
-          end.to raise_error(ArgumentError)
-        end
-      end
-
-      context "if passed a valid Module as a data extension" do
-        it "should update the available_data_extensions" do
-          num_extensions = RFlow::Configuration.available_data_extensions['data_type'].size
-          expect do
-            RFlow::Configuration.add_available_data_extension('data_type', Module.new)
-          end.to_not raise_error
-          RFlow::Configuration.available_data_extensions['data_type'].should have(num_extensions+1).items
-        end
-      end
-    end
-
-    it "should perform simple 'prefix'-based inheritance for extensions" do
-      RFlow::Configuration.add_available_data_extension('A', A = Module.new)
-      RFlow::Configuration.add_available_data_extension('A::B', B = Module.new)
-      RFlow::Configuration.add_available_data_extension('A::B::C', C = Module.new)
-      RFlow::Configuration.add_available_data_extension('A::B::C::D', D = Module.new)
-
-      RFlow::Configuration.available_data_extensions['A'].should have(1).item
-      RFlow::Configuration.available_data_extensions['A'].should == [A]
-
-      RFlow::Configuration.available_data_extensions['A::B'].should have(2).item
-      RFlow::Configuration.available_data_extensions['A::B'].should == [A, B]
-
-      RFlow::Configuration.available_data_extensions['A::B::C'].should have(3).item
-      RFlow::Configuration.available_data_extensions['A::B::C'].should == [A, B, C]
-
-      RFlow::Configuration.available_data_extensions['A::B::C::D'].should have(4).item
-      RFlow::Configuration.available_data_extensions['A::B::C::D'].should == [A, B, C, D]
-
-      RFlow::Configuration.available_data_extensions['A::B::C::D::E'].should have(4).item
-      RFlow::Configuration.available_data_extensions['A::B::C::D::E'].should == [A, B, C, D]
-    end
-  end
-end