rflow 1.0.1 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b52acbd7c200a903c0e3c17982a220a2a9ee0169
-  data.tar.gz: 6f16e7be6007f0de84de616bffaa82279bf90a59
+  metadata.gz: 325fd6dd2c2d5409dc692a641c36da3b2139f7e8
+  data.tar.gz: d71bb8ce7e17c7e86fc1e2de7f299e73efdaeb5a
 SHA512:
-  metadata.gz: ffcde5466c777d5e36c815142a1b54b8da90fab13f6e7188519148482fdfc961250149a21f59d1998b674ce520f8436dd1a6185750af76063403742767aa4201
-  data.tar.gz: 15ea4a44084a9d8fa11691f45788ec9205b442cbabd4084b18e13d7ad94016e2f9ef37fb6d6e22bd6eeabee392bcf2fbef585cd3cc991c70c47abc87c3bc20e1
+  metadata.gz: e5a602dc11534aa9a48b81bfc5d2035103b048b67d96684587f84c03152f90b415a43f6e19050616cbdccd37cfbad3ddf67e5c45ecfb4690c2dd40cfaa1b5883
+  data.tar.gz: e7d78c206614c6df0824b4d87f50a1c48edb54468defc498f5fa2319b05c707a4a4ee440514a8a11058998202b76ab93885859a8a50b1dbe85bc33bc82a5c041

data/.ruby-version

@@ -1 +1 @@
-ruby-2.1.2
+ruby-2.3.0

data/.travis.yml

@@ -4,6 +4,8 @@ rvm:
   - 1.9.3
   - 2.0.0
   - 2.1.0
+  - 2.2.0
+  - 2.3.0
 
 before_install:
   - sudo apt-get install libtool autoconf automake uuid-dev build-essential
@@ -12,6 +14,7 @@ before_install:
 #  - sudo add-apt-repository -y ppa:chris-lea/zeromq
 #  - sudo apt-get update
 #  - sudo apt-get install libzmq3 libzmq3-dev
+  - gem update bundler
 
 script: bundle exec rspec spec
 

data/Gemfile

@@ -1,8 +1,9 @@
-source "https://rubygems.org"
+source 'https://rubygems.org'
 
 gemspec
 
 group :development do
   gem 'guard'
   gem 'guard-rspec'
+  gem 'listen', '<3.1'
 end

data/Guardfile

@@ -4,5 +4,5 @@
 guard :rspec do
   watch(%r{^spec/.+_spec\.rb$})
   watch(%r{^lib/(.+)\.rb$})     { |m| "spec/lib/#{m[1]}_spec.rb" }
-  watch('spec/spec_helper.rb')  { "spec" }
+  watch('spec/spec_helper.rb')  { 'spec' }
 end

data/README.md

@@ -13,7 +13,7 @@ RFlow is a Ruby framework inspired by
 
 In short, components communicate with each other by sending/receiving
 messages via their output/input ports over connections. Ports are
-"wired" together output->input with connections, and messages are
+'wired' together output->input with connections, and messages are
 explicitly serialized before being sent over the connection. RFlow
 supports generalized connection types and message serialization,
 however only two are in current use, namely ZeroMQ connections and
@@ -58,7 +58,7 @@ work. (You will probably get errors saying arcane things like
 
 * __Port__ - a named entity on each component that is responsible for
   receiving data (and input port) or sending data (and output port).
-  Ports can be "keyed" or "indexed" to allow better multiplexing of
+  Ports can be 'keyed' or 'indexed' to allow better multiplexing of
   messages out/in a single port, as well as allow a single port to be
   accessed by an array.
 
@@ -70,7 +70,7 @@ work. (You will probably get errors saying arcane things like
 * __Message__ - a bit of serialized data that is sent out an output
   port and received on an input port. Due to the serialization,
   message types and schemas are explicitly defined. In a departure
-  from "pure" FBP, RFlow supports sending multiple message types via a
+  from 'pure' FBP, RFlow supports sending multiple message types via a
   single connection.
 
 * __Workflow__ - the common name for the digraph created when the
@@ -126,7 +126,7 @@ end
   component to clean up any external resources that it might have
   outstanding, such as file handles or network sockets.
 
-"Source" components will often do all of their work within the `run!`
+'Source' components will often do all of their work within the `run!`
 method, and often gather message data from an external source, such as
 file, database, or network socket. The following component generates a
 set of integers between a configured start/finish, incrementing by a
@@ -158,7 +158,7 @@ class RFlow::Components::GenerateIntegerSequence < RFlow::Component
 end
 ```
 
-"Middle" components receive messages on input port(s), perform a bit
+'Middle' components receive messages on input port(s), perform a bit
 of computation, and then send a message out the output port(s). The
 following component accepts a Ruby expression string via its config,
 and then uses that as an expression to determine what port to send an
@@ -189,7 +189,7 @@ class RFlow::Components::RubyProcFilter < RFlow::Component
 end
 ```
 
-"Sink" components accept messages on an input port and do not have an
+'Sink' components accept messages on an input port and do not have an
 output port. They often operate on external sinks, such as writing
 messages to a file, database, or network socket. The following
 component writes the inspected message to a file (defined via the
@@ -220,10 +220,10 @@ end
 
 RFlow messages are instances of
 [`RFlow::Message`](lib/rflow/message.rb), which are ultimately
-serialized via an Avro [schema](schema/message.zvsc).
+serialized via an Avro [schema](schema/message.avsc).
 
-There are two parts of the message "envelope": a provenance and the
-embedded data object "payload".
+There are two parts of the message 'envelope': a provenance and the
+embedded data object 'payload'.
 
 The `provenance` is a way for a component to annotate a message with a
 bit of data that should (by convention) be carried through the

data/Vagrantfile

@@ -1,16 +1,16 @@
 # -*- mode: ruby -*-
 # vi: set ft=ruby :
-VAGRANTFILE_API_VERSION = "2"
+VAGRANTFILE_API_VERSION = '2'
 
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   config.vm.define 'centos62' do |c|
-    c.vm.box = "jstoneham/rflow-centos62"
+    c.vm.box = 'jstoneham/rflow-centos62'
   end
   config.vm.define 'centos64' do |c|
-    c.vm.box = "box-cutter/centos64"
+    c.vm.box = 'box-cutter/centos64'
   end
   config.vm.define 'centos65' do |c|
-    c.vm.box = "chef/centos-6.5"
+    c.vm.box = 'chef/centos-6.5'
   end
 
   config.vm.synced_folder '.', '/rflow'
@@ -21,28 +21,28 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   config.vm.synced_folder '../rflow-components-http', '/rflow-components-http'
 
   # forward http for rflow testing
-  config.vm.network "forwarded_port", guest: 8000, host: 8000
+  config.vm.network 'forwarded_port', guest: 8000, host: 8000
 
   # install RPM dependencies for rflow and zeromq
-  config.vm.provision "shell", privileged: true, inline: <<-EOS
+  config.vm.provision 'shell', privileged: true, inline: <<-EOS
     curl -O https://dl.fedoraproject.org/pub/epel/6/i386/epel-release-6-8.noarch.rpm
     rpm -ivh epel-release-6-8.noarch.rpm
     yum -y install libyaml-devel patch libffi-devel glibc-headers autoconf gcc-c++ glibc-devel readline-devel zlib-devel openssl-devel automake libtool bison git sqlite-devel rpm-build libuuid-devel vim
   EOS
 
   # build zeromq as vagrant user
-  config.vm.provision "shell", privileged: false, inline: <<-EOS
+  config.vm.provision 'shell', privileged: false, inline: <<-EOS
     curl -O http://download.zeromq.org/zeromq-3.2.4.tar.gz
     rpmbuild -tb zeromq-3.2.4.tar.gz
   EOS
 
   # install zeromq
-  config.vm.provision "shell", privileged: true, inline: <<-EOS
+  config.vm.provision 'shell', privileged: true, inline: <<-EOS
     rpm -ivh ~vagrant/rpmbuild/RPMS/x86_64/zeromq-*
   EOS
 
   # set up RVM and bundler
-  config.vm.provision "shell", privileged: false, inline: <<-EOS
+  config.vm.provision 'shell', privileged: false, inline: <<-EOS
     rm -f .profile
     curl -sSL https://get.rvm.io | bash -s stable
     source .rvm/scripts/rvm

data/bin/rflow

@@ -16,41 +16,41 @@ option_parser = OptionParser.new do |opts|
 Usage: #{File.basename $0} [options] (start|stop|status|load)
 EOB
 
-  opts.on("-d", "--database DB", "Config database (sqlite) path (GENERALLY REQUIRED)") do |db|
+  opts.on('-d', '--database DB', 'Config database (sqlite) path (GENERALLY REQUIRED)') do |db|
     options[:config_database_path] = File.expand_path(db)
   end
 
-  opts.on("-c", "--config CONFIG", "Config file path (only valid for load)") do |config|
+  opts.on('-c', '--config CONFIG', 'Config file path (only valid for load)') do |config|
     options[:config_file_path] = File.expand_path(config)
   end
 
-  opts.on("-e", "--extensions FILE1[,FILE_N]", Array, "Extension file paths (will load)") do |extensions|
+  opts.on('-e', '--extensions FILE1[,FILE_N]', Array, 'Extension file paths (will load)') do |extensions|
     options[:extensions_file_paths] += extensions.map {|extension| File.expand_path(extension)}
   end
 
-  opts.on("-g", "--gems GEM1[,GEM_N]", Array, "Extension gems (will require)") do |gems|
+  opts.on('-g', '--gems GEM1[,GEM_N]', Array, 'Extension gems (will require)') do |gems|
     options[:gems] += gems
   end
 
-  opts.on("-l", "--log LOGFILE", "Initial startup log file (in addition to stdout)") do |log|
+  opts.on('-l', '--log LOGFILE', 'Initial startup log file (in addition to stdout)') do |log|
     options[:startup_log_file_path] = File.expand_path(log)
   end
 
-  opts.on("-v", "--verbose [LEVEL]", [:DEBUG, :INFO, :WARN], "Control the startup log (and stdout) verbosity (DEBUG, INFO, WARN) defaults to INFO") do |level|
+  opts.on('-v', '--verbose [LEVEL]', [:DEBUG, :INFO, :WARN], 'Control the startup log (and stdout) verbosity (DEBUG, INFO, WARN) defaults to INFO') do |level|
     options[:startup_log_level] = level || :DEBUG
   end
 
-  opts.on("-f", "Run in the foreground") do |f|
+  opts.on('-f', 'Run in the foreground') do |f|
     options[:daemonize] = false
   end
 
-  opts.on_tail("--version", "Show RFlow version and exit") do
+  opts.on_tail('--version', 'Show RFlow version and exit') do
     require 'rflow/version'
     puts RFlow::VERSION
     exit 0
   end
 
-  opts.on_tail("-h", "--help", "Show this message and exit") do
+  opts.on_tail('-h', '--help', 'Show this message and exit') do
     puts opts
     exit 0
   end

data/example/http_extensions.rb

@@ -98,10 +98,10 @@ class HTTPServer < RFlow::Component
   # This is done by inspecting the provenance, specifically the
   # context attribute that we stored originally
   def process_message(input_port, input_port_key, connection, message)
-    RFlow.logger.debug "Received a message"
+    RFlow.logger.debug 'Received a message'
     return unless message.data_type_name == 'HTTPResponse'
 
-    RFlow.logger.debug "Received a HTTPResponse message, determining if its mine"
+    RFlow.logger.debug 'Received a HTTPResponse message, determining if its mine'
     my_events = message.provenance.find_all {|processing_event| processing_event.component_instance_uuid == instance_uuid}
     RFlow.logger.debug "Found #{my_events.size} processing events from me"
 
@@ -122,7 +122,7 @@ class HTTPServer < RFlow::Component
     attr_accessor :server, :client_ip, :client_port
 
     def post_init
-      @client_port, @client_ip = Socket.unpack_sockaddr_in(get_peername) rescue ["?", "?.?.?.?"]
+      @client_port, @client_ip = Socket.unpack_sockaddr_in(get_peername) rescue ['?', '?.?.?.?']
       RFlow.logger.debug "Connection from #{@client_ip}:#{@client_port}"
       super
       no_environment_strings
@@ -154,9 +154,9 @@ class HTTPServer < RFlow::Component
 
       # Default values
       resp.status                  = 200
-      resp.content                 = ""
-      resp.headers["Content-Type"] = "text/html; charset=UTF-8"
-      resp.headers["Server"]       = "Apache/2.2.3 (CentOS)"
+      resp.content                 = ''
+      resp.headers['Content-Type'] = 'text/html; charset=UTF-8'
+      resp.headers['Server']       = 'Apache/2.2.3 (CentOS)'
 
       if response_message
         resp.status  = response_message.data.status
@@ -170,7 +170,7 @@ class HTTPServer < RFlow::Component
     # Called when a connection is torn down for whatever reason.
     # Remove this connection from the server's list
     def unbind
-      RFlow.logger.debug "Connection lost"
+      RFlow.logger.debug 'Connection lost'
       server.connections.delete(self.signature)
     end
   end

data/lib/rflow.rb

@@ -1,5 +1,5 @@
-require "rubygems"
-require "bundler/setup"
+require 'rubygems'
+require 'bundler/setup'
 require 'time'
 require 'active_record'
 require 'eventmachine'
@@ -38,7 +38,7 @@ class RFlow
   def self.establish_configuration
     @configuration = Configuration.new(@config_database_path)
     unless configuration['rflow.application_directory_path']
-      raise ArgumentError, "Empty configuration database!  Use a view/controller (such as the RubyDSL) to create a configuration"
+      raise ArgumentError, 'Empty configuration database!  Use a view/controller (such as the RubyDSL) to create a configuration'
     end
   end
 
@@ -60,4 +60,25 @@ class RFlow
     master.daemonize! if @daemonize
     master.run! # blocks until EventMachine stops
   end
+
+  # This ought to be in EM, but we'll put it here instead of monkey-patching
+  def self.next_tick_and_wait
+    mutex = Mutex.new
+    condition = ConditionVariable.new
+
+    mutex.synchronize do # while locked...
+      EM.next_tick do # schedule a job that will...
+        mutex.synchronize do # grab the lock
+          begin
+            yield # do its thing...
+            condition.signal # then wake us up when it's done...
+          rescue
+            condition.signal # even if the thing fails
+            raise
+          end
+        end
+      end
+      condition.wait(mutex) # drop the mutex to allow the job to run
+    end
+  end
 end

data/lib/rflow/child_process.rb

@@ -1,3 +1,5 @@
+require 'fcntl'
+
 class RFlow
   class ChildProcess
     attr_reader :pid, :name

data/lib/rflow/component.rb

@@ -91,11 +91,11 @@ class RFlow
 
     # Returns a list of connected input ports.  Each port will have
     # one or more keys associated with a particular connection.
-    def input_ports; ports.by_type["RFlow::Component::InputPort"]; end
+    def input_ports; ports.by_type['RFlow::Component::InputPort']; end
 
     # Returns a list of connected output ports.  Each port will have
     # one or more keys associated with the particular connection.
-    def output_ports; ports.by_type["RFlow::Component::OutputPort"]; end
+    def output_ports; ports.by_type['RFlow::Component::OutputPort']; end
 
     def configure_input_port!(port_name, options = {})
       RFlow.logger.debug "Configuring component '#{name}' (#{uuid}) input port '#{port_name}' (#{options[:uuid]})"

data/lib/rflow/component/port.rb

@@ -1,7 +1,7 @@
 class RFlow
   class Component
     # TODO: make this into a class to limit the amount of extensions
-    # that we have to do when operating on these "Arrays", i.e. when
+    # that we have to do when operating on these 'Arrays', i.e. when
     # adding two together
     module ConnectionCollection
       def send_message(message)
@@ -44,6 +44,30 @@ class RFlow
       def connected?; connected; end
     end
 
+    # Stateless class to help with a nicer API
+    class HashSubPort
+      def initialize(hash_port, key)
+        @hash_port = hash_port
+        @key = key
+      end
+
+      def send_message(message)
+        connections.each {|connection| connection.send_message(message) }
+      end
+
+      def connections
+        @hash_port.connections_for(@key)
+      end
+
+      def direct_connect(other_port)
+        @hash_port.direct_connect(@key, other_port)
+      end
+
+      def each
+        connections.each {|connection| yield connection }
+      end
+    end
+
     # Allows for a list of connections to be assigned to each port/key
     # combination.  Note that binding an input port to an un-indexed
     # output port will result in messages from all indexed connections
@@ -53,41 +77,44 @@ class RFlow
     class HashPort < Port
       attr_accessor :name, :uuid
 
-      protected
-      attr_reader :connections_for
-
       public
       def initialize(component, args = {})
         super(component)
         self.uuid = args[:uuid]
         self.name = args[:name]
-        @connections_for = Hash.new {|hash, key| hash[key] = [].extend(ConnectionCollection)}
+        @connections_for = Hash.new {|hash, key| hash[key] = []}
       end
 
-      # Returns an extended Array of all the connections that should
-      # be sent/received on this port.  Merges the nil-keyed port
+      # Get the subport for a given key, which can be used to send messages
+      # or direct connection
+      def [](key)
+        HashSubPort.new(self, key)
+      end
+
+      # Returns an Array of all the connections that should
+      # be sent/received on this subport.  Merges the nil-keyed port
       # (i.e. any connections for a port without a key) to those
       # specific for the key, so should only be used to read a list of
       # connections, not to add new ones.  Use add_connection to add a
       # new connection for a given key.
-      def [](key)
+      def connections_for(key)
         case key
-        when nil; connections_for[nil]
-        else connections_for[key] + connections_for[nil]
-        end.extend(ConnectionCollection)
+        when nil; @connections_for[nil]
+        else @connections_for[key] + @connections_for[nil]
+        end
       end
 
       # Adds a connection for a given key
       def add_connection(key, connection)
         RFlow.logger.debug "Attaching #{connection.class.name} connection '#{connection.name}' (#{connection.uuid}) to port '#{name}' (#{uuid}), key '#{connection.input_port_key}'"
-        connections_for[key] << connection
+        @connections_for[key] << connection
         @all_connections = nil
       end
 
       # Removes a connection from a given key
       def remove_connection(key, connection)
         RFlow.logger.debug "Removing #{connection.class.name} connection '#{connection.name}' (#{connection.uuid}) from port '#{name}' (#{uuid}), key '#{connection.input_port_key}'"
-        connections_for[key].delete(connection)
+        @connections_for[key].delete(connection)
         @all_connections = nil
       end
 
@@ -105,41 +132,39 @@ class RFlow
         end
       end
 
-      def direct_connect(other_port)
+      def direct_connect(key = nil, other_port)
         case other_port
-        when InputPort; add_connection nil, ForwardToInputPort.new(other_port)
-        when OutputPort; add_connection nil, ForwardToOutputPort.new(other_port)
+        when InputPort; add_connection key, ForwardToInputPort.new(other_port)
+        when OutputPort; add_connection key, ForwardToOutputPort.new(other_port)
         else raise ArgumentError, "Unknown port type #{other_port.class.name}"
         end
       end
 
       # Return a list of connected keys
       def keys
-        connections_for.keys
+        @connections_for.keys
       end
 
-      # Enumerate through all the ConnectionCollections
-      # TODO: simplify with enumerators and procs
       def each
-        connections_for.values.each {|connections| yield connections }
+        @connections_for.values.each {|connections| yield connections }
       end
 
       def send_message(message)
-        def connect!; raise NotImplementedError, "Raw ports do not know how to send messages"; end
+        raise NotImplementedError, 'Raw ports do not know how to send messages'
       end
 
       # Should be overridden.  Called when it is time to actually
       # establish the connection
-      def connect!; raise NotImplementedError, "Raw ports do not know which direction to connect"; end
+      def connect!; raise NotImplementedError, 'Raw ports do not know which direction to connect'; end
 
       def all_connections
-        @all_connections ||= connections_for.values.flatten.uniq.extend(ConnectionCollection)
+        @all_connections ||= @connections_for.values.flatten.uniq.extend(ConnectionCollection)
       end
     end
 
     class InputPort < HashPort
       def connect!
-        connections_for.each {|key, conns| conns.each {|c| c.connect_input! } }
+        @connections_for.each {|key, conns| conns.each {|c| c.connect_input! } }
         @connected = true
       end
 
@@ -149,7 +174,7 @@ class RFlow
       end
 
       def recv_callback=(callback)
-        connections_for.each do |key, connections|
+        @connections_for.each do |key, connections|
           connections.each do |connection|
             connection.recv_callback = Proc.new do |message|
               callback.call self, key, connection, message
@@ -161,7 +186,7 @@ class RFlow
 
     class OutputPort < HashPort
       def connect!
-        connections_for.each {|key, conns| conns.each {|c| c.connect_output! } }
+        @connections_for.each {|key, conns| conns.each {|c| c.connect_output! } }
         @connected = true
       end