rflow 1.3.0 → 1.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -3,14 +3,29 @@ require 'rflow/configuration/uuid_keyed'
3
3
 
4
4
  class RFlow
5
5
  class Configuration
6
+ # Represents a component definition in the SQLite database.
6
7
  class Component < ConfigurationItem
7
8
  include UUIDKeyed
8
9
  include ActiveModel::Validations
9
10
 
11
+ # @!attribute options
12
+ # Open-ended Hash of component options, serialized via YAML to a single column.
13
+ # @return [Hash]
10
14
  serialize :options, Hash
11
15
 
16
+ # @!attribute shard
17
+ # The {Shard} in which this {Component} is to run.
18
+ # @return [Shard]
12
19
  belongs_to :shard, :primary_key => 'uuid', :foreign_key => 'shard_uuid'
20
+
21
+ # @!attribute input_ports
22
+ # The {InputPort}s of this component.
23
+ # @return [Array<InputPort>]
13
24
  has_many :input_ports, :primary_key => 'uuid', :foreign_key => 'component_uuid'
25
+
26
+ # @!attribute output_ports
27
+ # The {OutputPort}s of this component.
28
+ # @return [Array<OutputPort>]
14
29
  has_many :output_ports, :primary_key => 'uuid', :foreign_key => 'component_uuid'
15
30
 
16
31
  #TODO: Get this to work
@@ -3,15 +3,27 @@ require 'rflow/configuration/uuid_keyed'
3
3
 
4
4
  class RFlow
5
5
  class Configuration
6
+ # Represents a component-to-component connection in the SQLite database.
6
7
  class Connection < ConfigurationItem
8
+ # Exception for when the connection is invalid.
7
9
  class ConnectionInvalid < StandardError; end
8
10
 
9
11
  include UUIDKeyed
10
12
  include ActiveModel::Validations
11
13
 
14
+ # @!attribute options
15
+ # Open-ended Hash of component options, serialized via YAML to a single column.
16
+ # @return [Hash]
12
17
  serialize :options, Hash
13
18
 
19
+ # @!attribute input_port
20
+ # The {InputPort} to which this {Connection} delivers messages.
21
+ # @return [InputPort]
14
22
  belongs_to :input_port, :primary_key => 'uuid', :foreign_key => 'input_port_uuid'
23
+
24
+ # @!attribute output_port
25
+ # The {OutputPort} from which this {Connection} receives messages.
26
+ # @return [OutputPort]
15
27
  belongs_to :output_port,:primary_key => 'uuid', :foreign_key => 'output_port_uuid'
16
28
 
17
29
  before_create :merge_default_options!
@@ -21,6 +33,7 @@ class RFlow
21
33
 
22
34
  validate :all_required_options_present?
23
35
 
36
+ # @!visibility private
24
37
  def all_required_options_present?
25
38
  self.class.required_options.each do |option_name|
26
39
  unless self.options.include? option_name.to_s
@@ -29,6 +42,7 @@ class RFlow
29
42
  end
30
43
  end
31
44
 
45
+ # @!visibility private
32
46
  def merge_default_options!
33
47
  self.options ||= {}
34
48
  self.class.default_options.each do |name, default_value_or_proc|
@@ -37,22 +51,29 @@ class RFlow
37
51
  end
38
52
 
39
53
  # Should return a list of require option names which will be
40
- # used in validations. To be overridden.
54
+ # used in validations. To be overridden by subclasses.
55
+ # @return [Array<String>]
41
56
  def self.required_options; []; end
42
57
 
43
58
  # Should return a hash of default options, where the keys are
44
59
  # the option names and the values are either default option
45
60
  # values or Procs that take a single connection argument. This
46
61
  # allow defaults to use other parameters in the connection to
47
- # construct the appropriate default value.
62
+ # construct the appropriate default value. To be overridden
63
+ # by subclasses.
64
+ # @return [Hash]
48
65
  def self.default_options; {}; end
49
66
 
50
67
  # By default, no broker processes are required to manage a connection.
68
+ # To be overridden by subclasses.
69
+ # @return [Array<Broker>]
51
70
  def brokers; []; end
52
71
  end
53
72
 
54
- # STI Subclass for ZMQ connections and their required options
73
+ # Subclass of {Connection} for ZMQ connections and their required options.
55
74
  class ZMQConnection < Connection
75
+ # Default options required for ZeroMQ connection.
76
+ # @return [Hash]
56
77
  def self.default_options
57
78
  {
58
79
  'output_socket_type' => 'PUSH',
@@ -65,15 +86,16 @@ class RFlow
65
86
  end
66
87
  end
67
88
 
68
- # STI Subclass for brokered ZMQ connections and their required options
89
+ # Subclass of {Connection} for brokered ZMQ connections and their required options.
69
90
  #
70
91
  # We name the IPCs to resemble a quasi-component. Outputting to this
71
- # connection goes to the 'in' of the IPC pair. Reading input from this
72
- # connection comes from the 'out' of the IPC pair.
92
+ # connection goes to the +in+ of the IPC pair. Reading input from this
93
+ # connection comes from the +out+ of the IPC pair.
73
94
  #
74
95
  # The broker shuttles messages between the two to support the many-to-many
75
96
  # delivery pattern.
76
97
  class BrokeredZMQConnection < Connection
98
+ # Default ZeroMQ options required for broker connection.
77
99
  def self.default_options
78
100
  {
79
101
  'output_socket_type' => 'PUSH',
@@ -86,6 +108,7 @@ class RFlow
86
108
  end
87
109
 
88
110
  # A brokered ZMQ connection requires one broker process.
111
+ # @return [Array<Broker>]
89
112
  def brokers
90
113
  @brokers ||= [ZMQStreamer.new(self)]
91
114
  end
@@ -95,6 +118,8 @@ class RFlow
95
118
  # that can't be derived from the connection. Not persisted in the database -
96
119
  # it's encapsulated in the nature of the connection.
97
120
  class ZMQStreamer
121
+ # Backreference to the {Connection}.
122
+ # @return [Connection]
98
123
  attr_reader :connection
99
124
 
100
125
  def initialize(connection)
@@ -102,9 +127,15 @@ class RFlow
102
127
  end
103
128
  end
104
129
 
105
- # for testing purposes
130
+ # For testing purposes only.
131
+ # @!visibility private
106
132
  class NullConnectionConfiguration
107
- attr_accessor :name, :uuid, :options, :input_port_key, :output_port_key, :delivery
133
+ attr_accessor :name
134
+ attr_accessor :uuid
135
+ attr_accessor :options
136
+ attr_accessor :input_port_key
137
+ attr_accessor :output_port_key
138
+ attr_accessor :delivery
108
139
  end
109
140
  end
110
141
  end
@@ -3,24 +3,43 @@ require 'rflow/configuration/uuid_keyed'
3
3
 
4
4
  class RFlow
5
5
  class Configuration
6
+ # Represents a component port in the SQLite database.
6
7
  class Port < ConfigurationItem
7
8
  include UUIDKeyed
8
9
  include ActiveModel::Validations
9
10
 
11
+ # @!attribute component
12
+ # The {Component} to which this port belongs.
13
+ # @return [Component]
10
14
  belongs_to :component, :primary_key => 'uuid', :foreign_key => 'component_uuid'
11
15
 
12
16
  # TODO: Make some sort of component/port validation work here
13
17
  #validate :component_has_defined_port
14
18
  end
15
19
 
16
- # STI-based classes
20
+ # Subclass of {Port} to represent input ports.
17
21
  class InputPort < Port
22
+ # @!attribute input_connections
23
+ # The connections delivering messages to this {InputPort}.
24
+ # @return [Array<Connection>]
18
25
  has_many :input_connections, :class_name => 'RFlow::Configuration::Connection', :primary_key => 'uuid', :foreign_key => 'input_port_uuid'
26
+
27
+ # @!attribute connections
28
+ # Synonym for {input_connections}.
29
+ # @return [Array<Connection>]
19
30
  has_many :connections, :class_name => 'RFlow::Configuration::Connection', :primary_key => 'uuid', :foreign_key => 'input_port_uuid'
20
31
  end
21
32
 
33
+ # Subclass of {Port} to represent output ports.
22
34
  class OutputPort < Port
35
+ # @!attribute output_connections
36
+ # The connections receiving messages from this {OutputPort}.
37
+ # @return [Array<Connection>]
23
38
  has_many :output_connections, :class_name => 'RFlow::Configuration::Connection', :primary_key => 'uuid', :foreign_key => 'output_port_uuid'
39
+
40
+ # @!attribute connections
41
+ # Synonym for {output_connections}.
42
+ # @return [Array<Connection>]
24
43
  has_many :connections, :class_name => 'RFlow::Configuration::Connection', :primary_key => 'uuid', :foreign_key => 'output_port_uuid'
25
44
  end
26
45
  end
@@ -3,11 +3,13 @@ require 'rflow/configuration/uuid_keyed'
3
3
 
4
4
  class RFlow
5
5
  class Configuration
6
+ # Represents a setting in the SQLite database.
6
7
  class Setting < ConfigurationItem
7
8
  include ActiveModel::Validations
8
9
 
9
10
  self.primary_key = 'name'
10
11
 
12
+ # Default settings.
11
13
  DEFAULTS = {
12
14
  'rflow.application_name' => 'rflow',
13
15
  'rflow.application_directory_path' => '.',
@@ -18,6 +20,7 @@ class RFlow
18
20
  'rflow.log_level' => 'INFO',
19
21
  }
20
22
 
23
+ private
21
24
  DIRECTORY_PATHS = [
22
25
  'rflow.application_directory_path',
23
26
  'rflow.pid_directory_path',
@@ -52,6 +55,9 @@ class RFlow
52
55
  end
53
56
  end
54
57
 
58
+ public
59
+ # Look up a {Setting} by name from the SQLite database.
60
+ # @return [Setting]
55
61
  def self.[](name)
56
62
  Setting.find(name).value rescue nil
57
63
  end
@@ -3,20 +3,26 @@ require 'rflow/configuration/uuid_keyed'
3
3
 
4
4
  class RFlow
5
5
  class Configuration
6
+ # Represents a process shard in the SQLite database.
6
7
  class Shard < ConfigurationItem
7
8
  include UUIDKeyed
8
9
  include ActiveModel::Validations
9
10
 
11
+ # Exception for when the shard is invalid.
10
12
  class ShardInvalid < StandardError; end
11
13
 
14
+ # @!attribute components
15
+ # The {Component}s that are to run in this {Shard}.
16
+ # @return [Array<Component>]
12
17
  has_many :components, :primary_key => 'uuid', :foreign_key => 'shard_uuid'
13
18
 
14
19
  validates_uniqueness_of :name
15
20
  validates_numericality_of :count, :only_integer => true, :greater_than => 0
16
21
  end
17
22
 
18
- # STI-based classes
23
+ # Subclass of {Shard} representing a shard instantiated by a process.
19
24
  class ProcessShard < Shard; end
25
+ # Subclass of {Shard} representing a shard instantiated by a thread.
20
26
  class ThreadShard < Shard; end
21
27
  end
22
28
  end
@@ -2,7 +2,12 @@ require 'uuidtools'
2
2
 
3
3
  class RFlow
4
4
  class Configuration
5
+ # Mixin for any {ConfigurationItem} that has a UUID key.
6
+ # Sets +primary_key+ column to be +uuid+ and initializes the
7
+ # UUID on creation.
8
+ # @!visibility private
5
9
  module UUIDKeyed
10
+ # @!visibility private
6
11
  def self.included(base)
7
12
  base.class_eval do
8
13
  self.primary_key = 'uuid'
@@ -1,8 +1,11 @@
1
1
  require 'rflow/message'
2
2
 
3
3
  class RFlow
4
+ # Represents an RFlow connection from one component to another.
4
5
  class Connection
5
6
  class << self
7
+ # Build an appropriate subclass of {Connection} based on the configuration.
8
+ # @return [Connection]
6
9
  def build(config)
7
10
  case config.type
8
11
  when 'RFlow::Configuration::ZMQConnection'
@@ -15,7 +18,22 @@ class RFlow
15
18
  end
16
19
  end
17
20
 
18
- attr_accessor :config, :uuid, :name, :options
21
+ # The reference to the connection's configuration.
22
+ # @return [Configuration::Connection]
23
+ attr_accessor :config
24
+
25
+ # The connection's UUID.
26
+ # @return [String]
27
+ attr_accessor :uuid
28
+
29
+ # The connection's name.
30
+ # @return [String]
31
+ attr_accessor :name
32
+
33
+ # The connection's options Hash.
34
+ # @return [Hash]
35
+ attr_accessor :options
36
+
19
37
  attr_writer :recv_callback
20
38
  protected
21
39
  attr_reader :recv_callback
@@ -28,44 +46,55 @@ class RFlow
28
46
  @options = config.options
29
47
  end
30
48
 
31
- # Subclass and implement to be able to handle future 'recv'
49
+ # Subclass and implement to be able to handle future +recv+
32
50
  # methods. Will only be called in the context of a running
33
- # EventMachine reactor
51
+ # EventMachine reactor.
52
+ # @return [void]
34
53
  def connect_input!
35
54
  raise NotImplementedError, 'Raw connections do not support connect_input. Please subclass and define a connect_input method.'
36
55
  end
37
56
 
38
- # Subclass and implement to be able to handle future 'send'
57
+ # Subclass and implement to be able to handle future +send+
39
58
  # methods. Will only be called in the context of a running
40
- # EventMachine reactor
59
+ # EventMachine reactor.
60
+ # @return [void]
41
61
  def connect_output!
42
62
  raise NotImplementedError, 'Raw connections do not support connect_output. Please subclass and define a connect_output method.'
43
63
  end
44
64
 
45
65
  # Subclass and implement to handle outgoing messages. The message
46
- # will be a RFlow::Message object and the subclasses are expected
66
+ # will be a {RFlow::Message} object and the subclasses are expected
47
67
  # to marshal it up into something that will be unmarshalled on the
48
- # other side
68
+ # other side.
69
+ # @return [void]
49
70
  def send_message(message)
50
71
  raise NotImplementedError, 'Raw connections do not support send_message. Please subclass and define a send_message method.'
51
72
  end
52
73
 
53
74
  # Parent component will set this attribute if it expects to
54
- # recieve messages. Connection subclass should call it
55
- # (recv_callback.call(message)) when it gets a new message, which
75
+ # receive messages. {Connection} subclass should call it
76
+ # (<tt>recv_callback.call(message)</tt>) when it gets a new message, which
56
77
  # will be transmitted back to the parent component's
57
- # process_message method. Sublcass is responsible for
58
- # deserializing whatever was on the wire into a RFlow::Message object
78
+ # {Component#process_message} method. Subclass is responsible for
79
+ # deserializing whatever was on the wire into a {RFlow::Message} object.
80
+ # @return [Proc]
59
81
  def recv_callback
60
82
  @recv_callback ||= Proc.new {|message|}
61
83
  end
62
84
 
85
+ # If we are connected to an {Component::InputPort} subport, the key for that subport.
86
+ # @return [String]
63
87
  def input_port_key; config.input_port_key; end
88
+
89
+ # If we are connected to an {Component::OutputPort} subport, the key for that subport.
90
+ # @return [String]
64
91
  def output_port_key; config.output_port_key; end
65
92
  end
66
93
 
67
94
  # Primarily for testing purposes. Captures whatever messages are sent on it.
68
95
  class MessageCollectingConnection < Connection
96
+ # The messages that were collected.
97
+ # @return [Array<RFlow::Message>]
69
98
  attr_accessor :messages
70
99
 
71
100
  def initialize
@@ -73,6 +102,8 @@ class RFlow
73
102
  @messages = []
74
103
  end
75
104
 
105
+ # Override of {send_message} which adds the message to {messages}.
106
+ # @return [void]
76
107
  def send_message(message)
77
108
  @messages << message
78
109
  end
@@ -88,6 +119,8 @@ class RFlow
88
119
  @target_port = target_port
89
120
  end
90
121
 
122
+ # Override of {send_message} which forwards the message to the target port.
123
+ # @return [void]
91
124
  def send_message(message)
92
125
  @target_port.send_message(message)
93
126
  end
@@ -104,6 +137,8 @@ class RFlow
104
137
  @target_port = target_port
105
138
  end
106
139
 
140
+ # Override of {send_message} which forwards the message to the target port.
141
+ # @return [void]
107
142
  def send_message(message)
108
143
  @receiver.process_message(@target_port, nil, self, message)
109
144
  end
@@ -9,11 +9,16 @@ require 'rflow/broker'
9
9
  require 'sys/filesystem'
10
10
 
11
11
  class RFlow
12
+ # Contains all connections classes.
12
13
  module Connections
14
+ # Represents a ZeroMQ connection.
13
15
  class ZMQConnection < RFlow::Connection
14
16
  class << self
17
+ # The ZeroMQ context object.
18
+ # @return [EM::ZeroMQ::Context]
15
19
  attr_accessor :zmq_context
16
20
 
21
+ # @!visibility private
17
22
  def create_zmq_context
18
23
  version = LibZMQ::version
19
24
  RFlow.logger.debug { "Creating a new ZeroMQ context; ZeroMQ version is #{version[:major]}.#{version[:minor]}.#{version[:patch]}" }
@@ -23,12 +28,13 @@ class RFlow
23
28
  EM::ZeroMQ::Context.new(1)
24
29
  end
25
30
 
26
- # Returns the current ZeroMQ context object or creates it if it does not exist.
27
31
  def zmq_context
28
32
  @zmq_context ||= create_zmq_context
29
33
  end
30
34
  end
31
35
 
36
+ # The ZeroMQ context object.
37
+ # @return [EM::ZeroMQ::Context]
32
38
  def zmq_context; ZMQConnection.zmq_context; end
33
39
 
34
40
  private
@@ -41,6 +47,8 @@ class RFlow
41
47
  zmq_context # cause the ZMQ context to be created before the reactor is running
42
48
  end
43
49
 
50
+ # Hook up the input to the real ZeroMQ sockets.
51
+ # @return [void]
44
52
  def connect_input!
45
53
  RFlow.logger.debug "Connecting input #{uuid} with #{options.find_all {|k, v| k.to_s =~ /input/}}"
46
54
  check_address(options['input_address'])
@@ -65,6 +73,8 @@ class RFlow
65
73
  input_socket
66
74
  end
67
75
 
76
+ # Hook up the output to the real ZeroMQ sockets.
77
+ # @return [void]
68
78
  def connect_output!
69
79
  RFlow.logger.debug "Connecting output #{uuid} with #{options.find_all {|k, v| k.to_s =~ /output/}}"
70
80
  check_address(options['output_address'])
@@ -74,6 +84,8 @@ class RFlow
74
84
  output_socket
75
85
  end
76
86
 
87
+ # Send a message along the connection into ZeroMQ.
88
+ # @return [void]
77
89
  def send_message(message)
78
90
  RFlow.logger.debug "#{name}: Sending message of type '#{message.data_type_name.to_s}'"
79
91
 
@@ -123,6 +135,9 @@ class RFlow
123
135
  end
124
136
  end
125
137
 
138
+ # Subclass of {ZMQConnection} representing a brokered ZeroMQ connection
139
+ # (one where messages are sent to a separate process performing the
140
+ # many-to-many brokering function).
126
141
  class BrokeredZMQConnection < ZMQConnection
127
142
  end
128
143
 
@@ -139,6 +154,8 @@ class RFlow
139
154
  super("broker-#{connection.name}", 'Broker')
140
155
  end
141
156
 
157
+ # Start the broker process. Returns when things are shutting down.
158
+ # @return [void]
142
159
  def run_process
143
160
  version = LibZMQ::version
144
161
  RFlow.logger.debug { "Creating a new ZeroMQ context; ZeroMQ version is #{version[:major]}.#{version[:minor]}.#{version[:patch]}" }