rflow 0.0.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +6 -0
  3. data/.rvmrc +1 -0
  4. data/Gemfile +5 -0
  5. data/NOTES +187 -0
  6. data/README +0 -0
  7. data/Rakefile +16 -0
  8. data/bin/rflow +215 -0
  9. data/example/basic_config.rb +49 -0
  10. data/example/basic_extensions.rb +142 -0
  11. data/example/http_config.rb +21 -0
  12. data/example/http_extensions.rb +262 -0
  13. data/lib/rflow.rb +440 -0
  14. data/lib/rflow/component.rb +192 -0
  15. data/lib/rflow/component/port.rb +150 -0
  16. data/lib/rflow/components.rb +10 -0
  17. data/lib/rflow/components/raw.rb +26 -0
  18. data/lib/rflow/components/raw/extensions.rb +18 -0
  19. data/lib/rflow/configuration.rb +290 -0
  20. data/lib/rflow/configuration/component.rb +27 -0
  21. data/lib/rflow/configuration/connection.rb +98 -0
  22. data/lib/rflow/configuration/migrations/20010101000001_create_settings.rb +14 -0
  23. data/lib/rflow/configuration/migrations/20010101000002_create_components.rb +19 -0
  24. data/lib/rflow/configuration/migrations/20010101000003_create_ports.rb +24 -0
  25. data/lib/rflow/configuration/migrations/20010101000004_create_connections.rb +27 -0
  26. data/lib/rflow/configuration/port.rb +30 -0
  27. data/lib/rflow/configuration/ruby_dsl.rb +183 -0
  28. data/lib/rflow/configuration/setting.rb +67 -0
  29. data/lib/rflow/configuration/uuid_keyed.rb +18 -0
  30. data/lib/rflow/connection.rb +59 -0
  31. data/lib/rflow/connections.rb +2 -0
  32. data/lib/rflow/connections/zmq_connection.rb +101 -0
  33. data/lib/rflow/message.rb +191 -0
  34. data/lib/rflow/port.rb +4 -0
  35. data/lib/rflow/util.rb +19 -0
  36. data/lib/rflow/version.rb +3 -0
  37. data/rflow.gemspec +42 -0
  38. data/schema/message.avsc +36 -0
  39. data/schema/raw.avsc +9 -0
  40. data/spec/fixtures/config_ints.rb +61 -0
  41. data/spec/fixtures/extensions_ints.rb +141 -0
  42. data/spec/rflow_configuration_spec.rb +73 -0
  43. data/spec/rflow_message_data_raw.rb +26 -0
  44. data/spec/rflow_message_data_spec.rb +60 -0
  45. data/spec/rflow_message_spec.rb +182 -0
  46. data/spec/rflow_spec.rb +100 -0
  47. data/spec/schema_spec.rb +28 -0
  48. data/spec/spec_helper.rb +37 -0
  49. data/temp.rb +295 -0
  50. metadata +270 -0
@@ -0,0 +1,27 @@
1
+ require 'active_record'
2
+ require 'rflow/configuration/uuid_keyed'
3
+
4
+ class RFlow
5
+ class Configuration
6
+ class Component < ConfigDB
7
+ include UUIDKeyed
8
+ include ActiveModel::Validations
9
+
10
+ class ComponentInvalid < StandardError; end
11
+ class ComponentNotFound < StandardError; end
12
+
13
+ serialize :options, Hash
14
+
15
+ has_many :input_ports, :primary_key => 'uuid', :foreign_key => 'component_uuid'
16
+ has_many :output_ports, :primary_key => 'uuid', :foreign_key => 'component_uuid'
17
+
18
+ #TODO: Get this to work
19
+ #has_many :input_connections, :through => :input_ports, :source => :input_connections
20
+ #has_many :output_connections, :through => :output_ports, :source => :output_connection
21
+
22
+
23
+ validates_uniqueness_of :name
24
+
25
+ end
26
+ end
27
+ end
@@ -0,0 +1,98 @@
1
+ require 'active_record'
2
+ require 'rflow/configuration/uuid_keyed'
3
+
4
+ class RFlow
5
+ class Configuration
6
+ class Connection < ConfigDB
7
+ class ConnectionInvalid < StandardError; end
8
+
9
+ include UUIDKeyed
10
+ include ActiveModel::Validations
11
+
12
+ serialize :options, Hash
13
+
14
+ belongs_to :input_port, :primary_key => 'uuid', :foreign_key => 'input_port_uuid'
15
+ belongs_to :output_port,:primary_key => 'uuid', :foreign_key => 'output_port_uuid'
16
+
17
+ before_create :merge_default_options!
18
+
19
+ validates_uniqueness_of :uuid
20
+ validates_presence_of :output_port_uuid, :input_port_uuid
21
+
22
+ validate :all_required_options_present
23
+
24
+
25
+ def all_required_options_present
26
+ self.class.required_options.each do |option_name|
27
+ unless self.options.include? option_name.to_s
28
+ errors.add(:options, "must include #{option_name} for #{self.class.to_s}")
29
+ end
30
+ end
31
+ end
32
+
33
+
34
+ def merge_default_options!
35
+ self.options ||= {}
36
+ self.class.default_options.each do |option_name, default_value_or_proc|
37
+ self.options[option_name.to_s] ||= default_value_or_proc.is_a?(Proc) ? default_value_or_proc.call(self) : default_value_or_proc
38
+ end
39
+ end
40
+
41
+
42
+ # Should return a list of require option names which will be
43
+ # used in validations. To be overridden.
44
+ def self.required_options; []; end
45
+
46
+
47
+ # Should return a hash of default options, where the keys are
48
+ # the option names and the values are either default option
49
+ # values or Procs that take a single connection argument. This
50
+ # allow defaults to use other parameters in the connection to
51
+ # construct the appropriate default value.
52
+ def self.default_options; {}; end
53
+
54
+ end
55
+
56
+
57
+ # STI Subclass for ZMQ connections and their required options
58
+ class ZMQConnection < Connection
59
+
60
+ def self.default_options
61
+ {
62
+ 'output_socket_type' => 'PUSH',
63
+ 'output_address' => lambda{|conn| "ipc://rflow.#{conn.uuid}"},
64
+ 'output_responsibility' => 'bind',
65
+ 'input_socket_type' => 'PULL',
66
+ 'input_address' => lambda{|conn| "ipc://rflow.#{conn.uuid}"},
67
+ 'input_responsibility' => 'connect',
68
+ }
69
+ end
70
+ end
71
+
72
+
73
+ # STI Subclass for AMQP connections and their required options
74
+ class AMQPConnection < Connection
75
+
76
+ def self.default_options
77
+ {
78
+ 'host' => 'localhost',
79
+ 'port' => 5672,
80
+ 'insist' => true,
81
+ 'vhost' => '/',
82
+ 'username' => 'guest',
83
+ 'password' => 'guest',
84
+
85
+ # If a queue is created, these are the default parameters
86
+ # for said queue type
87
+ 'queue_passive' => false,
88
+ 'queue_durable' => true,
89
+ 'queue_exclusive' => false,
90
+ 'queue_auto_delete' => false,
91
+ 'queue_nowait' => true,
92
+ }
93
+ end
94
+
95
+ end
96
+ end
97
+ end
98
+
@@ -0,0 +1,14 @@
1
+ class CreateSettings < ActiveRecord::Migration
2
+ def self.up
3
+ create_table(:settings, :id => false) do |t|
4
+ t.string :name, :primary => true
5
+ t.text :value
6
+
7
+ t.timestamps
8
+ end
9
+ end
10
+
11
+ def self.down
12
+ drop_table :settings
13
+ end
14
+ end
@@ -0,0 +1,19 @@
1
+ class CreateComponents < ActiveRecord::Migration
2
+ def self.up
3
+ create_table(:components, :id => false) do |t|
4
+ t.string :uuid, :limit => 36, :primary => true
5
+ t.string :name
6
+ t.boolean :managed, :default => true
7
+ t.text :specification
8
+ t.text :options
9
+
10
+ t.timestamps
11
+ end
12
+
13
+ add_index :components, :uuid, :unique => true
14
+ end
15
+
16
+ def self.down
17
+ drop_table :components
18
+ end
19
+ end
@@ -0,0 +1,24 @@
1
+ class CreatePorts < ActiveRecord::Migration
2
+ def self.up
3
+ create_table(:ports, :id => false) do |t|
4
+ t.string :uuid, :limit => 36, :primary => true
5
+ t.string :name
6
+
7
+ # For STI
8
+ t.text :type
9
+
10
+ # UUID version of belongs_to :component
11
+ t.string :component_uuid
12
+
13
+ t.timestamps
14
+ end
15
+
16
+ add_index :ports, :uuid, :unique => true
17
+ add_index :ports, :component_uuid
18
+ add_index :ports, [:component_uuid, :name], :unique => true
19
+ end
20
+
21
+ def self.down
22
+ drop_table :ports
23
+ end
24
+ end
@@ -0,0 +1,27 @@
1
+ class CreateConnections < ActiveRecord::Migration
2
+ def self.up
3
+ create_table(:connections, :id => false) do |t|
4
+ t.string :uuid, :limit => 36, :primary => false
5
+ t.string :name
6
+
7
+ # To allow for multiple types of connections
8
+ t.string :type
9
+
10
+ # Data flows from an output port to an input port
11
+ t.string :output_port_uuid
12
+ t.string :output_port_key, :default => '0'
13
+ t.string :input_port_uuid
14
+ t.string :input_port_key, :default => '0'
15
+
16
+ t.text :options
17
+
18
+ t.timestamps
19
+ end
20
+
21
+ add_index :connections, :uuid, :unique => true
22
+ end
23
+
24
+ def self.down
25
+ drop_table :connections
26
+ end
27
+ end
@@ -0,0 +1,30 @@
1
+ require 'active_record'
2
+ require 'rflow/configuration/uuid_keyed'
3
+
4
+ class RFlow
5
+ class Configuration
6
+ class Port < ConfigDB
7
+ include UUIDKeyed
8
+ include ActiveModel::Validations
9
+
10
+ class PortInvalid < StandardError; end
11
+
12
+ belongs_to :component, :primary_key => 'uuid', :foreign_key => 'component_uuid'
13
+
14
+ # TODO: Make some sort of component/port validation work here
15
+ #validate :component_has_defined_port
16
+ end
17
+
18
+ # STI-based classes
19
+ class InputPort < Port;
20
+ has_many :input_connections, :class_name => 'RFlow::Configuration::Connection', :primary_key => 'uuid', :foreign_key => 'input_port_uuid'
21
+ has_many :output_ports, :through => :connections
22
+ end
23
+
24
+ class OutputPort < Port;
25
+ has_many :output_connections, :class_name => 'RFlow::Configuration::Connection', :primary_key => 'uuid', :foreign_key => 'output_port_uuid'
26
+ has_many :input_ports, :through => :connections
27
+ end
28
+ end
29
+ end
30
+
@@ -0,0 +1,183 @@
1
+ require 'rflow/configuration'
2
+
3
+ class RFlow
4
+ class Configuration
5
+
6
+ # Ruby DSL config file controller.
7
+ # TODO: more docs and examples
8
+ class RubyDSL
9
+ attr_accessor :setting_specs, :component_specs, :connection_specs, :allocated_system_ports
10
+
11
+ def initialize
12
+ @setting_specs = []
13
+ @component_specs = []
14
+ @connection_specs = []
15
+ end
16
+
17
+ # Helper function to extract the line of the config that
18
+ # specified the operation. Useful in printing helpful error messages
19
+ def get_config_line(call_history)
20
+ call_history.first.split(':in').first
21
+ end
22
+
23
+ # DSL method to specify a name/value pair. RFlow core uses the
24
+ # 'rflow.' prefix on all of its settings. Custom settings
25
+ # should use a custom (unique) prefix
26
+ def setting(setting_name, setting_value)
27
+ setting_specs << {:name => setting_name.to_s, :value => setting_value.to_s, :config_line => get_config_line(caller)}
28
+ end
29
+
30
+ # DSL method to specify a component. Expects a name,
31
+ # specification, and set of component specific options, that
32
+ # must be marshallable into the database (i.e. should all be strings)
33
+ def component(component_name, component_specification, component_options={})
34
+ component_specs << {
35
+ :name => component_name,
36
+ :specification => component_specification.to_s, :options => component_options,
37
+ :config_line => get_config_line(caller)
38
+ }
39
+ end
40
+
41
+ # DSL method to specify a connection between a
42
+ # component/output_port and another component/input_port. The
43
+ # component/port specification is a string where the names of
44
+ # the two elements are separated by '#', and the "connection" is
45
+ # specified by a Ruby Hash, i.e.:
46
+ # connect 'componentA#output' => 'componentB#input'
47
+ # Array ports are specified with an key suffix in standard
48
+ # progamming syntax, i.e.
49
+ # connect 'componentA#arrayport[2]' => 'componentB#in[1]'
50
+ # Uses the model to assign random UUIDs
51
+ def connect(connection_hash)
52
+ config_file_line = get_config_line(caller)
53
+ connection_hash.each do |output_string, input_string|
54
+ output_component_name, output_port_name, output_port_key = parse_connection_string(output_string)
55
+ input_component_name, input_port_name, input_port_key = parse_connection_string(input_string)
56
+
57
+ connection_specs << {
58
+ :name => output_string + '=>' + input_string,
59
+ :output_component_name => output_component_name,
60
+ :output_port_name => output_port_name, :output_port_key => output_port_key,
61
+ :output_string => output_string,
62
+ :input_component_name => input_component_name,
63
+ :input_port_name => input_port_name, :input_port_key => input_port_key,
64
+ :input_string => input_string,
65
+ :config_line => config_file_line,
66
+ }
67
+ end
68
+ end
69
+
70
+ # Splits the connection string into component/port parts
71
+ COMPONENT_PORT_STRING_REGEX = /^(\w+)#(\w+)(?:\[([^\]]+)\])?$/
72
+ def parse_connection_string(connection_string)
73
+ matched = COMPONENT_PORT_STRING_REGEX.match(connection_string)
74
+ raise ArgumentError, "Invalid component/port string specification: #{connection_string}" unless matched
75
+ # component_name, port_name, port_key
76
+ [matched[1], matched[2], (matched[3] || nil)]
77
+ end
78
+
79
+
80
+ # Method to process the 'DSL' objects into the config database
81
+ # via ActiveRecord
82
+ def process
83
+ process_setting_specs
84
+ process_component_specs
85
+ process_connection_specs
86
+ end
87
+
88
+
89
+ # Iterates through each setting specified in the DSL and
90
+ # creates rows in the database corresponding to the setting
91
+ def process_setting_specs
92
+ setting_specs.each do |setting_spec|
93
+ RFlow.logger.debug "Found config file setting '#{setting_spec[:name]}' = (#{Dir.getwd}) '#{setting_spec[:value]}'"
94
+ RFlow::Configuration::Setting.create! :name => setting_spec[:name], :value => setting_spec[:value]
95
+ end
96
+ end
97
+
98
+
99
+ # Iterates through each component specified in the DSL and
100
+ # creates rows in the database corresponding to the component.
101
+ def process_component_specs
102
+ component_specs.each do |component_spec|
103
+ RFlow.logger.debug "Found component '#{component_spec[:name]}', creating"
104
+ RFlow::Configuration::Component.create! :name => component_spec[:name], :specification => component_spec[:specification], :options => component_spec[:options]
105
+ end
106
+ end
107
+
108
+
109
+ # Iterates through each component specified in the DSL and uses
110
+ # 'process_connection' to insert all the parts of the connection
111
+ # into the database
112
+ def process_connection_specs
113
+ connection_specs.each do |connection_spec|
114
+ process_connection_spec(connection_spec)
115
+ end
116
+ end
117
+
118
+ # For the given connection, break up each input/output
119
+ # component/port specification, ensure that the component
120
+ # already exists in the database (by name). Also, only supports
121
+ # ZeroMQ ipc sockets
122
+ def process_connection_spec(connection_spec)
123
+ RFlow.logger.debug "Found connection from '#{connection_spec[:output_string]}' to '#{connection_spec[:input_string]}', creating"
124
+
125
+ # an input port can be associated with multiple outputs, but
126
+ # an output port can only be associated with one input
127
+ output_component = RFlow::Configuration::Component.find_by_name connection_spec[:output_component_name]
128
+ raise RFlow::Configuration::Component::ComponentNotFound, "#{connection_spec[:output_component_name]}" unless output_component
129
+ output_port = output_component.output_ports.find_or_initialize_by_name :name => connection_spec[:output_port_name]
130
+ output_port.save!
131
+
132
+ input_component = RFlow::Configuration::Component.find_by_name connection_spec[:input_component_name]
133
+ raise RFlow::Configuration::Component::ComponentNotFound, "#{connection_spec[:input_component_name]}" unless input_component
134
+ input_port = input_component.input_ports.find_or_initialize_by_name :name => connection_spec[:input_port_name]
135
+ input_port.save!
136
+
137
+ # Create a unique ZMQ address
138
+ # zmq_address = "ipc://run/rflow.#{output_component.uuid}.#{output_port.uuid}"
139
+ # if connection_spec[:output_port_key]
140
+ # zmq_address << ".#{connection_spec[:output_port_key].gsub(/[^\w]/, '').downcase}"
141
+ # end
142
+
143
+ connection = RFlow::Configuration::ZMQConnection.new(:name => connection_spec[:name],
144
+ :output_port_key => connection_spec[:output_port_key],
145
+ :input_port_key => connection_spec[:input_port_key])
146
+ # :options => {
147
+ # 'output_socket_type' => "PUSH",
148
+ # 'output_address' => zmq_address,
149
+ # 'output_responsibility' => "bind",
150
+ # 'input_socket_type' => "PULL",
151
+ # 'input_address' => zmq_address,
152
+ # 'input_responsibility' => "connect",
153
+ # })
154
+
155
+ connection.output_port = output_port
156
+ connection.input_port = input_port
157
+ connection.save!
158
+
159
+ rescue RFlow::Configuration::Component::ComponentNotFound => e
160
+ error_message = "Component '#{e.message}' not found at #{connection_spec[:config_line]}"
161
+ RFlow.logger.error error_message
162
+ raise RFlow::Configuration::Connection::ConnectionInvalid, error_message
163
+ rescue Exception => e
164
+ # TODO: Figure out why an ArgumentError doesn't put the
165
+ # offending message into e.message, even though it is printed
166
+ # out if not caught
167
+ error_message = "#{e.class}: #{e.message} at config '#{connection_spec[:config_line]}'"
168
+ RFlow.logger.debug "Exception #{e.class} - " + error_message
169
+ RFlow.logger.error error_message
170
+ raise RFlow::Configuration::Connection::ConnectionInvalid, error_message
171
+ end
172
+
173
+
174
+ # Method called within the config file itself
175
+ def self.configure
176
+ config_file = self.new
177
+ yield config_file
178
+ config_file.process
179
+ end
180
+
181
+ end # class RubyDSL
182
+ end # class Configuration
183
+ end # class RFlow
@@ -0,0 +1,67 @@
1
+ require 'active_record'
2
+ require 'rflow/configuration/uuid_keyed'
3
+
4
+ class RFlow
5
+ class Configuration
6
+ class Setting < ConfigDB
7
+ class SettingInvalid < StandardError; end
8
+
9
+ include ActiveModel::Validations
10
+
11
+ set_primary_key :name
12
+ attr_accessible :name, :value
13
+
14
+ DEFAULTS = {
15
+ 'rflow.application_name' => 'rflow',
16
+
17
+ 'rflow.application_directory_path' => '.',
18
+ 'rflow.pid_directory_path' => 'run', #lambda {File.join(Setting['rflow.application_directory_path'], 'run')},
19
+ 'rflow.log_directory_path' => 'log', #lambda {File.join(Setting['rflow.application_directory_path'], 'log')},
20
+
21
+ 'rflow.log_file_path' => lambda {File.join(Setting['rflow.log_directory_path'], Setting['rflow.application_name'] + '.log')},
22
+ 'rflow.pid_file_path' => lambda {File.join(Setting['rflow.pid_directory_path'], Setting['rflow.application_name'] + '.pid')},
23
+
24
+ 'rflow.log_level' => 'INFO',
25
+ }
26
+
27
+ DIRECTORY_PATHS = [
28
+ 'rflow.application_directory_path',
29
+ 'rflow.pid_directory_path',
30
+ 'rflow.log_directory_path',
31
+ ]
32
+ FILE_PATHS = [
33
+ 'rflow.log_file_path',
34
+ 'rflow.pid_file_path',
35
+ ]
36
+
37
+
38
+ # TODO: fix these validations, as they run without the
39
+ #application directory path context for subdirectories
40
+ #validate :valid_directory_path, :if => :directory_path?
41
+ #validate :valid_writable_path, :if => :directory_path?
42
+
43
+ # TODO: Think about making this a regex check to pull in other,
44
+ # externally-defined settings
45
+ def directory_path?
46
+ DIRECTORY_PATHS.include? self.name
47
+ end
48
+
49
+ def valid_directory_path
50
+ unless File.directory? self.value
51
+ errors.add :value, "setting '#{self.name}' is not a directory ('#{File.expand_path self.value}')"
52
+ end
53
+ end
54
+
55
+ def valid_writable_path
56
+ unless File.writable? self.value
57
+ errors.add :value, "setting '#{self.name}' is not writable ('#{File.expand_path self.value}')"
58
+ end
59
+ end
60
+
61
+ def self.[](setting_name)
62
+ Setting.find(setting_name).value rescue nil
63
+ end
64
+
65
+ end
66
+ end
67
+ end