rflow 1.0.0a1 → 1.0.0a2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +2 -0
- data/.rspec +1 -0
- data/Gemfile +0 -1
- data/NOTES +0 -13
- data/README.md +6 -1
- data/bin/rflow +2 -9
- data/example/basic_config.rb +1 -33
- data/example/basic_extensions.rb +0 -98
- data/example/http_config.rb +2 -3
- data/example/http_extensions.rb +6 -63
- data/lib/rflow.rb +31 -39
- data/lib/rflow/child_process.rb +112 -0
- data/lib/rflow/component.rb +77 -148
- data/lib/rflow/component/port.rb +38 -41
- data/lib/rflow/components.rb +4 -8
- data/lib/rflow/components/clock.rb +49 -0
- data/lib/rflow/components/integer.rb +39 -0
- data/lib/rflow/components/raw.rb +10 -6
- data/lib/rflow/components/replicate.rb +20 -0
- data/lib/rflow/components/ruby_proc_filter.rb +27 -0
- data/lib/rflow/configuration.rb +105 -184
- data/lib/rflow/configuration/component.rb +1 -4
- data/lib/rflow/configuration/connection.rb +11 -16
- data/lib/rflow/configuration/port.rb +3 -5
- data/lib/rflow/configuration/ruby_dsl.rb +105 -119
- data/lib/rflow/configuration/setting.rb +19 -25
- data/lib/rflow/configuration/shard.rb +1 -3
- data/lib/rflow/connection.rb +47 -10
- data/lib/rflow/connections.rb +0 -1
- data/lib/rflow/connections/zmq_connection.rb +34 -38
- data/lib/rflow/daemon_process.rb +155 -0
- data/lib/rflow/logger.rb +41 -25
- data/lib/rflow/master.rb +23 -105
- data/lib/rflow/message.rb +78 -108
- data/lib/rflow/pid_file.rb +37 -37
- data/lib/rflow/shard.rb +33 -100
- data/lib/rflow/version.rb +2 -2
- data/rflow.gemspec +2 -2
- data/schema/tick.avsc +10 -0
- data/spec/fixtures/config_ints.rb +4 -40
- data/spec/fixtures/config_shards.rb +1 -2
- data/spec/fixtures/extensions_ints.rb +0 -98
- data/spec/rflow/component/port_spec.rb +61 -0
- data/spec/rflow/components/clock_spec.rb +72 -0
- data/spec/rflow/configuration/ruby_dsl_spec.rb +150 -0
- data/spec/rflow/configuration_spec.rb +54 -0
- data/spec/rflow/forward_to_input_port_spec.rb +48 -0
- data/spec/rflow/forward_to_output_port_spec.rb +40 -0
- data/spec/rflow/logger_spec.rb +48 -0
- data/spec/rflow/message/data/raw_spec.rb +29 -0
- data/spec/rflow/message/data_spec.rb +58 -0
- data/spec/rflow/message_spec.rb +154 -0
- data/spec/rflow_spec.rb +94 -124
- data/spec/spec_helper.rb +8 -12
- metadata +46 -22
- data/lib/rflow/components/raw/extensions.rb +0 -18
- data/lib/rflow/port.rb +0 -4
- data/lib/rflow/util.rb +0 -19
- data/spec/rflow_component_port_spec.rb +0 -58
- data/spec/rflow_configuration_ruby_dsl_spec.rb +0 -148
- data/spec/rflow_configuration_spec.rb +0 -73
- data/spec/rflow_message_data_raw.rb +0 -26
- data/spec/rflow_message_data_spec.rb +0 -60
- data/spec/rflow_message_spec.rb +0 -182
- data/spec/schema_spec.rb +0 -28
- data/temp.rb +0 -295
data/lib/rflow/component/port.rb
CHANGED
@@ -1,14 +1,11 @@
|
|
1
1
|
class RFlow
|
2
2
|
class Component
|
3
|
-
|
4
3
|
# TODO: make this into a class to limit the amount of extensions
|
5
4
|
# that we have to do when operating on these "Arrays", i.e. when
|
6
5
|
# adding two together
|
7
6
|
module ConnectionCollection
|
8
7
|
def send_message(message)
|
9
|
-
each
|
10
|
-
connection.send_message(message)
|
11
|
-
end
|
8
|
+
each {|connection| connection.send_message(message) }
|
12
9
|
end
|
13
10
|
end
|
14
11
|
|
@@ -18,9 +15,9 @@ class RFlow
|
|
18
15
|
attr_reader :ports, :by_uuid, :by_name, :by_type
|
19
16
|
|
20
17
|
def initialize
|
21
|
-
@ports =
|
22
|
-
@by_uuid =
|
23
|
-
@by_name =
|
18
|
+
@ports = []
|
19
|
+
@by_uuid = {}
|
20
|
+
@by_name = {}
|
24
21
|
@by_type = Hash.new {|hash, key| hash[key.to_s] = []}
|
25
22
|
end
|
26
23
|
|
@@ -32,24 +29,19 @@ class RFlow
|
|
32
29
|
self
|
33
30
|
end
|
34
31
|
|
35
|
-
|
36
32
|
# Enumerate through each connected (or disconnected but
|
37
33
|
# referenced) port
|
38
34
|
# TODO: simplify with enumerators and procs
|
39
35
|
def each
|
40
|
-
ports.each
|
41
|
-
yield port
|
42
|
-
end
|
36
|
+
ports.each {|port| yield port }
|
43
37
|
end
|
44
38
|
end
|
45
39
|
|
46
|
-
|
47
40
|
class Port
|
48
41
|
attr_reader :connected
|
49
|
-
def connected?;
|
42
|
+
def connected?; connected; end
|
50
43
|
end
|
51
44
|
|
52
|
-
|
53
45
|
# Allows for a list of connections to be assigned to each port/key
|
54
46
|
# combination. Note that binding an input port to an un-indexed
|
55
47
|
# output port will result in messages from all indexed connections
|
@@ -57,16 +49,19 @@ class RFlow
|
|
57
49
|
# result in the same message being sent to all indexed
|
58
50
|
# connections.
|
59
51
|
class HashPort < Port
|
60
|
-
attr_reader :config, :name, :uuid
|
52
|
+
attr_reader :config, :name, :uuid
|
53
|
+
|
54
|
+
protected
|
55
|
+
attr_reader :connections_for
|
61
56
|
|
57
|
+
public
|
62
58
|
def initialize(config)
|
63
59
|
@config = config
|
64
60
|
@name = config.name
|
65
61
|
@uuid = config.uuid
|
66
|
-
@connections_for = Hash.new {|hash, key| hash[key] =
|
62
|
+
@connections_for = Hash.new {|hash, key| hash[key] = [].extend(ConnectionCollection)}
|
67
63
|
end
|
68
64
|
|
69
|
-
|
70
65
|
# Returns an extended Array of all the connections that should
|
71
66
|
# be sent/received on this port. Merges the nil-keyed port
|
72
67
|
# (i.e. any connections for a port without a key) to those
|
@@ -74,78 +69,80 @@ class RFlow
|
|
74
69
|
# connections, not to add new ones. Use add_connection to add a
|
75
70
|
# new connection for a given key.
|
76
71
|
def [](key)
|
77
|
-
|
72
|
+
case key
|
73
|
+
when nil; connections_for[nil]
|
74
|
+
else connections_for[key] + connections_for[nil]
|
75
|
+
end.extend(ConnectionCollection)
|
78
76
|
end
|
79
77
|
|
80
|
-
|
81
78
|
# Adds a connection for a given key
|
82
79
|
def add_connection(key, connection)
|
83
80
|
connections_for[key] << connection
|
84
81
|
end
|
85
82
|
|
86
|
-
|
87
83
|
# Return a list of connected keys
|
88
84
|
def keys
|
89
85
|
connections_for.keys
|
90
86
|
end
|
91
87
|
|
92
|
-
|
93
88
|
# Enumerate through all the ConnectionCollections
|
94
89
|
# TODO: simplify with enumerators and procs
|
95
90
|
def each
|
96
|
-
connections_for.values.each
|
97
|
-
yield connections
|
98
|
-
end
|
91
|
+
connections_for.values.each {|connections| yield connections }
|
99
92
|
end
|
100
93
|
|
101
|
-
|
102
|
-
# Send a message to all connections on all keys for this port,
|
103
|
-
# but only once per connection.
|
104
94
|
def send_message(message)
|
105
|
-
|
95
|
+
def connect!; raise NotImplementedError, "Raw ports do not know how to send messages"; end
|
106
96
|
end
|
107
97
|
|
108
|
-
|
109
98
|
# Should be overridden. Called when it is time to actually
|
110
99
|
# establish the connection
|
111
100
|
def connect!; raise NotImplementedError, "Raw ports do not know which direction to connect"; end
|
112
101
|
|
113
102
|
private
|
114
|
-
|
115
103
|
def all_connections
|
116
|
-
@all_connections ||= connections_for.
|
117
|
-
connections
|
118
|
-
end.flatten.uniq.extend(ConnectionCollection)
|
104
|
+
@all_connections ||= connections_for.values.flatten.uniq.extend(ConnectionCollection)
|
119
105
|
end
|
120
|
-
|
121
106
|
end
|
122
107
|
|
123
|
-
|
124
108
|
class InputPort < HashPort
|
125
109
|
def connect!
|
126
|
-
connections_for.each do |
|
110
|
+
connections_for.each do |key, connections|
|
127
111
|
connections.each do |connection|
|
128
112
|
connection.connect_input!
|
129
113
|
@connected = true
|
130
114
|
end
|
131
115
|
end
|
132
116
|
end
|
133
|
-
end
|
134
117
|
|
118
|
+
def recv_callback=(callback)
|
119
|
+
connections_for.each do |key, connections|
|
120
|
+
connections.each do |connection|
|
121
|
+
connection.recv_callback = Proc.new do |message|
|
122
|
+
callback.call self, key, connection, message
|
123
|
+
end
|
124
|
+
end
|
125
|
+
end
|
126
|
+
end
|
127
|
+
end
|
135
128
|
|
136
129
|
class OutputPort < HashPort
|
137
130
|
def connect!
|
138
|
-
connections_for.each do |
|
139
|
-
|
131
|
+
connections_for.each do |key, connections|
|
132
|
+
connections.each do |connection|
|
140
133
|
connection.connect_output!
|
141
134
|
@connected = true
|
142
135
|
end
|
143
136
|
end
|
144
137
|
end
|
145
|
-
end
|
146
138
|
|
139
|
+
# Send a message to all connections on all keys for this port,
|
140
|
+
# but only once per connection.
|
141
|
+
def send_message(message)
|
142
|
+
all_connections.send_message(message)
|
143
|
+
end
|
144
|
+
end
|
147
145
|
|
148
146
|
class DisconnectedPort < HashPort; end
|
149
|
-
|
150
147
|
end
|
151
148
|
end
|
data/lib/rflow/components.rb
CHANGED
@@ -1,10 +1,6 @@
|
|
1
1
|
require 'rflow/component'
|
2
|
-
|
3
|
-
|
4
|
-
module Components
|
5
|
-
end
|
6
|
-
end
|
7
|
-
|
2
|
+
require 'rflow/components/clock'
|
3
|
+
require 'rflow/components/integer'
|
8
4
|
require 'rflow/components/raw'
|
9
|
-
|
10
|
-
|
5
|
+
require 'rflow/components/replicate'
|
6
|
+
require 'rflow/components/ruby_proc_filter'
|
@@ -0,0 +1,49 @@
|
|
1
|
+
class RFlow
|
2
|
+
module Components
|
3
|
+
class Clock < Component
|
4
|
+
module Tick
|
5
|
+
SCHEMA_DIRECTORY = ::File.expand_path(::File.join(::File.dirname(__FILE__), '..', '..', '..', 'schema'))
|
6
|
+
SCHEMA_FILES = {'tick.avsc' => 'RFlow::Message::Clock::Tick'}
|
7
|
+
SCHEMA_FILES.each do |file_name, data_type_name|
|
8
|
+
schema_string = ::File.read(::File.join(SCHEMA_DIRECTORY, file_name))
|
9
|
+
RFlow::Configuration.add_available_data_type data_type_name, 'avro', schema_string
|
10
|
+
end
|
11
|
+
module Extension
|
12
|
+
def self.extended(base_data); base_data.data_object ||= {}; end
|
13
|
+
def name; data_object['name']; end
|
14
|
+
def name=(new_name); data_object['name'] = new_name; end
|
15
|
+
def timestamp; data_object['timestamp']; end
|
16
|
+
def timestamp=(new_ts); data_object['timestamp'] = new_ts; end
|
17
|
+
end
|
18
|
+
RFlow::Configuration.add_available_data_extension('RFlow::Message::Clock::Tick', Extension)
|
19
|
+
end
|
20
|
+
|
21
|
+
output_port :tick_port
|
22
|
+
|
23
|
+
DEFAULT_CONFIG = {
|
24
|
+
'name' => 'Clock',
|
25
|
+
'tick_interval' => 1
|
26
|
+
}
|
27
|
+
|
28
|
+
attr_reader :config, :tick_interval
|
29
|
+
|
30
|
+
def configure!(config)
|
31
|
+
@config = DEFAULT_CONFIG.merge config
|
32
|
+
@tick_interval = Float(@config['tick_interval'])
|
33
|
+
end
|
34
|
+
|
35
|
+
def clock_name; config['name']; end
|
36
|
+
|
37
|
+
def run!
|
38
|
+
@timer = EventMachine::PeriodicTimer.new(tick_interval) { tick }
|
39
|
+
end
|
40
|
+
|
41
|
+
def tick
|
42
|
+
tick_port.send_message(RFlow::Message.new('RFlow::Message::Clock::Tick').tap do |m|
|
43
|
+
m.data.name = clock_name
|
44
|
+
m.data.timestamp = Integer(Time.now.to_f * 1000) # ms since epoch
|
45
|
+
end)
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
class RFlow
|
2
|
+
module Components
|
3
|
+
Configuration.add_available_data_type('RFlow::Message::Data::Integer', 'avro', '{"type": "long"}')
|
4
|
+
|
5
|
+
class GenerateIntegerSequence < Component
|
6
|
+
output_port :out
|
7
|
+
output_port :even_odd_out
|
8
|
+
|
9
|
+
def configure!(config)
|
10
|
+
@start = config['start'].to_i
|
11
|
+
@finish = config['finish'].to_i
|
12
|
+
@step = config['step'] ? config['step'].to_i : 1
|
13
|
+
# If interval seconds is not given, it will default to 0
|
14
|
+
@interval_seconds = config['interval_seconds'].to_i
|
15
|
+
end
|
16
|
+
|
17
|
+
# Note that this uses the timer (sometimes with 0 interval) so as
|
18
|
+
# not to block the reactor
|
19
|
+
def run!
|
20
|
+
@timer = EM::PeriodicTimer.new(@interval_seconds) { generate }
|
21
|
+
end
|
22
|
+
|
23
|
+
def generate
|
24
|
+
Message.new('RFlow::Message::Data::Integer').tap do |m|
|
25
|
+
m.data.data_object = @start
|
26
|
+
out.send_message m
|
27
|
+
if @start % 2 == 0
|
28
|
+
even_odd_out['even'].send_message m
|
29
|
+
else
|
30
|
+
even_odd_out['odd'].send_message m
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
@start += @step
|
35
|
+
@timer.cancel if @start > @finish && @timer
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
data/lib/rflow/components/raw.rb
CHANGED
@@ -1,10 +1,17 @@
|
|
1
|
-
require 'rflow/components/raw/extensions'
|
2
|
-
|
3
1
|
class RFlow
|
4
2
|
module Components
|
5
3
|
module Raw
|
4
|
+
module Extensions
|
5
|
+
module RawExtension
|
6
|
+
def self.extended(base_data)
|
7
|
+
base_data.data_object ||= {'raw' => ''}
|
8
|
+
end
|
9
|
+
|
10
|
+
def raw; data_object['raw']; end
|
11
|
+
def raw=(new_raw); data_object['raw'] = new_raw; end
|
12
|
+
end
|
13
|
+
end
|
6
14
|
|
7
|
-
# Load the schemas
|
8
15
|
SCHEMA_DIRECTORY = ::File.expand_path(::File.join(::File.dirname(__FILE__), '..', '..', '..', 'schema'))
|
9
16
|
|
10
17
|
SCHEMA_FILES = {
|
@@ -16,11 +23,8 @@ class RFlow
|
|
16
23
|
RFlow::Configuration.add_available_data_type data_type_name, 'avro', schema_string
|
17
24
|
end
|
18
25
|
|
19
|
-
# Load the data extensions
|
20
26
|
RFlow::Configuration.add_available_data_extension('RFlow::Message::Data::Raw',
|
21
27
|
RFlow::Components::Raw::Extensions::RawExtension)
|
22
|
-
|
23
|
-
|
24
28
|
end
|
25
29
|
end
|
26
30
|
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
class RFlow
|
2
|
+
module Components
|
3
|
+
class Replicate < Component
|
4
|
+
input_port :in
|
5
|
+
output_port :out
|
6
|
+
output_port :errored
|
7
|
+
|
8
|
+
def process_message(input_port, input_port_key, connection, message)
|
9
|
+
out.each do |connections|
|
10
|
+
begin
|
11
|
+
connections.send_message message
|
12
|
+
rescue Exception => e
|
13
|
+
RFlow.logger.debug "#{self.class} Message caused exception: #{e.class}: #{e.message}: #{e.backtrace}"
|
14
|
+
errored.send_message message
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
class RFlow
|
2
|
+
module Components
|
3
|
+
class RubyProcFilter < Component
|
4
|
+
input_port :in
|
5
|
+
output_port :filtered
|
6
|
+
output_port :dropped
|
7
|
+
output_port :errored
|
8
|
+
|
9
|
+
def configure!(config)
|
10
|
+
@filter_proc = eval("lambda {|message| #{config['filter_proc_string']} }")
|
11
|
+
end
|
12
|
+
|
13
|
+
def process_message(input_port, input_port_key, connection, message)
|
14
|
+
begin
|
15
|
+
if @filter_proc.call(message)
|
16
|
+
filtered.send_message message
|
17
|
+
else
|
18
|
+
dropped.send_message message
|
19
|
+
end
|
20
|
+
rescue Exception => e
|
21
|
+
RFlow.logger.debug "#{self.class} Message caused exception: #{e.class}: #{e.message}: #{e.backtrace}"
|
22
|
+
errored.send_message message
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
data/lib/rflow/configuration.rb
CHANGED
@@ -1,7 +1,4 @@
|
|
1
|
-
require 'rflow/util'
|
2
|
-
|
3
1
|
class RFlow
|
4
|
-
|
5
2
|
# Contains all the configuration data and methods for RFlow.
|
6
3
|
# Interacts directly with underlying sqlite database, and keeps a
|
7
4
|
# registry of available data types, extensions, and components.
|
@@ -13,58 +10,47 @@ class RFlow
|
|
13
10
|
# subclasses, the controllers are things like RubyDSL, and the views
|
14
11
|
# are defined relative to the controllers
|
15
12
|
class Configuration
|
16
|
-
|
17
|
-
# An exception class
|
18
|
-
class ConfigurationInvalid < StandardError; end
|
19
|
-
|
20
|
-
|
21
|
-
# A class to hold DB config and connection information
|
22
|
-
class ConfigDB < ActiveRecord::Base
|
23
|
-
self.abstract_class = true
|
24
|
-
end
|
25
|
-
|
26
|
-
|
27
13
|
# A collection class for data extensions that supports a naive
|
28
14
|
# prefix-based 'inheritance' on lookup. When looking up a key
|
29
15
|
# with [] all existing keys will be examined to determine if the
|
30
16
|
# existing key is a string prefix of the lookup key. All the
|
31
17
|
# results are consolidated into a single, flattened array.
|
32
18
|
class DataExtensionCollection
|
33
|
-
|
34
19
|
def initialize
|
35
20
|
# TODO: choose a different data structure ...
|
36
|
-
@
|
21
|
+
@extensions_for = Hash.new {|hash, key| hash[key] = []}
|
37
22
|
end
|
38
23
|
|
39
24
|
# Return an array of all of the values that have keys that are
|
40
25
|
# prefixes of the lookup key.
|
41
26
|
def [](key)
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
end.flatten.compact
|
27
|
+
@extensions_for.
|
28
|
+
find_all {|data_type, _| key.to_s.start_with?(data_type) }.
|
29
|
+
flat_map {|_, extensions| extensions }
|
46
30
|
end
|
47
31
|
|
48
32
|
# Adds a data extension for a given data type to the collection
|
49
33
|
def add(data_type, extension)
|
50
|
-
@
|
34
|
+
@extensions_for[data_type.to_s] << extension
|
51
35
|
end
|
52
36
|
|
53
37
|
# Remove all elements from the collection. Useful for testing,
|
54
38
|
# not much else
|
55
39
|
def clear
|
56
|
-
@
|
40
|
+
@extensions_for.clear
|
57
41
|
end
|
58
|
-
|
59
42
|
end
|
60
43
|
|
44
|
+
# Base class for persisted RFlow configuration items.
|
45
|
+
class ConfigurationItem < ActiveRecord::Base
|
46
|
+
self.abstract_class = true
|
47
|
+
end
|
61
48
|
|
62
49
|
class << self
|
63
|
-
|
64
50
|
# A collection of data types (schemas) indexed by their name and
|
65
51
|
# their schema type ('avro').
|
66
52
|
def available_data_types
|
67
|
-
@available_data_types ||= Hash.new {|hash, key| hash[key] =
|
53
|
+
@available_data_types ||= Hash.new {|hash, key| hash[key] = {}}
|
68
54
|
end
|
69
55
|
|
70
56
|
# A DataExtensionCollection to hold available extensions that
|
@@ -76,172 +62,128 @@ class RFlow
|
|
76
62
|
# A Hash of defined components, usually automatically populated
|
77
63
|
# when a component subclasses RFlow::Component
|
78
64
|
def available_components
|
79
|
-
@available_components ||=
|
65
|
+
@available_components ||= {}
|
80
66
|
end
|
81
|
-
end
|
82
67
|
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
# Add a schema to the available_data_types class attribute.
|
87
|
-
# Schema is indexed by data_type_name and schema/serialization
|
88
|
-
# type. 'avro' is currently the only supported
|
89
|
-
# data_serialization_type.
|
90
|
-
def self.add_available_data_type(data_type_name, data_serialization_type, data_schema)
|
91
|
-
unless data_serialization_type == 'avro'
|
92
|
-
error_message = "Data serialization_type must be 'avro' for '#{data_type_name}'"
|
93
|
-
RFlow.logger.error error_message
|
94
|
-
raise ArgumentError, error_message
|
95
|
-
end
|
68
|
+
# TODO: refactor each of these add_available_* into collections to
|
69
|
+
# make DRYer. Also figure out what to do with all to to_syms
|
96
70
|
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
71
|
+
# Add a schema to the available_data_types class attribute.
|
72
|
+
# Schema is indexed by data_type_name and schema/serialization
|
73
|
+
# type. 'avro' is currently the only supported serialization_type.
|
74
|
+
def add_available_data_type(name, serialization_type, schema)
|
75
|
+
raise ArgumentError, "Data serialization_type must be 'avro' for '#{name}'" unless serialization_type == 'avro'
|
102
76
|
|
103
|
-
|
104
|
-
|
77
|
+
if available_data_types[name.to_s].include? serialization_type.to_s
|
78
|
+
raise ArgumentError, "Data type '#{name}' already defined for serialization_type '#{serialization_type}'"
|
79
|
+
end
|
105
80
|
|
106
|
-
|
107
|
-
# attributes. The data_extension parameter should be the name of
|
108
|
-
# a ruby module that will extend RFlow::Message::Data object to
|
109
|
-
# provide additional methods/capability. Naive, prefix-based
|
110
|
-
# inheritance is possible, see available_data_extensions or the
|
111
|
-
# DataExtensionCollection class
|
112
|
-
def self.add_available_data_extension(data_type_name, data_extension)
|
113
|
-
unless data_extension.is_a? Module
|
114
|
-
error_message = "Invalid data extension #{data_extension} for #{data_type_name}. Only Ruby Modules allowed"
|
115
|
-
RFlow.logger.error error_message
|
116
|
-
raise ArgumentError, error_message
|
81
|
+
available_data_types[name.to_s][serialization_type.to_s] = schema
|
117
82
|
end
|
118
83
|
|
119
|
-
available_data_extensions
|
120
|
-
|
121
|
-
|
84
|
+
# Add a data extension to the available_data_extensions class
|
85
|
+
# attributes. The data_extension parameter should be the name of
|
86
|
+
# a ruby module that will extend RFlow::Message::Data object to
|
87
|
+
# provide additional methods/capability. Naive, prefix-based
|
88
|
+
# inheritance is possible, see available_data_extensions or the
|
89
|
+
# DataExtensionCollection class
|
90
|
+
def add_available_data_extension(data_type_name, extension)
|
91
|
+
unless extension.is_a? Module
|
92
|
+
raise ArgumentError, "Invalid data extension #{extension} for #{data_type_name}. Only Ruby Modules allowed"
|
93
|
+
end
|
122
94
|
|
123
|
-
|
124
|
-
# available component to the list.
|
125
|
-
def self.add_available_component(component)
|
126
|
-
if available_components.include?(component.name)
|
127
|
-
error_message = "Component already '#{component.name}' already defined"
|
128
|
-
RFlow.logger.error error_message
|
129
|
-
raise ArgumentError, error_message
|
95
|
+
available_data_extensions.add data_type_name, extension
|
130
96
|
end
|
131
|
-
available_components[component.name] = component
|
132
|
-
end
|
133
97
|
|
98
|
+
# Used when RFlow::Component is subclassed to add another
|
99
|
+
# available component to the list.
|
100
|
+
def add_available_component(component)
|
101
|
+
if available_components.include?(component.name)
|
102
|
+
raise ArgumentError, "Component already '#{component.name}' already defined"
|
103
|
+
end
|
104
|
+
available_components[component.name] = component
|
105
|
+
end
|
134
106
|
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
end
|
144
|
-
|
107
|
+
# Connect to the configuration sqlite database, but use the
|
108
|
+
# ConfigurationItem class to protect the connection information from
|
109
|
+
# other ActiveRecord apps (i.e. Rails)
|
110
|
+
def establish_config_database_connection(database_path)
|
111
|
+
RFlow.logger.debug "Establishing connection to config database (#{Dir.getwd}) '#{database_path}'"
|
112
|
+
ActiveRecord::Base.logger = RFlow.logger
|
113
|
+
ConfigurationItem.establish_connection(:adapter => "sqlite3", :database => database_path)
|
114
|
+
end
|
145
115
|
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
116
|
+
# Using default ActiveRecord migrations, attempt to migrate the
|
117
|
+
# database to the latest version.
|
118
|
+
def migrate_database
|
119
|
+
RFlow.logger.debug "Applying default migrations to config database"
|
120
|
+
migrations_path = File.join(File.dirname(__FILE__), 'configuration', 'migrations')
|
121
|
+
ActiveRecord::Migration.verbose = false
|
122
|
+
ActiveRecord::Migrator.migrate migrations_path
|
123
|
+
end
|
154
124
|
|
125
|
+
# Load the config file, which should load/process/store all the
|
126
|
+
# elements. Only run this after the database has been setup
|
127
|
+
def process_config_file(path)
|
128
|
+
RFlow.logger.info "Processing config file (#{Dir.getwd}) '#{path}'"
|
129
|
+
load path
|
130
|
+
end
|
155
131
|
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
load config_file_path
|
161
|
-
end
|
132
|
+
# Connect to the configuration database, migrate it to the latest
|
133
|
+
# version, and process a config file if provided.
|
134
|
+
def initialize_database(database_path, config_file_path = nil)
|
135
|
+
RFlow.logger.debug "Initializing config database (#{Dir.getwd}) '#{database_path}'"
|
162
136
|
|
137
|
+
# TODO should not need this line
|
138
|
+
ActiveRecord::Base.establish_connection(:adapter => "sqlite3", :database => database_path)
|
163
139
|
|
164
|
-
|
165
|
-
|
166
|
-
def self.initialize_database(config_database_path, config_file_path=nil)
|
167
|
-
RFlow.logger.debug "Initializing config database (#{Dir.getwd}) '#{config_database_path}'"
|
140
|
+
establish_config_database_connection database_path
|
141
|
+
migrate_database
|
168
142
|
|
169
|
-
|
170
|
-
|
171
|
-
ActiveRecord::Base.establish_connection(:adapter => "sqlite3",
|
172
|
-
:database => config_database_path)
|
143
|
+
working_dir = Dir.getwd
|
144
|
+
Dir.chdir File.dirname(database_path)
|
173
145
|
|
174
|
-
|
146
|
+
if config_file_path
|
147
|
+
process_config_file File.expand_path(config_file_path)
|
148
|
+
end
|
175
149
|
|
176
|
-
|
150
|
+
RFlow.logger.debug "Defaulting non-existing config values"
|
151
|
+
merge_defaults!
|
177
152
|
|
178
|
-
|
179
|
-
Dir.chdir File.dirname(config_database_path)
|
153
|
+
Dir.chdir working_dir
|
180
154
|
|
181
|
-
|
182
|
-
process_config_file(expanded_config_file_path)
|
155
|
+
self.new(database_path)
|
183
156
|
end
|
184
157
|
|
185
|
-
|
186
|
-
merge_defaults!
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
# Make sure that the configuration has all the necessary values set.
|
195
|
-
def self.merge_defaults!
|
196
|
-
Setting::DEFAULTS.each do |name, default_value_or_proc|
|
197
|
-
setting = Setting.find_or_create_by_name(:name => name,
|
198
|
-
:value => default_value_or_proc.is_a?(Proc) ? default_value_or_proc.call() : default_value_or_proc)
|
199
|
-
unless setting.valid?
|
200
|
-
error_message = setting.errors.map do |attribute, error_string|
|
201
|
-
error_string
|
202
|
-
end.join ', '
|
203
|
-
raise RuntimeError, error_message
|
158
|
+
# Make sure that the configuration has all the necessary values set.
|
159
|
+
def merge_defaults!
|
160
|
+
Setting::DEFAULTS.each do |name, default_value_or_proc|
|
161
|
+
value = default_value_or_proc.is_a?(Proc) ? default_value_or_proc.call() : default_value_or_proc
|
162
|
+
setting = Setting.find_or_create_by_name(:name => name, :value => value)
|
163
|
+
unless setting.valid?
|
164
|
+
raise RuntimeError, setting.errors.map {|_, msg| msg }.join(', ')
|
165
|
+
end
|
204
166
|
end
|
205
167
|
end
|
206
168
|
end
|
207
169
|
|
208
|
-
|
209
|
-
attr_accessor :config_database_path
|
210
|
-
attr_accessor :cached_settings
|
211
|
-
attr_accessor :cached_components
|
212
|
-
attr_accessor :cached_ports
|
213
|
-
attr_accessor :cached_connections
|
214
|
-
|
215
|
-
|
216
|
-
def initialize(config_database_path=nil)
|
217
|
-
@cached_settings = Hash.new
|
218
|
-
@cached_components = Hash.new
|
219
|
-
@cached_ports = []
|
220
|
-
@cached_connections = []
|
221
|
-
|
170
|
+
def initialize(database_path = nil)
|
222
171
|
# If there is not a config DB path, assume that an AR
|
223
|
-
#
|
224
|
-
if
|
225
|
-
@
|
226
|
-
|
172
|
+
# connection has already been established
|
173
|
+
if database_path
|
174
|
+
@database_path = database_path
|
175
|
+
Configuration.establish_config_database_connection(database_path)
|
227
176
|
end
|
228
177
|
|
229
|
-
# Validate the connected database.
|
230
|
-
# complete, i.e. validate the various columns
|
178
|
+
# Validate the connected database.
|
179
|
+
# TODO: make this more complete, i.e. validate the various columns
|
231
180
|
begin
|
232
|
-
Setting.first
|
233
|
-
Shard.first
|
234
|
-
Component.first
|
235
|
-
Port.first
|
236
|
-
Connection.first
|
181
|
+
[Setting, Shard, Component, Port, Connection].each(&:first)
|
237
182
|
rescue ActiveRecord::StatementInvalid => e
|
238
|
-
|
239
|
-
RFlow.logger.error error_message
|
240
|
-
raise ArgumentError, error_message
|
183
|
+
raise ArgumentError, "Invalid schema in configuration database: #{e.message}"
|
241
184
|
end
|
242
185
|
end
|
243
186
|
|
244
|
-
|
245
187
|
def to_s
|
246
188
|
string = "Configuration:\n"
|
247
189
|
|
@@ -266,34 +208,13 @@ class RFlow
|
|
266
208
|
string
|
267
209
|
end
|
268
210
|
|
269
|
-
|
270
|
-
def
|
271
|
-
|
272
|
-
end
|
273
|
-
|
274
|
-
def
|
275
|
-
|
276
|
-
end
|
277
|
-
|
278
|
-
def shards
|
279
|
-
Shard.all
|
280
|
-
end
|
281
|
-
|
282
|
-
def shard(shard_uuid)
|
283
|
-
Shard.find_by_uuid shard_uuid
|
284
|
-
end
|
285
|
-
|
286
|
-
def components
|
287
|
-
Component.all
|
288
|
-
end
|
289
|
-
|
290
|
-
def component(component_uuid)
|
291
|
-
Component.find_by_uuid component_uuid
|
292
|
-
end
|
293
|
-
|
294
|
-
def available_components
|
295
|
-
self.class.available_components
|
296
|
-
end
|
211
|
+
def [](name); Setting.find_by_name(name).value rescue nil; end
|
212
|
+
def settings; Setting.all; end
|
213
|
+
def shards; Shard.all; end
|
214
|
+
def shard(uuid); Shard.find_by_uuid uuid; end
|
215
|
+
def components; Component.all; end
|
216
|
+
def component(uuid); Component.find_by_uuid uuid; end
|
217
|
+
def available_components; Configuration.available_components; end
|
297
218
|
end
|
298
219
|
end
|
299
220
|
|