rflow 1.0.0a1 → 1.0.0a2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +2 -0
  3. data/.rspec +1 -0
  4. data/Gemfile +0 -1
  5. data/NOTES +0 -13
  6. data/README.md +6 -1
  7. data/bin/rflow +2 -9
  8. data/example/basic_config.rb +1 -33
  9. data/example/basic_extensions.rb +0 -98
  10. data/example/http_config.rb +2 -3
  11. data/example/http_extensions.rb +6 -63
  12. data/lib/rflow.rb +31 -39
  13. data/lib/rflow/child_process.rb +112 -0
  14. data/lib/rflow/component.rb +77 -148
  15. data/lib/rflow/component/port.rb +38 -41
  16. data/lib/rflow/components.rb +4 -8
  17. data/lib/rflow/components/clock.rb +49 -0
  18. data/lib/rflow/components/integer.rb +39 -0
  19. data/lib/rflow/components/raw.rb +10 -6
  20. data/lib/rflow/components/replicate.rb +20 -0
  21. data/lib/rflow/components/ruby_proc_filter.rb +27 -0
  22. data/lib/rflow/configuration.rb +105 -184
  23. data/lib/rflow/configuration/component.rb +1 -4
  24. data/lib/rflow/configuration/connection.rb +11 -16
  25. data/lib/rflow/configuration/port.rb +3 -5
  26. data/lib/rflow/configuration/ruby_dsl.rb +105 -119
  27. data/lib/rflow/configuration/setting.rb +19 -25
  28. data/lib/rflow/configuration/shard.rb +1 -3
  29. data/lib/rflow/connection.rb +47 -10
  30. data/lib/rflow/connections.rb +0 -1
  31. data/lib/rflow/connections/zmq_connection.rb +34 -38
  32. data/lib/rflow/daemon_process.rb +155 -0
  33. data/lib/rflow/logger.rb +41 -25
  34. data/lib/rflow/master.rb +23 -105
  35. data/lib/rflow/message.rb +78 -108
  36. data/lib/rflow/pid_file.rb +37 -37
  37. data/lib/rflow/shard.rb +33 -100
  38. data/lib/rflow/version.rb +2 -2
  39. data/rflow.gemspec +2 -2
  40. data/schema/tick.avsc +10 -0
  41. data/spec/fixtures/config_ints.rb +4 -40
  42. data/spec/fixtures/config_shards.rb +1 -2
  43. data/spec/fixtures/extensions_ints.rb +0 -98
  44. data/spec/rflow/component/port_spec.rb +61 -0
  45. data/spec/rflow/components/clock_spec.rb +72 -0
  46. data/spec/rflow/configuration/ruby_dsl_spec.rb +150 -0
  47. data/spec/rflow/configuration_spec.rb +54 -0
  48. data/spec/rflow/forward_to_input_port_spec.rb +48 -0
  49. data/spec/rflow/forward_to_output_port_spec.rb +40 -0
  50. data/spec/rflow/logger_spec.rb +48 -0
  51. data/spec/rflow/message/data/raw_spec.rb +29 -0
  52. data/spec/rflow/message/data_spec.rb +58 -0
  53. data/spec/rflow/message_spec.rb +154 -0
  54. data/spec/rflow_spec.rb +94 -124
  55. data/spec/spec_helper.rb +8 -12
  56. metadata +46 -22
  57. data/lib/rflow/components/raw/extensions.rb +0 -18
  58. data/lib/rflow/port.rb +0 -4
  59. data/lib/rflow/util.rb +0 -19
  60. data/spec/rflow_component_port_spec.rb +0 -58
  61. data/spec/rflow_configuration_ruby_dsl_spec.rb +0 -148
  62. data/spec/rflow_configuration_spec.rb +0 -73
  63. data/spec/rflow_message_data_raw.rb +0 -26
  64. data/spec/rflow_message_data_spec.rb +0 -60
  65. data/spec/rflow_message_spec.rb +0 -182
  66. data/spec/schema_spec.rb +0 -28
  67. data/temp.rb +0 -295
@@ -1,14 +1,11 @@
1
1
  class RFlow
2
2
  class Component
3
-
4
3
  # TODO: make this into a class to limit the amount of extensions
5
4
  # that we have to do when operating on these "Arrays", i.e. when
6
5
  # adding two together
7
6
  module ConnectionCollection
8
7
  def send_message(message)
9
- each do |connection|
10
- connection.send_message(message)
11
- end
8
+ each {|connection| connection.send_message(message) }
12
9
  end
13
10
  end
14
11
 
@@ -18,9 +15,9 @@ class RFlow
18
15
  attr_reader :ports, :by_uuid, :by_name, :by_type
19
16
 
20
17
  def initialize
21
- @ports = Array.new
22
- @by_uuid = Hash.new
23
- @by_name = Hash.new
18
+ @ports = []
19
+ @by_uuid = {}
20
+ @by_name = {}
24
21
  @by_type = Hash.new {|hash, key| hash[key.to_s] = []}
25
22
  end
26
23
 
@@ -32,24 +29,19 @@ class RFlow
32
29
  self
33
30
  end
34
31
 
35
-
36
32
  # Enumerate through each connected (or disconnected but
37
33
  # referenced) port
38
34
  # TODO: simplify with enumerators and procs
39
35
  def each
40
- ports.each do |port|
41
- yield port
42
- end
36
+ ports.each {|port| yield port }
43
37
  end
44
38
  end
45
39
 
46
-
47
40
  class Port
48
41
  attr_reader :connected
49
- def connected?; @connected; end
42
+ def connected?; connected; end
50
43
  end
51
44
 
52
-
53
45
  # Allows for a list of connections to be assigned to each port/key
54
46
  # combination. Note that binding an input port to an un-indexed
55
47
  # output port will result in messages from all indexed connections
@@ -57,16 +49,19 @@ class RFlow
57
49
  # result in the same message being sent to all indexed
58
50
  # connections.
59
51
  class HashPort < Port
60
- attr_reader :config, :name, :uuid, :connections_for
52
+ attr_reader :config, :name, :uuid
53
+
54
+ protected
55
+ attr_reader :connections_for
61
56
 
57
+ public
62
58
  def initialize(config)
63
59
  @config = config
64
60
  @name = config.name
65
61
  @uuid = config.uuid
66
- @connections_for = Hash.new {|hash, key| hash[key] = Array.new.extend(ConnectionCollection)}
62
+ @connections_for = Hash.new {|hash, key| hash[key] = [].extend(ConnectionCollection)}
67
63
  end
68
64
 
69
-
70
65
  # Returns an extended Array of all the connections that should
71
66
  # be sent/received on this port. Merges the nil-keyed port
72
67
  # (i.e. any connections for a port without a key) to those
@@ -74,78 +69,80 @@ class RFlow
74
69
  # connections, not to add new ones. Use add_connection to add a
75
70
  # new connection for a given key.
76
71
  def [](key)
77
- (connections_for[key] + connections_for[nil]).extend(ConnectionCollection)
72
+ case key
73
+ when nil; connections_for[nil]
74
+ else connections_for[key] + connections_for[nil]
75
+ end.extend(ConnectionCollection)
78
76
  end
79
77
 
80
-
81
78
  # Adds a connection for a given key
82
79
  def add_connection(key, connection)
83
80
  connections_for[key] << connection
84
81
  end
85
82
 
86
-
87
83
  # Return a list of connected keys
88
84
  def keys
89
85
  connections_for.keys
90
86
  end
91
87
 
92
-
93
88
  # Enumerate through all the ConnectionCollections
94
89
  # TODO: simplify with enumerators and procs
95
90
  def each
96
- connections_for.values.each do |connections|
97
- yield connections
98
- end
91
+ connections_for.values.each {|connections| yield connections }
99
92
  end
100
93
 
101
-
102
- # Send a message to all connections on all keys for this port,
103
- # but only once per connection.
104
94
  def send_message(message)
105
- all_connections.send_message(message)
95
+ def connect!; raise NotImplementedError, "Raw ports do not know how to send messages"; end
106
96
  end
107
97
 
108
-
109
98
  # Should be overridden. Called when it is time to actually
110
99
  # establish the connection
111
100
  def connect!; raise NotImplementedError, "Raw ports do not know which direction to connect"; end
112
101
 
113
102
  private
114
-
115
103
  def all_connections
116
- @all_connections ||= connections_for.map do |port_key, connections|
117
- connections
118
- end.flatten.uniq.extend(ConnectionCollection)
104
+ @all_connections ||= connections_for.values.flatten.uniq.extend(ConnectionCollection)
119
105
  end
120
-
121
106
  end
122
107
 
123
-
124
108
  class InputPort < HashPort
125
109
  def connect!
126
- connections_for.each do |port_key, connections|
110
+ connections_for.each do |key, connections|
127
111
  connections.each do |connection|
128
112
  connection.connect_input!
129
113
  @connected = true
130
114
  end
131
115
  end
132
116
  end
133
- end
134
117
 
118
+ def recv_callback=(callback)
119
+ connections_for.each do |key, connections|
120
+ connections.each do |connection|
121
+ connection.recv_callback = Proc.new do |message|
122
+ callback.call self, key, connection, message
123
+ end
124
+ end
125
+ end
126
+ end
127
+ end
135
128
 
136
129
  class OutputPort < HashPort
137
130
  def connect!
138
- connections_for.each do |port_key, keyed_connections|
139
- keyed_connections.each do |connection|
131
+ connections_for.each do |key, connections|
132
+ connections.each do |connection|
140
133
  connection.connect_output!
141
134
  @connected = true
142
135
  end
143
136
  end
144
137
  end
145
- end
146
138
 
139
+ # Send a message to all connections on all keys for this port,
140
+ # but only once per connection.
141
+ def send_message(message)
142
+ all_connections.send_message(message)
143
+ end
144
+ end
147
145
 
148
146
  class DisconnectedPort < HashPort; end
149
-
150
147
  end
151
148
  end
@@ -1,10 +1,6 @@
1
1
  require 'rflow/component'
2
-
3
- class RFlow
4
- module Components
5
- end
6
- end
7
-
2
+ require 'rflow/components/clock'
3
+ require 'rflow/components/integer'
8
4
  require 'rflow/components/raw'
9
-
10
-
5
+ require 'rflow/components/replicate'
6
+ require 'rflow/components/ruby_proc_filter'
@@ -0,0 +1,49 @@
1
+ class RFlow
2
+ module Components
3
+ class Clock < Component
4
+ module Tick
5
+ SCHEMA_DIRECTORY = ::File.expand_path(::File.join(::File.dirname(__FILE__), '..', '..', '..', 'schema'))
6
+ SCHEMA_FILES = {'tick.avsc' => 'RFlow::Message::Clock::Tick'}
7
+ SCHEMA_FILES.each do |file_name, data_type_name|
8
+ schema_string = ::File.read(::File.join(SCHEMA_DIRECTORY, file_name))
9
+ RFlow::Configuration.add_available_data_type data_type_name, 'avro', schema_string
10
+ end
11
+ module Extension
12
+ def self.extended(base_data); base_data.data_object ||= {}; end
13
+ def name; data_object['name']; end
14
+ def name=(new_name); data_object['name'] = new_name; end
15
+ def timestamp; data_object['timestamp']; end
16
+ def timestamp=(new_ts); data_object['timestamp'] = new_ts; end
17
+ end
18
+ RFlow::Configuration.add_available_data_extension('RFlow::Message::Clock::Tick', Extension)
19
+ end
20
+
21
+ output_port :tick_port
22
+
23
+ DEFAULT_CONFIG = {
24
+ 'name' => 'Clock',
25
+ 'tick_interval' => 1
26
+ }
27
+
28
+ attr_reader :config, :tick_interval
29
+
30
+ def configure!(config)
31
+ @config = DEFAULT_CONFIG.merge config
32
+ @tick_interval = Float(@config['tick_interval'])
33
+ end
34
+
35
+ def clock_name; config['name']; end
36
+
37
+ def run!
38
+ @timer = EventMachine::PeriodicTimer.new(tick_interval) { tick }
39
+ end
40
+
41
+ def tick
42
+ tick_port.send_message(RFlow::Message.new('RFlow::Message::Clock::Tick').tap do |m|
43
+ m.data.name = clock_name
44
+ m.data.timestamp = Integer(Time.now.to_f * 1000) # ms since epoch
45
+ end)
46
+ end
47
+ end
48
+ end
49
+ end
@@ -0,0 +1,39 @@
1
+ class RFlow
2
+ module Components
3
+ Configuration.add_available_data_type('RFlow::Message::Data::Integer', 'avro', '{"type": "long"}')
4
+
5
+ class GenerateIntegerSequence < Component
6
+ output_port :out
7
+ output_port :even_odd_out
8
+
9
+ def configure!(config)
10
+ @start = config['start'].to_i
11
+ @finish = config['finish'].to_i
12
+ @step = config['step'] ? config['step'].to_i : 1
13
+ # If interval seconds is not given, it will default to 0
14
+ @interval_seconds = config['interval_seconds'].to_i
15
+ end
16
+
17
+ # Note that this uses the timer (sometimes with 0 interval) so as
18
+ # not to block the reactor
19
+ def run!
20
+ @timer = EM::PeriodicTimer.new(@interval_seconds) { generate }
21
+ end
22
+
23
+ def generate
24
+ Message.new('RFlow::Message::Data::Integer').tap do |m|
25
+ m.data.data_object = @start
26
+ out.send_message m
27
+ if @start % 2 == 0
28
+ even_odd_out['even'].send_message m
29
+ else
30
+ even_odd_out['odd'].send_message m
31
+ end
32
+ end
33
+
34
+ @start += @step
35
+ @timer.cancel if @start > @finish && @timer
36
+ end
37
+ end
38
+ end
39
+ end
@@ -1,10 +1,17 @@
1
- require 'rflow/components/raw/extensions'
2
-
3
1
  class RFlow
4
2
  module Components
5
3
  module Raw
4
+ module Extensions
5
+ module RawExtension
6
+ def self.extended(base_data)
7
+ base_data.data_object ||= {'raw' => ''}
8
+ end
9
+
10
+ def raw; data_object['raw']; end
11
+ def raw=(new_raw); data_object['raw'] = new_raw; end
12
+ end
13
+ end
6
14
 
7
- # Load the schemas
8
15
  SCHEMA_DIRECTORY = ::File.expand_path(::File.join(::File.dirname(__FILE__), '..', '..', '..', 'schema'))
9
16
 
10
17
  SCHEMA_FILES = {
@@ -16,11 +23,8 @@ class RFlow
16
23
  RFlow::Configuration.add_available_data_type data_type_name, 'avro', schema_string
17
24
  end
18
25
 
19
- # Load the data extensions
20
26
  RFlow::Configuration.add_available_data_extension('RFlow::Message::Data::Raw',
21
27
  RFlow::Components::Raw::Extensions::RawExtension)
22
-
23
-
24
28
  end
25
29
  end
26
30
  end
@@ -0,0 +1,20 @@
1
+ class RFlow
2
+ module Components
3
+ class Replicate < Component
4
+ input_port :in
5
+ output_port :out
6
+ output_port :errored
7
+
8
+ def process_message(input_port, input_port_key, connection, message)
9
+ out.each do |connections|
10
+ begin
11
+ connections.send_message message
12
+ rescue Exception => e
13
+ RFlow.logger.debug "#{self.class} Message caused exception: #{e.class}: #{e.message}: #{e.backtrace}"
14
+ errored.send_message message
15
+ end
16
+ end
17
+ end
18
+ end
19
+ end
20
+ end
@@ -0,0 +1,27 @@
1
+ class RFlow
2
+ module Components
3
+ class RubyProcFilter < Component
4
+ input_port :in
5
+ output_port :filtered
6
+ output_port :dropped
7
+ output_port :errored
8
+
9
+ def configure!(config)
10
+ @filter_proc = eval("lambda {|message| #{config['filter_proc_string']} }")
11
+ end
12
+
13
+ def process_message(input_port, input_port_key, connection, message)
14
+ begin
15
+ if @filter_proc.call(message)
16
+ filtered.send_message message
17
+ else
18
+ dropped.send_message message
19
+ end
20
+ rescue Exception => e
21
+ RFlow.logger.debug "#{self.class} Message caused exception: #{e.class}: #{e.message}: #{e.backtrace}"
22
+ errored.send_message message
23
+ end
24
+ end
25
+ end
26
+ end
27
+ end
@@ -1,7 +1,4 @@
1
- require 'rflow/util'
2
-
3
1
  class RFlow
4
-
5
2
  # Contains all the configuration data and methods for RFlow.
6
3
  # Interacts directly with underlying sqlite database, and keeps a
7
4
  # registry of available data types, extensions, and components.
@@ -13,58 +10,47 @@ class RFlow
13
10
  # subclasses, the controllers are things like RubyDSL, and the views
14
11
  # are defined relative to the controllers
15
12
  class Configuration
16
-
17
- # An exception class
18
- class ConfigurationInvalid < StandardError; end
19
-
20
-
21
- # A class to hold DB config and connection information
22
- class ConfigDB < ActiveRecord::Base
23
- self.abstract_class = true
24
- end
25
-
26
-
27
13
  # A collection class for data extensions that supports a naive
28
14
  # prefix-based 'inheritance' on lookup. When looking up a key
29
15
  # with [] all existing keys will be examined to determine if the
30
16
  # existing key is a string prefix of the lookup key. All the
31
17
  # results are consolidated into a single, flattened array.
32
18
  class DataExtensionCollection
33
-
34
19
  def initialize
35
20
  # TODO: choose a different data structure ...
36
- @hash = Hash.new {|hash, key| hash[key] = Array.new}
21
+ @extensions_for = Hash.new {|hash, key| hash[key] = []}
37
22
  end
38
23
 
39
24
  # Return an array of all of the values that have keys that are
40
25
  # prefixes of the lookup key.
41
26
  def [](key)
42
- key_string = key.to_s
43
- @hash.map do |data_type, extensions|
44
- key_string.start_with?(data_type) ? extensions : nil
45
- end.flatten.compact
27
+ @extensions_for.
28
+ find_all {|data_type, _| key.to_s.start_with?(data_type) }.
29
+ flat_map {|_, extensions| extensions }
46
30
  end
47
31
 
48
32
  # Adds a data extension for a given data type to the collection
49
33
  def add(data_type, extension)
50
- @hash[data_type.to_s] << extension
34
+ @extensions_for[data_type.to_s] << extension
51
35
  end
52
36
 
53
37
  # Remove all elements from the collection. Useful for testing,
54
38
  # not much else
55
39
  def clear
56
- @hash.clear
40
+ @extensions_for.clear
57
41
  end
58
-
59
42
  end
60
43
 
44
+ # Base class for persisted RFlow configuration items.
45
+ class ConfigurationItem < ActiveRecord::Base
46
+ self.abstract_class = true
47
+ end
61
48
 
62
49
  class << self
63
-
64
50
  # A collection of data types (schemas) indexed by their name and
65
51
  # their schema type ('avro').
66
52
  def available_data_types
67
- @available_data_types ||= Hash.new {|hash, key| hash[key] = Hash.new}
53
+ @available_data_types ||= Hash.new {|hash, key| hash[key] = {}}
68
54
  end
69
55
 
70
56
  # A DataExtensionCollection to hold available extensions that
@@ -76,172 +62,128 @@ class RFlow
76
62
  # A Hash of defined components, usually automatically populated
77
63
  # when a component subclasses RFlow::Component
78
64
  def available_components
79
- @available_components ||= Hash.new
65
+ @available_components ||= {}
80
66
  end
81
- end
82
67
 
83
- # TODO: refactor each of these add_available_* into collections to
84
- # make DRYer. Also figure out what to do with all to to_syms
85
-
86
- # Add a schema to the available_data_types class attribute.
87
- # Schema is indexed by data_type_name and schema/serialization
88
- # type. 'avro' is currently the only supported
89
- # data_serialization_type.
90
- def self.add_available_data_type(data_type_name, data_serialization_type, data_schema)
91
- unless data_serialization_type == 'avro'
92
- error_message = "Data serialization_type must be 'avro' for '#{data_type_name}'"
93
- RFlow.logger.error error_message
94
- raise ArgumentError, error_message
95
- end
68
+ # TODO: refactor each of these add_available_* into collections to
69
+ # make DRYer. Also figure out what to do with all to to_syms
96
70
 
97
- if available_data_types[data_type_name.to_s].include? data_serialization_type.to_s
98
- error_message = "Data type '#{data_type_name}' already defined for serialization_type '#{data_serialization_type}'"
99
- RFlow.logger.error error_message
100
- raise ArgumentError, error_message
101
- end
71
+ # Add a schema to the available_data_types class attribute.
72
+ # Schema is indexed by data_type_name and schema/serialization
73
+ # type. 'avro' is currently the only supported serialization_type.
74
+ def add_available_data_type(name, serialization_type, schema)
75
+ raise ArgumentError, "Data serialization_type must be 'avro' for '#{name}'" unless serialization_type == 'avro'
102
76
 
103
- available_data_types[data_type_name.to_s][data_serialization_type.to_s] = data_schema
104
- end
77
+ if available_data_types[name.to_s].include? serialization_type.to_s
78
+ raise ArgumentError, "Data type '#{name}' already defined for serialization_type '#{serialization_type}'"
79
+ end
105
80
 
106
- # Add a data extension to the available_data_extensions class
107
- # attributes. The data_extension parameter should be the name of
108
- # a ruby module that will extend RFlow::Message::Data object to
109
- # provide additional methods/capability. Naive, prefix-based
110
- # inheritance is possible, see available_data_extensions or the
111
- # DataExtensionCollection class
112
- def self.add_available_data_extension(data_type_name, data_extension)
113
- unless data_extension.is_a? Module
114
- error_message = "Invalid data extension #{data_extension} for #{data_type_name}. Only Ruby Modules allowed"
115
- RFlow.logger.error error_message
116
- raise ArgumentError, error_message
81
+ available_data_types[name.to_s][serialization_type.to_s] = schema
117
82
  end
118
83
 
119
- available_data_extensions.add data_type_name, data_extension
120
- end
121
-
84
+ # Add a data extension to the available_data_extensions class
85
+ # attributes. The data_extension parameter should be the name of
86
+ # a ruby module that will extend RFlow::Message::Data object to
87
+ # provide additional methods/capability. Naive, prefix-based
88
+ # inheritance is possible, see available_data_extensions or the
89
+ # DataExtensionCollection class
90
+ def add_available_data_extension(data_type_name, extension)
91
+ unless extension.is_a? Module
92
+ raise ArgumentError, "Invalid data extension #{extension} for #{data_type_name}. Only Ruby Modules allowed"
93
+ end
122
94
 
123
- # Used when RFlow::Component is subclassed to add another
124
- # available component to the list.
125
- def self.add_available_component(component)
126
- if available_components.include?(component.name)
127
- error_message = "Component already '#{component.name}' already defined"
128
- RFlow.logger.error error_message
129
- raise ArgumentError, error_message
95
+ available_data_extensions.add data_type_name, extension
130
96
  end
131
- available_components[component.name] = component
132
- end
133
97
 
98
+ # Used when RFlow::Component is subclassed to add another
99
+ # available component to the list.
100
+ def add_available_component(component)
101
+ if available_components.include?(component.name)
102
+ raise ArgumentError, "Component already '#{component.name}' already defined"
103
+ end
104
+ available_components[component.name] = component
105
+ end
134
106
 
135
- # Connect to the configuration sqlite database, but use the
136
- # ConfigDB subclass to protect the connection information from
137
- # other ActiveRecord apps (i.e. Rails)
138
- def self.establish_config_database_connection(config_database_path)
139
- RFlow.logger.debug "Establishing connection to config database (#{Dir.getwd}) '#{config_database_path}'"
140
- ActiveRecord::Base.logger = RFlow.logger
141
- ConfigDB.establish_connection(:adapter => "sqlite3",
142
- :database => config_database_path)
143
- end
144
-
107
+ # Connect to the configuration sqlite database, but use the
108
+ # ConfigurationItem class to protect the connection information from
109
+ # other ActiveRecord apps (i.e. Rails)
110
+ def establish_config_database_connection(database_path)
111
+ RFlow.logger.debug "Establishing connection to config database (#{Dir.getwd}) '#{database_path}'"
112
+ ActiveRecord::Base.logger = RFlow.logger
113
+ ConfigurationItem.establish_connection(:adapter => "sqlite3", :database => database_path)
114
+ end
145
115
 
146
- # Using default ActiveRecord migrations, attempt to migrate the
147
- # database to the latest version.
148
- def self.migrate_database
149
- RFlow.logger.debug "Applying default migrations to config database"
150
- migrations_directory_path = File.join(File.dirname(__FILE__), 'configuration', 'migrations')
151
- # ActiveRecord::Migration.verbose = RFlow.logger
152
- ActiveRecord::Migrator.migrate migrations_directory_path
153
- end
116
+ # Using default ActiveRecord migrations, attempt to migrate the
117
+ # database to the latest version.
118
+ def migrate_database
119
+ RFlow.logger.debug "Applying default migrations to config database"
120
+ migrations_path = File.join(File.dirname(__FILE__), 'configuration', 'migrations')
121
+ ActiveRecord::Migration.verbose = false
122
+ ActiveRecord::Migrator.migrate migrations_path
123
+ end
154
124
 
125
+ # Load the config file, which should load/process/store all the
126
+ # elements. Only run this after the database has been setup
127
+ def process_config_file(path)
128
+ RFlow.logger.info "Processing config file (#{Dir.getwd}) '#{path}'"
129
+ load path
130
+ end
155
131
 
156
- # Load the config file, which should load/process/store all the
157
- # elements. Only run this after the database has been setup
158
- def self.process_config_file(config_file_path)
159
- RFlow.logger.info "Processing config file (#{Dir.getwd}) '#{config_file_path}'"
160
- load config_file_path
161
- end
132
+ # Connect to the configuration database, migrate it to the latest
133
+ # version, and process a config file if provided.
134
+ def initialize_database(database_path, config_file_path = nil)
135
+ RFlow.logger.debug "Initializing config database (#{Dir.getwd}) '#{database_path}'"
162
136
 
137
+ # TODO should not need this line
138
+ ActiveRecord::Base.establish_connection(:adapter => "sqlite3", :database => database_path)
163
139
 
164
- # Connect to the configuration database, migrate it to the latest
165
- # version, and process a config file if provided.
166
- def self.initialize_database(config_database_path, config_file_path=nil)
167
- RFlow.logger.debug "Initializing config database (#{Dir.getwd}) '#{config_database_path}'"
140
+ establish_config_database_connection database_path
141
+ migrate_database
168
142
 
169
- RFlow.logger.debug "Establishing connection to config database (#{Dir.getwd}) '#{config_database_path}'"
170
- ActiveRecord::Base.logger = RFlow.logger
171
- ActiveRecord::Base.establish_connection(:adapter => "sqlite3",
172
- :database => config_database_path)
143
+ working_dir = Dir.getwd
144
+ Dir.chdir File.dirname(database_path)
173
145
 
174
- migrate_database
146
+ if config_file_path
147
+ process_config_file File.expand_path(config_file_path)
148
+ end
175
149
 
176
- expanded_config_file_path = File.expand_path config_file_path if config_file_path
150
+ RFlow.logger.debug "Defaulting non-existing config values"
151
+ merge_defaults!
177
152
 
178
- working_dir = Dir.getwd
179
- Dir.chdir File.dirname(config_database_path)
153
+ Dir.chdir working_dir
180
154
 
181
- if config_file_path
182
- process_config_file(expanded_config_file_path)
155
+ self.new(database_path)
183
156
  end
184
157
 
185
- RFlow.logger.debug "Defaulting non-existing config values"
186
- merge_defaults!
187
-
188
- Dir.chdir working_dir
189
-
190
- self.new(config_database_path)
191
- end
192
-
193
-
194
- # Make sure that the configuration has all the necessary values set.
195
- def self.merge_defaults!
196
- Setting::DEFAULTS.each do |name, default_value_or_proc|
197
- setting = Setting.find_or_create_by_name(:name => name,
198
- :value => default_value_or_proc.is_a?(Proc) ? default_value_or_proc.call() : default_value_or_proc)
199
- unless setting.valid?
200
- error_message = setting.errors.map do |attribute, error_string|
201
- error_string
202
- end.join ', '
203
- raise RuntimeError, error_message
158
+ # Make sure that the configuration has all the necessary values set.
159
+ def merge_defaults!
160
+ Setting::DEFAULTS.each do |name, default_value_or_proc|
161
+ value = default_value_or_proc.is_a?(Proc) ? default_value_or_proc.call() : default_value_or_proc
162
+ setting = Setting.find_or_create_by_name(:name => name, :value => value)
163
+ unless setting.valid?
164
+ raise RuntimeError, setting.errors.map {|_, msg| msg }.join(', ')
165
+ end
204
166
  end
205
167
  end
206
168
  end
207
169
 
208
-
209
- attr_accessor :config_database_path
210
- attr_accessor :cached_settings
211
- attr_accessor :cached_components
212
- attr_accessor :cached_ports
213
- attr_accessor :cached_connections
214
-
215
-
216
- def initialize(config_database_path=nil)
217
- @cached_settings = Hash.new
218
- @cached_components = Hash.new
219
- @cached_ports = []
220
- @cached_connections = []
221
-
170
+ def initialize(database_path = nil)
222
171
  # If there is not a config DB path, assume that an AR
223
- # conncection has already been established
224
- if config_database_path
225
- @config_database_path = config_database_path
226
- self.class.establish_config_database_connection(config_database_path)
172
+ # connection has already been established
173
+ if database_path
174
+ @database_path = database_path
175
+ Configuration.establish_config_database_connection(database_path)
227
176
  end
228
177
 
229
- # Validate the connected database. TODO: make this more
230
- # complete, i.e. validate the various columns
178
+ # Validate the connected database.
179
+ # TODO: make this more complete, i.e. validate the various columns
231
180
  begin
232
- Setting.first
233
- Shard.first
234
- Component.first
235
- Port.first
236
- Connection.first
181
+ [Setting, Shard, Component, Port, Connection].each(&:first)
237
182
  rescue ActiveRecord::StatementInvalid => e
238
- error_message = "Invalid schema in configuration database: #{e.message}"
239
- RFlow.logger.error error_message
240
- raise ArgumentError, error_message
183
+ raise ArgumentError, "Invalid schema in configuration database: #{e.message}"
241
184
  end
242
185
  end
243
186
 
244
-
245
187
  def to_s
246
188
  string = "Configuration:\n"
247
189
 
@@ -266,34 +208,13 @@ class RFlow
266
208
  string
267
209
  end
268
210
 
269
- # Helper method to access settings with minimal syntax
270
- def [](setting_name)
271
- Setting.find_by_name(setting_name).value rescue nil
272
- end
273
-
274
- def settings
275
- Setting.all
276
- end
277
-
278
- def shards
279
- Shard.all
280
- end
281
-
282
- def shard(shard_uuid)
283
- Shard.find_by_uuid shard_uuid
284
- end
285
-
286
- def components
287
- Component.all
288
- end
289
-
290
- def component(component_uuid)
291
- Component.find_by_uuid component_uuid
292
- end
293
-
294
- def available_components
295
- self.class.available_components
296
- end
211
+ def [](name); Setting.find_by_name(name).value rescue nil; end
212
+ def settings; Setting.all; end
213
+ def shards; Shard.all; end
214
+ def shard(uuid); Shard.find_by_uuid uuid; end
215
+ def components; Component.all; end
216
+ def component(uuid); Component.find_by_uuid uuid; end
217
+ def available_components; Configuration.available_components; end
297
218
  end
298
219
  end
299
220