orchestrator 1.0.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (58) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE.md +158 -0
  3. data/README.md +13 -0
  4. data/Rakefile +7 -0
  5. data/app/controllers/orchestrator/api/dependencies_controller.rb +109 -0
  6. data/app/controllers/orchestrator/api/modules_controller.rb +183 -0
  7. data/app/controllers/orchestrator/api/systems_controller.rb +294 -0
  8. data/app/controllers/orchestrator/api/zones_controller.rb +62 -0
  9. data/app/controllers/orchestrator/api_controller.rb +13 -0
  10. data/app/controllers/orchestrator/base.rb +59 -0
  11. data/app/controllers/orchestrator/persistence_controller.rb +29 -0
  12. data/app/models/orchestrator/access_log.rb +35 -0
  13. data/app/models/orchestrator/control_system.rb +160 -0
  14. data/app/models/orchestrator/dependency.rb +87 -0
  15. data/app/models/orchestrator/mod/by_dependency/map.js +6 -0
  16. data/app/models/orchestrator/mod/by_module_type/map.js +6 -0
  17. data/app/models/orchestrator/module.rb +127 -0
  18. data/app/models/orchestrator/sys/by_modules/map.js +9 -0
  19. data/app/models/orchestrator/sys/by_zones/map.js +9 -0
  20. data/app/models/orchestrator/zone.rb +47 -0
  21. data/app/models/orchestrator/zone/all/map.js +6 -0
  22. data/config/routes.rb +43 -0
  23. data/lib/generators/module/USAGE +8 -0
  24. data/lib/generators/module/module_generator.rb +52 -0
  25. data/lib/orchestrator.rb +52 -0
  26. data/lib/orchestrator/control.rb +303 -0
  27. data/lib/orchestrator/core/mixin.rb +123 -0
  28. data/lib/orchestrator/core/module_manager.rb +258 -0
  29. data/lib/orchestrator/core/request_proxy.rb +109 -0
  30. data/lib/orchestrator/core/requests_proxy.rb +47 -0
  31. data/lib/orchestrator/core/schedule_proxy.rb +49 -0
  32. data/lib/orchestrator/core/system_proxy.rb +153 -0
  33. data/lib/orchestrator/datagram_server.rb +114 -0
  34. data/lib/orchestrator/dependency_manager.rb +131 -0
  35. data/lib/orchestrator/device/command_queue.rb +213 -0
  36. data/lib/orchestrator/device/manager.rb +83 -0
  37. data/lib/orchestrator/device/mixin.rb +35 -0
  38. data/lib/orchestrator/device/processor.rb +441 -0
  39. data/lib/orchestrator/device/transport_makebreak.rb +221 -0
  40. data/lib/orchestrator/device/transport_tcp.rb +139 -0
  41. data/lib/orchestrator/device/transport_udp.rb +89 -0
  42. data/lib/orchestrator/engine.rb +70 -0
  43. data/lib/orchestrator/errors.rb +23 -0
  44. data/lib/orchestrator/logger.rb +115 -0
  45. data/lib/orchestrator/logic/manager.rb +18 -0
  46. data/lib/orchestrator/logic/mixin.rb +11 -0
  47. data/lib/orchestrator/service/manager.rb +63 -0
  48. data/lib/orchestrator/service/mixin.rb +56 -0
  49. data/lib/orchestrator/service/transport_http.rb +55 -0
  50. data/lib/orchestrator/status.rb +229 -0
  51. data/lib/orchestrator/system.rb +108 -0
  52. data/lib/orchestrator/utilities/constants.rb +41 -0
  53. data/lib/orchestrator/utilities/transcoder.rb +57 -0
  54. data/lib/orchestrator/version.rb +3 -0
  55. data/lib/orchestrator/websocket_manager.rb +425 -0
  56. data/orchestrator.gemspec +35 -0
  57. data/spec/orchestrator/queue_spec.rb +200 -0
  58. metadata +271 -0
@@ -0,0 +1,35 @@
1
+
2
+ module Orchestrator
3
+ class AccessLog < Couchbase::Model
4
+ design_document :alog
5
+ include ::CouchbaseId::Generator
6
+
7
+
8
+ TTL = Rails.env.production? ? 2.weeks.to_i : 120
9
+
10
+
11
+ belongs_to :user, class_name: "::User"
12
+ attribute :systems, default: lambda { [] }
13
+
14
+ attribute :persisted, default: false
15
+ attribute :suspected, default: false
16
+ attribute :notes
17
+
18
+ attribute :created_at
19
+ attribute :ended_at, default: lambda { Time.now.to_i }
20
+
21
+
22
+ def initialize
23
+ super
24
+ self.created_at = Time.now.to_i
25
+ end
26
+
27
+ def save
28
+ if self.persisted
29
+ super
30
+ else
31
+ super(ttl: TTL)
32
+ end
33
+ end
34
+ end
35
+ end
@@ -0,0 +1,160 @@
1
+ require 'set'
2
+ require 'addressable/uri'
3
+
4
+
5
+ module Orchestrator
6
+ class ControlSystem < Couchbase::Model
7
+ design_document :sys
8
+ include ::CouchbaseId::Generator
9
+
10
+ # Allows us to lookup systems by names
11
+ before_save :update_name
12
+ after_save :expire_cache
13
+
14
+ before_delete :cleanup_modules
15
+ after_delete :expire_cache
16
+
17
+
18
+ attribute :name
19
+ define_attribute_methods :name # dirty attributes for name!
20
+
21
+ attribute :description
22
+
23
+ attribute :zones, default: lambda { [] }
24
+ attribute :modules, default: lambda { [] }
25
+ attribute :settings, default: lambda { {} }
26
+
27
+ attribute :created_at, default: lambda { Time.now.to_i }
28
+
29
+ # Provide a field for simplifying support
30
+ attribute :support_url
31
+
32
+
33
+ def self.find_by_name(name)
34
+ id = ControlSystem.bucket.get("sysname-#{self.name.downcase}", {quiet: true})
35
+ ControlSystem.find_by_id(id) if id
36
+ end
37
+
38
+
39
+ def name=(new_name)
40
+ new_name.strip!
41
+ write_attribute(:name, new_name)
42
+ end
43
+
44
+ def expire_cache(noUpdate = nil)
45
+ ::Orchestrator::System.expire(self.id || @old_id)
46
+ ctrl = ::Orchestrator::Control.instance
47
+
48
+ # If not deleted and control is running
49
+ # then we want to trigger updates on the logic modules
50
+ if !@old_id && noUpdate.nil? && ctrl.ready
51
+ (::Orchestrator::Module.find_by_id(self.modules) || []).each do |mod|
52
+ if mod.control_system_id
53
+ manager = ctrl.loaded? mod.id
54
+ manager.reloaded(mod) if manager
55
+ end
56
+ end
57
+ end
58
+ end
59
+
60
+
61
+ def self.using_module(mod_id)
62
+ by_modules({key: mod_id, stale: false})
63
+ end
64
+ view :by_modules
65
+
66
+ def self.in_zone(zone_id)
67
+ by_zones({key: zone_id, stale: false})
68
+ end
69
+ view :by_zones
70
+
71
+
72
+ # Methods for obtaining the modules and zones as objects
73
+ def module_data
74
+ (::Orchestrator::Module.find_by_id(modules) || []).collect do |mod|
75
+ mod.as_json({
76
+ include: {
77
+ dependency: {
78
+ only: [:name, :module_name]
79
+ }
80
+ }
81
+ })
82
+ end
83
+ end
84
+
85
+ def zone_data
86
+ ::Orchestrator::Zone.find_by_id(zones) || []
87
+ end
88
+
89
+
90
+ protected
91
+
92
+
93
+ # Zones and settings are only required for confident coding
94
+ validates :name, presence: true
95
+ validates :zones, presence: true
96
+
97
+ validate :support_link
98
+
99
+ def support_link
100
+ if self.support_url.nil? || self.support_url.empty?
101
+ self.support_url = nil
102
+ else
103
+ begin
104
+ url = Addressable::URI.parse(self.support_url)
105
+ url.scheme && url.host && url
106
+ rescue
107
+ errors.add(:support_url, 'is an invalid URI')
108
+ end
109
+ end
110
+ end
111
+
112
+ validate :name_unique
113
+
114
+ def name_unique
115
+ return false if self.name.blank?
116
+
117
+ result = ControlSystem.bucket.get("sysname-#{name.downcase}", {quiet: true})
118
+ if result != nil && result != self.id
119
+ errors.add(:name, 'has already been taken')
120
+ end
121
+ end
122
+
123
+ def update_name
124
+ if self.name_changed?
125
+ old_name = self.name_was
126
+ old_name.downcase! if old_name
127
+ elsif not self.exists?
128
+ old_name = false
129
+ else
130
+ return
131
+ end
132
+
133
+ current_name = self.name.downcase
134
+
135
+ if old_name != current_name
136
+ bucket = ControlSystem.bucket
137
+ bucket.delete("sysname-#{old_name}", {quiet: true}) if old_name
138
+ bucket.set("sysname-#{current_name}", self.id)
139
+ end
140
+ end
141
+
142
+ # 1. Find systems that have each of the modules specified
143
+ # 2. If this is the last system we remove the modules
144
+ def cleanup_modules
145
+ ControlSystem.bucket.delete("sysname-#{self.name.downcase}", {quiet: true})
146
+
147
+ self.modules.each do |mod_id|
148
+ systems = ControlSystem.using_module(mod_id).fetch_all
149
+
150
+ if systems.length <= 1
151
+ # We don't use the model's delete method as it looks up control systems
152
+ ::Orchestrator::Control.instance.unload(mod_id)
153
+ ::Orchestrator::Module.bucket.delete(mod_id, {quiet: true})
154
+ end
155
+ end
156
+
157
+ @old_id = self.id # not sure if required
158
+ end
159
+ end
160
+ end
@@ -0,0 +1,87 @@
1
+ require 'set'
2
+
3
+
4
+ module Orchestrator
5
+ class Dependency < Couchbase::Model
6
+ design_document :dep
7
+ include ::CouchbaseId::Generator
8
+
9
+
10
+ after_save :update_modules
11
+ before_delete :cleanup_modules
12
+
13
+
14
+ ROLES = Set.new([:device, :service, :logic])
15
+
16
+
17
+ attribute :name
18
+ attribute :role
19
+ attribute :description
20
+ attribute :default # default data (port or URI)
21
+
22
+ # Override default role accessors
23
+ def role
24
+ @role ||= self.attributes[:role].to_sym if self.attributes[:role]
25
+ end
26
+ def role=(name)
27
+ @role = name.to_sym
28
+ self.attributes[:role] = name
29
+ end
30
+
31
+ attribute :class_name
32
+ attribute :module_name
33
+ attribute :settings, default: lambda { {} }
34
+
35
+ attribute :created_at, default: lambda { Time.now.to_i }
36
+
37
+
38
+ # Find the modules that rely on this dependency
39
+ def modules
40
+ ::Orchestrator::Module.dependent_on(self.id)
41
+ end
42
+
43
+ def default_port=(port)
44
+ self.role = :device
45
+ self.default = port
46
+ end
47
+
48
+ def default_uri=(uri)
49
+ self.role = :service
50
+ self.default = uri
51
+ end
52
+
53
+
54
+ protected
55
+
56
+
57
+ # Validations
58
+ validates :name, presence: true
59
+ validates :class_name, presence: true
60
+ validates :module_name, presence: true
61
+ validate :role_exists
62
+
63
+
64
+ def role_exists
65
+ if self.role && ROLES.include?(self.role.to_sym)
66
+ self.role = self.role.to_s
67
+ else
68
+ errors.add(:role, 'is not valid')
69
+ end
70
+ end
71
+
72
+ # Delete all the module references relying on this dependency
73
+ def cleanup_modules
74
+ modules.each do |mod|
75
+ mod.delete
76
+ end
77
+ end
78
+
79
+ # Reload all modules to update their settings
80
+ def update_modules
81
+ modules.each do |mod|
82
+ manager = ctrl.loaded? mod.id
83
+ manager.reloaded(mod) if manager
84
+ end
85
+ end
86
+ end
87
+ end
@@ -0,0 +1,6 @@
1
+
2
+ function(doc) {
3
+ if(doc.type == "mod" && doc.dependency_id != null) {
4
+ emit(doc.dependency_id, null);
5
+ }
6
+ }
@@ -0,0 +1,6 @@
1
+
2
+ function(doc) {
3
+ if(doc.type === "mod") {
4
+ emit(doc.role, null);
5
+ }
6
+ }
@@ -0,0 +1,127 @@
1
+ require 'addressable/uri'
2
+
3
+
4
+ module Orchestrator
5
+ class Module < Couchbase::Model
6
+ design_document :mod
7
+ include ::CouchbaseId::Generator
8
+
9
+
10
+ # The classes / files that this module requires to execute
11
+ # Defines module type
12
+ # Requires dependency_id to be set
13
+ belongs_to :dependency, :class_name => "Orchestrator::Dependency"
14
+ belongs_to :control_system, :class_name => "Orchestrator::ControlSystem"
15
+
16
+
17
+ # Device module
18
+ def hostname; ip; end
19
+ def hostname=(host); ip = host; end
20
+ attribute :ip
21
+ attribute :tls
22
+ attribute :udp
23
+ attribute :port
24
+ attribute :makebreak, default: false
25
+
26
+ # HTTP Service module
27
+ attribute :uri
28
+
29
+ # Custom module names (in addition to what is defined in the dependency)
30
+ attribute :custom_name
31
+ attribute :settings, default: lambda { {} }
32
+
33
+ attribute :created_at, default: lambda { Time.now.to_i }
34
+ attribute :role # cache the dependency role locally for load order
35
+
36
+ # Connected state in model so we can filter and search on it
37
+ attribute :connected, default: true
38
+ attribute :running, default: false
39
+ attribute :notes
40
+
41
+
42
+ # helper method for looking up the manager
43
+ def manager
44
+ ::Orchestrator::Control.instance.loaded? self.id
45
+ end
46
+
47
+
48
+ # Loads all the modules for this node
49
+ def self.all
50
+ # ascending order by default (device, service then logic)
51
+ by_module_type(stale: false)
52
+ end
53
+ view :by_module_type
54
+
55
+ # Finds all the modules belonging to a particular dependency
56
+ def self.dependent_on(dep_id)
57
+ by_dependency({key: dep_id, stale: false})
58
+ end
59
+ view :by_dependency
60
+
61
+
62
+ protected
63
+
64
+
65
+ validates :dependency, presence: true
66
+ validate :configuration
67
+
68
+ def configuration
69
+ return unless dependency
70
+ case dependency.role
71
+ when :device
72
+ self.role = 1
73
+ self.port = (self.port || dependency.default).to_i
74
+
75
+ errors.add(:ip, 'cannot be blank') if self.ip.blank?
76
+ errors.add(:port, 'is invalid') unless self.port.between?(1, 65535)
77
+
78
+ # Ensure tls and upd values are correct
79
+ # can't have udp + tls
80
+ self.udp = !!self.udp
81
+ if self.udp
82
+ self.tls = false
83
+ else
84
+ self.tls = !!self.tls
85
+ end
86
+
87
+ begin
88
+ url = Addressable::URI.parse("http://#{self.ip}:#{self.port}/")
89
+ url.scheme && url.host && url
90
+ rescue
91
+ errors.add(:ip, 'address / hostname or port are not valid')
92
+ end
93
+ when :service
94
+ self.role = 2
95
+
96
+ self.tls = nil
97
+ self.udp = nil
98
+
99
+ begin
100
+ self.uri ||= dependency.default
101
+ url = Addressable::URI.parse(self.uri)
102
+ url.scheme && url.host && url
103
+ rescue
104
+ errors.add(:uri, 'is an invalid URI')
105
+ end
106
+ else # logic
107
+ self.connected = true # it is connectionless
108
+ self.tls = nil
109
+ self.udp = nil
110
+ self.role = 3
111
+ if control_system.nil?
112
+ errors.add(:control_system, 'must be associated')
113
+ end
114
+ end
115
+ end
116
+
117
+ before_delete :unload_module
118
+ def unload_module
119
+ ::Orchestrator::Control.instance.unload(self.id)
120
+ # Find all the systems with this module ID and remove it
121
+ ControlSystem.using_module(self.id).each do |cs|
122
+ cs.modules.delete(self.id)
123
+ cs.save
124
+ end
125
+ end
126
+ end
127
+ end
@@ -0,0 +1,9 @@
1
+
2
+ function(doc) {
3
+ if(doc.type == "sys") {
4
+ var i;
5
+ for (i = 0; i < doc.modules.length; i++) {
6
+ emit(doc.modules[i], null);
7
+ }
8
+ }
9
+ }
@@ -0,0 +1,9 @@
1
+
2
+ function(doc) {
3
+ if(doc.type == "sys") {
4
+ var i;
5
+ for (i = 0; i < doc.zones.length; i++) {
6
+ emit(doc.zones[i], null);
7
+ }
8
+ }
9
+ }