noah 0.4 → 0.6.pre

data/Rakefile

@@ -11,7 +11,6 @@ RSpec::Core::RakeTask.new(:spec) do |spec|
   spec.pattern = FileList['spec/**/*_spec.rb']
 end
 
-
 task :default => :run
 task :test => :run
 
@@ -40,75 +39,9 @@ end
 desc "Populate database with sample dataset"
 task :sample, :redis_url do |t, args|
   require 'noah'
-
   Ohm::connect(:url => args.redis_url)
-  Ohm::redis.flushdb 
-  puts "Creating watchers..."
-  Noah::Watcher.create :endpoint => "dummy://applications", :pattern => "//noah/applications"
-  Noah::Watcher.create :endpoint => "dummy://configurations", :pattern => "//noah/configurations"
-  Noah::Watcher.create :endpoint => "dummy://hosts", :pattern => "//noah/hosts"
-  Noah::Watcher.create :endpoint => "dummy://services", :pattern => "//noah/services"
-  Noah::Watcher.create :endpoint => "dummy://ephemerals", :pattern => "//noah/ephemerals"
-  puts "Creating Host entry for 'localhost'"
-  h = Noah::Host.create(:name => 'localhost', :status => "up")
-  if h.save
-    %w[redis noah].each do |service|
-      puts "Create Service entry for #{service}"
-      s = Noah::Service.create(:name => service, :status => "up", :host => h)
-      h.services << s
-    end
-  end
-
-  puts "Creating Application entry for 'noah'"
-  a = Noah::Application.create(:name => 'noah')
-  if a.save
-    puts "Creating Configuration entry for 'noah'"
-    cr = Noah::Configuration.create(:name => 'redis', :format => 'string', :body => 'redis://127.0.0.1:6379/0', :application => a)
-    ch = Noah::Configuration.create(:name => 'host', :format => 'string', :body => 'localhost', :application  => a)
-    cp = Noah::Configuration.create(:name => 'port', :format => 'string', :body => '9292', :application => a)
-    %w[cr ch cp].each do |c|
-      a.configurations << eval(c)
-    end
-  end
-
-  puts "Creating sample entries - Host and Service"
-  %w[host1.domain.com host2.domain.com host3.domain.com].each do |host|
-    h = Noah::Host.create(:name => host, :status => "up")
-    if h.save
-      %w[http https smtp mysql].each do |service|
-        s = Noah::Service.create(:name => service, :status => "pending", :host => h)
-        h.services << s
-      end
-    end
-  end
-
-  puts "Creating sample entries - Application and Configuration"
-  my_yaml = <<EOY
-  development:
-    database: development_database
-    adapter: mysql
-    username: dev_user
-    password: dev_password
-EOY
-  my_json = <<EOJ
-  {
-    "id":"hostname",
-    "data":"localhost"
-  }
-EOJ
-
-  a1 = Noah::Application.create(:name => 'myrailsapp1')
-  if a1.save
-    c1 = Noah::Configuration.create(:name => 'database.yml', :format => 'yaml', :body => my_yaml, :application => a1)
-    a1.configurations << c1
-  end
-
-  a2 = Noah::Application.create(:name => 'myrestapp1')
-  if a2.save
-    c2 = Noah::Configuration.create(:name => 'config.json', :format => 'json', :body => my_json, :application => a2)
-    a2.configurations << c2
-  end
-  puts "Sample data populated!"
+  sample_data = File.expand_path(File.join("..", "spec", "support","sample_data.rb"),__FILE__)
+  instance_eval(File.open(sample_data).read)
 end
 
 begin

data/lib/noah/agent.rb

@@ -2,6 +2,7 @@ $:.unshift(File.expand_path(File.join(File.dirname(__FILE__), "..", "lib")))
 require 'rubygems'
 require 'noah'
 require 'noah/agents/http_agent'
+require 'noah/agents/https_agent'
 require 'noah/agents/dummy_agent'
 begin
   candidates = []

data/lib/noah/agents/base_agent.rb

@@ -1,9 +1,10 @@
 module Noah::Agents
   class Base
 
-    PREFIX = "base"
+    PREFIX = "base://"
     NAME = "base-agent"
     DEFAULT_CONCURRENCY = 1
+    NEEDS_TRANSFORM = false
 
     def self.inherited(base)
       Noah::Watchers.register_agent(base)
@@ -11,12 +12,13 @@ module Noah::Agents
     end
 
     def notify(event, message, watch_list)
-      logger.info("#{self.class} worker initiated")
+      logger.info("#{self.class.to_s} worker initiated")
       worklist = []
       watch_list.select{|w| worklist << w[:endpoint] if (w[:endpoint] =~ /^#{self.class::PREFIX}/ && event =~ /^#{w[:pattern]}/) }
       if worklist.size >= 1
         logger.info("Dispatching message to #{worklist.size} #{self.class.to_s} endpoints")
         EM::Iterator.new(worklist, self.class::DEFAULT_CONCURRENCY).each do |ep, iter|
+          ep, message = transform(ep, message) if self.class::NEEDS_TRANSFORM == true
           work!(ep, message)
           iter.next
         end

data/lib/noah/agents/http_agent.rb

@@ -5,7 +5,7 @@ module Noah::Agents
 
     PREFIX = "http://"
     NAME = self.class.to_s
-    DEFAULT_CONCURRENCY = 500
+    DEFAULT_CONCURRENCY = 50
 
     def work!(ep, message)
         logger.info("Sending message to (#{ep})")

data/lib/noah/agents/https_agent.rb

@@ -0,0 +1,6 @@
+require File.join(File.dirname(__FILE__), 'http_agent')
+module Noah::Agents
+  class HttpsAgent < HttpAgent
+    PREFIX = "https://"
+  end
+end

data/lib/noah/app.rb

@@ -58,12 +58,14 @@ module Noah
       erb :'500'
     end
 
-    load File.join(File.dirname(__FILE__), 'host_routes.rb')
-    load File.join(File.dirname(__FILE__), 'service_routes.rb')
-    load File.join(File.dirname(__FILE__), 'application_routes.rb')
-    load File.join(File.dirname(__FILE__), 'configuration_routes.rb')
-    load File.join(File.dirname(__FILE__), 'watcher_routes.rb')
-    load File.join(File.dirname(__FILE__), 'ephemeral_routes.rb')
+    load File.join(File.dirname(__FILE__), 'routes/tags.rb')
+    load File.join(File.dirname(__FILE__), 'routes/hosts.rb')
+    load File.join(File.dirname(__FILE__), 'routes/services.rb')
+    load File.join(File.dirname(__FILE__), 'routes/applications.rb')
+    load File.join(File.dirname(__FILE__), 'routes/configurations.rb')
+    load File.join(File.dirname(__FILE__), 'routes/watchers.rb')
+    load File.join(File.dirname(__FILE__), 'routes/ephemerals.rb')
+    load File.join(File.dirname(__FILE__), 'routes/links.rb')
 
   end
 end

data/lib/noah/linkable.rb

@@ -0,0 +1,21 @@
+module Noah::Linkable
+  def self.included(model)
+    model.send :set, :links, ::Noah::Link
+    model.send :index, :links
+  end
+
+  def link!(link_name)
+    link = Noah::Link.find_or_create(:path => link_name)
+    link.nodes = self
+    links << link
+  end
+
+  def unlink!(link_name)
+  end
+
+  def to_hash
+    link_arr = Array.new
+    self.links.sort.each {|l| link_arr << l.path} if self.links.size != 0
+    super.merge(:links => link_arr)
+  end
+end

data/lib/noah/models/applications.rb

@@ -1,30 +1,36 @@
 require File.join(File.dirname(__FILE__), 'configurations')
 module Noah
   class Application < Model
+    include Taggable
+    include Linkable
     attribute :name
-    collection :configurations, Configuration
+    set :configurations, Configuration
 
     index :name
+    index :configurations
 
     def validate
       super
       assert_present :name
+      assert_unique :name
     end
 
     def to_hash
-      arr = []
-      configurations.sort.each {|c| arr << c.to_hash}
-      super.merge(:name => name, :created_at => created_at, :updated_at => updated_at, :configurations => arr)
+      cfg_hash = Hash.new
+      configurations.sort.each do |cfg|
+        cfg_hash["#{cfg.name}"] = {:format => cfg.to_hash[:format], :body => cfg.to_hash[:body]}
+      end
+      {name => {:id => id, :created_at => created_at, :updated_at => updated_at, :configurations => cfg_hash}}
     end
 
     class << self
     def find_or_create(opts = {})
       begin
-        find(opts).first.nil? ? (app = create(opts)) : (app = find(opts).first)
-        if app.valid?
-          app.save
+        find(opts).first.nil? ? (obj = new(opts)) : (obj = find(opts).first)
+        if obj.valid?
+          obj.save
         end
-        app
+        obj
       rescue Exception => e
         e.message
       end
@@ -35,7 +41,11 @@ module Noah
 
   class Applications
     def self.all(options = {})
-      options.empty? ? Application.all.sort : Application.find(options).sort
+      app_hash = Hash.new
+      options.empty? ? apps=Application.all.sort : apps=Application.find(options).sort
+      #apps.each {|x| app_hash["#{x.name}"] = x.to_hash.reject {|k,v| k == :name} }
+      apps.each {|x| app_hash.merge!(x.to_hash) }
+      app_hash
     end
   end
 end

data/lib/noah/models/configurations.rb

@@ -1,11 +1,11 @@
+require File.join(File.dirname(__FILE__), 'applications')
 module Noah
   class Configuration < Model
-
+    include Taggable
+    include Linkable
     attribute :name
     attribute :format
     attribute :body
-    attribute :new_record
-    reference :application, Application
 
     index :name
     index :format
@@ -16,23 +16,39 @@ module Noah
       assert_present :name
       assert_present :format
       assert_present :body
-      assert_present :application_id
-      assert_unique [:name, :application_id]
+      assert_unique :name
     end
 
     def to_hash
-      Application[application_id].nil? ? app_name=nil : app_name=Application[application_id].name
-      super.merge(:name => name, :format => format, :body => body, :created_at => created_at, :updated_at => updated_at, :application => app_name)
+      super.merge(:name => name, :format => format, :body => body, :created_at => created_at, :updated_at => updated_at)
+    end
+
+    # Because we're not doing a 2-way relationship
+    # we need to clean up any applications that use 
+    # this configuration ourself
+    def delete
+      @affected_applications = Array.new
+      Noah::Application.all.each do |app|
+        if app.configurations.member?(self)
+          app.configurations.delete(self)
+          @affected_applications << app.name
+        end
+      end
+      super
+    end
+
+    def affected_applications
+      @affected_applications
     end
 
     class << self
     def find_or_create(opts={})
       begin
-        if find(opts).first.nil?
-          conf = create(opts)
-        else  
-          conf = find(opts).first
-        end  
+        find(opts).first.nil? ? (obj = new(opts)) : (obj = find(opts).first)
+        if obj.valid?
+          obj.save
+        end
+        obj
       rescue Exception => e
         e.message
       end
@@ -43,7 +59,10 @@ module Noah
 
   class Configurations
     def self.all(options = {})
-      options.empty? ? Configuration.all.sort : Configuration.find(options).sort
+      config_hash = Hash.new
+      options.empty? ? configs=Configuration.all.sort : configs=Configuration.find(options).sort
+      configs.each {|x| config_hash["#{x.name}"] = x.to_hash.reject {|k,v| k == :name} }
+      config_hash
     end
   end 
 end

data/lib/noah/models/ephemerals.rb

@@ -1,7 +1,8 @@
 module Noah
   class Ephemeral < Model
     include EphemeralValidations
-
+    include Taggable
+    include Linkable
     attribute :path
     attribute :data
     attribute :lifetime
@@ -28,12 +29,12 @@ module Noah
     def find_or_create(opts = {})
       begin
         path, data = opts[:path], opts[:data]
-        find(:path => path).first.nil? ? (eph = new(:path => path)) : (eph = find(:path => path).first)
-        eph.data = data
-        if eph.valid?
-          eph.save
+        find(:path => path).first.nil? ? (obj = new(:path => path)) : (obj = find(:path => path).first)
+        obj.data = data
+        if obj.valid?
+          obj.save
         end
-        eph
+        obj
       rescue Exception => e
         e.message
       end

data/lib/noah/models/hosts.rb

@@ -3,7 +3,8 @@ module Noah
   class Host < Model
     # Host model
     # @return {Host} a {Host} object
-
+    include Taggable
+    include Linkable
     attribute :name
     attribute :status
     collection :services, Service
@@ -21,9 +22,11 @@ module Noah
 
     # @return [Hash] A hash representation of a {Host}
     def to_hash
-      arr = []
-      services.sort.each {|s| arr << s.to_hash}
-      h = {:name => name, :status => status, :created_at => created_at, :updated_at => updated_at, :services => arr}
+      services_hash = Hash.new
+      services.sort.each do |svc|
+        services_hash["#{svc.name}"] = svc.status
+      end
+      h = {:name => name, :status => status, :created_at => created_at, :updated_at => updated_at, :services => services_hash}
       super.merge(h)
     end
 
@@ -50,7 +53,10 @@ module Noah
     # @param [Hash] optional filters for results
     # @return [Array] Array of {Host} objects
     def self.all(options = {})
-      options.empty? ? Noah::Host.all.sort : Noah::Host.find(options).sort
+      host_hash = Hash.new
+      options.empty? ? hosts=Noah::Host.all.sort : hosts=Noah::Host.find(options).sort
+      hosts.each {|x| host_hash["#{x.name}"] = x.to_hash.reject {|k,v| k == :name } }
+      host_hash
     end
   end
 end

data/lib/noah/models/link.rb

@@ -1,45 +1,86 @@
-require File.join(File.dirname(__FILE__), 'link_member')
 module Noah
   class Link < Model
     attribute :path
-    list :nodes, LinkMember
-    
+    attribute :nodes
+
     index :path
+    index :nodes
+
+    def nodes
+      arr = []
+      self.key[:nodes].smembers.each do |node|
+        arr << node
+      end
+      arr
+    end
+
+    def nodes=(node)
+      case node.class.to_s
+      when "Array"
+        node.each do |n|
+          self.key[:nodes].sadd(n.key)
+          n.links << self
+        end
+      else
+        self.key[:nodes].sadd(node.key)
+        node.links << self
+      end
+    end
 
     def validate
       super
       assert_present :path
     end
-    # Nothing to see yet.
-    # This will be for creating "overlays" or "link" relationships
-    # between arbitrary objects or modeling your data the way you want.
-    #
-    # Example:
-    # path = "/myservers"
-    # path.nodes = ["/applications/myapp","/some/ephemeral/path", "sometag"]
-    #
-    # would result in a structure like:
-    # path/
-    #     applications/
-    #                  myapp
-    #     some/ephemeral/path/
-    #                        child1
-    #                        child2
-    #                        child3
-    #     sometag/
-    #            tagged_item1
-    #            tagged_item2
-    #            tagged_item4
-    #            tagged_item5
-    #
-    # The idea is to create a blended view across opinionated, tagged and
-    # ephemeral nodes.
-    #
-    # Almost a "choose your own model" thing.
+
+    def to_hash
+      # TODO Holy shit, is this messy or what?
+      # Prepopulate instance variables of each object type instead?
+      %w[applications configurations hosts services ephemerals].each {|x| instance_variable_set("@#{x}", Hash.new)}
+      if nodes.size > 0
+        nodes.each do |node|
+          rejected_vals = [:created_at, :updated_at, :links, :name]
+          n = node_to_class(node)
+          cls = class_to_lower(n.class.to_s)
+          hsh = instance_variable_get("@#{cls}s")
+          # all of this bs is because services are unique per [servicename, hostname]
+          # so if I add multiple services just by name to the hash, I'll wipe the previous one
+          # a better option would be for services to be named slug style 
+          hsh["#{n.name}"] = Hash.new unless hsh.has_key?(n.name)
+          case cls
+          when "service"
+            sh = Noah::Host[n.host_id].name
+            hsh["#{n.name}"]["#{sh}"] = Hash.new unless hsh["#{n.name}"].has_key?("#{sh}")
+            hsh["#{n.name}"]["#{sh}"].merge!({:id => n.id, :status => n.status, :tags => n.to_hash[:tags]})
+          when "host"
+            hsh["#{n.name}"].merge!({:id => n.id, :status => n.status, :tags => n.to_hash[:tags], :services => n.to_hash[:services]})
+          else
+            hsh["#{n.name}"].merge!(n.to_hash.reject{|key, value| rejected_vals.member?(key)})
+          end
+        end
+      end
+      h = {:name => name, :hosts => @hosts, :services => @services, :applications => @applications, :configurations => @configurations, :ephemerals => @ephemerals, :created_at => created_at, :updated_at => updated_at}
+      super.merge(h)
+    end
 
     def name
       @name = path
     end
 
+    class <<self
+    def find_or_create(opts={})
+      begin
+        find(opts).first.nil? ? obj=create(opts) : obj=find(opts).first
+        obj
+      rescue Exception => e
+        e.message
+      end
+    end
+    end
+
+    private
+    def node_to_class(node)
+      node.match(/^Noah::(.*):(\d+)$/)
+      Noah.const_get($1).send(:[], $2)
+    end
   end
 end

data/lib/noah/models/services.rb

@@ -1,7 +1,8 @@
 module Noah
 
   class Service < Model
-
+    include Taggable
+    include Linkable
     attribute :name
     attribute :status
     reference :host, Host
@@ -48,7 +49,15 @@ module Noah
 
   class Services
     def self.all(options = {})
-      options.empty? ? Service.all.sort : Service.find(options).sort
+      service_hash = Hash.new
+      options.empty? ? services=Service.all.sort : services=Service.find(options).sort
+      services.each do |svc|
+        service_hash["#{svc.name}"] = Hash.new unless service_hash.has_key?(svc.name)
+        host_name = Noah::Host[svc.host_id].name
+        service_hash["#{svc.name}"]["#{host_name}"] = Hash.new
+        service_hash["#{svc.name}"]["#{host_name}"].merge!({:id => svc.id, :status => svc.status, :tags => svc.to_hash[:tags], :links => svc.to_hash[:links], :created_at => svc.created_at, :updated_at => svc.updated_at})
+      end
+      service_hash
     end
   end
 end

data/lib/noah/models/tags.rb

@@ -1,13 +1,94 @@
 module Noah
+
   class Tag < Model
-    counter :total
+    attribute :name
+    attribute :members
+    index :members
+    index :name
+
+    def validate
+      super
+      assert_present :name
+      assert_unique :name
+    end
+
+    def members=(member)
+      self.key[:members].sadd(member.key)
+      member.tag! self.name unless member.tags.member?(self)
+    end
+
+    def members
+      hsh = Hash.new
+      self.key[:members].smembers.each do |member|
+        n = node_to_class(member)
+        cls = class_to_lower(n.class.to_s)
+        hash_key = "#{cls}s".to_sym
+        hsh[hash_key] = Array.new unless hsh.has_key?(hash_key)
+        hsh[hash_key] << n
+      end
+      hsh
+    end
+
+    def to_hash
+      h = {:name => name, :created_at => created_at, :updated_at => updated_at}
+      h.merge!(members_to_hash)
+      super.merge(h)
+    end
+
+    class <<self
+      def find_or_create(opts={})
+        begin
+          find(opts).first.nil? ? obj=create(opts) : obj=find(opts).first
+          obj
+        rescue Exception => e
+          e.message
+        end
+      end
+    end
+
+    private
+    def node_to_class(node)
+      node.match(/^Noah::(.*):(\d+)$/)
+      Noah.const_get($1).send(:[], $2)
+    end
+
+    def members_to_hash
+      new_hash = Hash.new
+      members.keys.each {|k| new_hash[k] = Hash.new}
+      members.each do |category, member_array|
+        rejected_vals = [:created_at, :updated_at, :tags, :name, :host]
+        h = new_hash[category]
+        member_array.each {|mem| h["#{mem.name}"] = Hash.new unless h.has_key?(mem.name)}
+        case category
+        when :services
+          member_array.each do |s|
+            sh = Noah::Host[s.host_id].name
+            h["#{s.name}"]["#{sh}"] = Hash.new unless h["#{s.name}"].has_key?(sh)
+            h["#{s.name}"]["#{sh}"].merge!(s.to_hash.reject{|k,v| rejected_vals.member?(k)})
+          end
+        else
+          member_array.each do |m|
+            h["#{m.name}"].merge!(m.to_hash.reject{|k,v| rejected_vals.member?(k)})
+          end
+        end
+      end
+      new_hash
+    end
+  end
 
-    def name
-      @name = id
+  class Tags
+    def self.all(options = {})
+      tag_hash = Hash.new
+      options.empty? ? tags=Tag.all.sort : tags=Tag.find(options).sort
+      tags.each {|t| tag_hash["#{t.name}"] = t.to_hash.reject {|k,v| k == :name } }
+      tag_hash
     end
 
-    def self.[](id)
-      super(encode(id)) || create(:id => encode(id))
+    class <<self
+      def tagged(tag)
+        t = Tag.find(:name => tag).first
+        t.members
+      end
     end
   end
 

data/lib/noah/models/watchers.rb

@@ -1,4 +1,3 @@
-require 'digest/sha1'
 module Noah
   class Watcher < Model
     include WatcherValidations