noah 0.0.5 → 0.0.9

data/Rakefile

@@ -1,48 +1,74 @@
+$:.unshift(File.expand_path(File.join(File.dirname(__FILE__), "lib")))
 require 'bundler'
-begin
-  Bundler.setup(:default, :development)
-rescue Bundler::BundlerError => e
-  $stderr.puts e.message
-  $stderr.puts "Run `bundle install` to install missing gems"
-  exit e.status_code
-end
-require 'rake'
-
 require 'rspec/core'
 require 'rspec/core/rake_task'
 
+REDIS_DIR = File.expand_path(File.join("..", "spec", "support"), __FILE__)
+REDIS_CNF = File.join(REDIS_DIR, "test-redis.conf")
+
 Bundler::GemHelper.install_tasks
+RSpec::Core::RakeTask.new(:spec) do |spec|
+  spec.pattern = FileList['spec/**/*_spec.rb']
+end
+
+
+task :default => :run
+task :test => [:start, :spec, :stop]
+
+desc "Run tests and manage server start/stop"
+task :run => [:start, :spec, :stop]
+
+desc "Start the Redis server"
+task :start do
+  puts "Starting redis-server"
+  system "redis-server #{REDIS_CNF}"
+end
+
+desc "Stop the Redis server"
+task :stop do
+  puts "Killing redis"
+  system "killall -TERM redis-server"
+end
+
+namespace :coverage do
+  desc "Delete aggregate coverage data."
+  task(:clean) { rm_f "coverage.data" }
+end
+desc "Run Rcov code coverage analysis"
+RSpec::Core::RakeTask.new(:coverage) do |t|
+  t.rcov = true
+  t.verbose = true
+  t.rcov_opts = %q[--aggregate coverage.data --sort coverage --text-report --exclude "config,.bundle/*,gems/*,spec/*" -o doc/coverage -Ilib -i "noah.rb"]
+end
 
 desc "Populate database with sample dataset"
 task :sample, :redis_url do |t, args|
-  require 'ohm'
-  begin
-    require 'yajl'
-  rescue LoadError
-    require 'json'
-  end
-  require File.join(File.dirname(__FILE__), 'lib','noah')
+  require 'noah'
 
-  
   Ohm::connect(:url => args.redis_url)
   Ohm::redis.flushdb 
+  puts "Creating watchers..."
+  Noah::Watcher.create :endpoint => "http://localhost:3000/webhook", :pattern => "//noah/application"
+  Noah::Watcher.create :endpoint => "http://localhost:3001/webhook", :pattern => "//noah/configuration"
+  Noah::Watcher.create :endpoint => "http://localhost:3002/webhook", :pattern => "//noah/host"
+  Noah::Watcher.create :endpoint => "http://localhost:3003/webhook", :pattern => "//noah/service"
   puts "Creating Host entry for 'localhost'"
-  h = Host.create(:name => 'localhost', :status => "up")
+  h = Noah::Host.create(:name => 'localhost', :status => "up")
   if h.save
     %w[redis noah].each do |service|
       puts "Create Service entry for #{service}"
-      s = Service.create(:name => service, :status => "up", :host => h)
+      s = Noah::Service.create(:name => service, :status => "up", :host => h)
       h.services << s
     end
   end
 
   puts "Creating Application entry for 'noah'"
-  a = Application.create(:name => 'noah')
+  a = Noah::Application.create(:name => 'noah')
   if a.save
     puts "Creating Configuration entry for 'noah'"
-    cr = Configuration.create(:name => 'redis', :format => 'string', :body => 'redis://127.0.0.1:6379/0', :application => a)
-    ch = Configuration.create(:name => 'host', :format => 'string', :body => 'localhost', :application  => a)
-    cp = Configuration.create(:name => 'port', :format => 'string', :body => '9292', :application => a)
+    cr = Noah::Configuration.create(:name => 'redis', :format => 'string', :body => 'redis://127.0.0.1:6379/0', :application => a)
+    ch = Noah::Configuration.create(:name => 'host', :format => 'string', :body => 'localhost', :application  => a)
+    cp = Noah::Configuration.create(:name => 'port', :format => 'string', :body => '9292', :application => a)
     %w[cr ch cp].each do |c|
       a.configurations << eval(c)
     end
@@ -50,10 +76,10 @@ task :sample, :redis_url do |t, args|
 
   puts "Creating sample entries - Host and Service"
   %w[host1.domain.com host2.domain.com host3.domain.com].each do |host|
-    h = Host.create(:name => host, :status => "up")
+    h = Noah::Host.create(:name => host, :status => "up")
     if h.save
       %w[http https smtp mysql].each do |service|
-        s = Service.create(:name => service, :status => "pending", :host => h)
+        s = Noah::Service.create(:name => service, :status => "pending", :host => h)
         h.services << s
       end
     end
@@ -74,32 +100,35 @@ EOY
   }
 EOJ
 
-  a1 = Application.create(:name => 'myrailsapp1')
+  a1 = Noah::Application.create(:name => 'myrailsapp1')
   if a1.save
-    c1 = Configuration.create(:name => 'database.yml', :format => 'yaml', :body => my_yaml, :application => a1)
+    c1 = Noah::Configuration.create(:name => 'database.yml', :format => 'yaml', :body => my_yaml, :application => a1)
     a1.configurations << c1
   end
 
-  a2 = Application.create(:name => 'myrestapp1')
+  a2 = Noah::Application.create(:name => 'myrestapp1')
   if a2.save
-    c2 = Configuration.create(:name => 'config.json', :format => 'json', :body => my_json, :application => a2)
+    c2 = Noah::Configuration.create(:name => 'config.json', :format => 'json', :body => my_json, :application => a2)
     a2.configurations << c2
   end
   puts "Sample data populated!"
 end
-  
 
-RSpec::Core::RakeTask.new(:spec) do |spec|
-  spec.pattern = FileList['spec/**/*_spec.rb']
+begin
+  require 'yard'
+  require 'yard/sinatra'
+  desc "Generate documentation"
+  YARD::Rake::YardocTask.new do |t|
+    t.files   = ['lib/**/*.rb']   # optional
+    t.options = ['--title', "Noah #{Noah::VERSION} Documentation"]
+    t.options += ['--plugin', "yard-sinatra"]
+    t.options += ['--protected', '--private'] # optional
+  end
+rescue LoadError
+  "You need YARD installed to generate docs"
 end
 
-namespace :coverage do
-  desc "Delete aggregate coverage data."
-  task(:clean) { rm_f "coverage.data" }
-end
-desc "Run Rcov code coverage analysis"
-RSpec::Core::RakeTask.new(:coverage) do |t|
-  t.rcov = true
-  t.verbose = true
-  t.rcov_opts = %q[--aggregate coverage.data --sort coverage --text-report --exclude "config,.bundle/*,gems/*,spec/*" -o doc/coverage -Ilib -i "noah.rb"]
+desc "Demo environment"
+task :start_demo do
+  puts "Soon, young padawan"
 end

data/TODO.md

@@ -0,0 +1,59 @@
+# Brain dump of stuff
+* Documentation
+
+    Need to finish documenting everything for YARD to pick up.
+
+* Stabilize API
+
+    I'm seriously considering moving to a single endpoint with JSON request bodies. Still up in the air. I'm not totally sold on the current layout.
+
+* Write my own CLI class
+
+    I think I might be bumping into some issues related to Vegas as well.
+
+* Watchers
+
+    This is going to be a fun task. I'm serious.
+
+* Ephemeral nodes
+
+    Not sure how I want to implement that. Not too keen on storing them as in-memory hashes. Maybe a LRU in Redis?
+
+* Examples
+
+    I need to make some example apps to really demonstrate what I'm trying to accomplish. 
+
+* Clean up deps
+
+    Feels like I'm relying too much on external libraries.
+
+* Benchmark
+
+    Need some client libs for this
+
+# Done TODOS
+* Github pages
+  - see [[http://lusis.github.com/Noah/]]
+
+* Bundle a war
+  - see [[https://github.com/downloads/lusis/Noah/noah.war]]
+
+* Examples
+  - Sort of done. Demo app is up! [[http://noah-demo.heroku.com/]]. Also see `examples` directory.
+
+* Consider ditching Sinatra::Namespace
+  - Done. It was also causing issues with YARD generation. @rkh confirmed known issue via twitter.
+
+* Namespace models
+  - Done.
+
+* Watchers
+  - Partially done. Framework is in place to create a custom Watcher by hooking directly into Redis. Need to expand that to "official" watchers
+
+
+## Watcher specific stuff
+* Implement a watcher endpoint
+* Implement webhooks
+* Implement AMQP
+* Implement REST
+* Implement JMX on JRuby

data/bin/noah

@@ -1,8 +1,9 @@
 #!/usr/bin/env ruby
 $:.unshift(File.expand_path(File.join(File.dirname(__FILE__), "..", "lib")))
+require 'rubygems'
 require 'noah'
 require 'vegas'
 
 Vegas::Runner.new(Noah::App, 'noah') do |runner, opts, app|
-  opts.on("-r", "--redis URL", "redis url to connect to (default: redis://localhost:6379/0)") {|r| ENV["REDIS_URL"] = r }
+  opts.on("-r", "--redis URL", "redis url to connect to (default: redis://localhost:6379/0)") {|r| ENV["REDIS_URL"] = r; Noah::App.set :redis_url, r }
 end

data/bin/noah-watcher.rb

@@ -0,0 +1,93 @@
+#!/usr/bin/env ruby
+$:.unshift(File.expand_path(File.join(File.dirname(__FILE__), "..", "lib")))
+require 'rubygems'
+require 'rbtrace'
+require 'logger'
+require 'optparse'
+require 'em-hiredis'
+require 'eventmachine'
+require 'em-http-request'
+require 'thin'
+require 'noah'
+require 'json'
+
+LOGGER = Logger.new(STDOUT)
+
+class EventMachine::NoahAgent
+  include EM::Deferrable
+
+  @@watchers = Noah::Watcher.watch_list
+
+  def initialize
+    @logger = LOGGER
+    @logger.debug("Initializing with #{@@watchers.size} registered watches")
+    if EventMachine.reactor_running?
+      @worker = EM.spawn {|event, message, watch_list|
+        logger = LOGGER
+        logger.debug("Worker initiated")
+        logger.info("got event on http worker: #{event}")
+        logger.info("got message on http worker: #{message}")
+        matches = watch_list.find_all{|w| event =~ /^#{Base64.decode64(w)}/}
+        logger.debug("Found #{matches.size} matches for #{event}")
+        EM::Iterator.new(matches).each do |watch, iter|
+          p, ep = Base64.decode64(watch).split("|")
+          logger.info("Sending message to: #{ep} for pattern: #{p}")
+          http = EM::HttpRequest.new(ep, :connection_timeout => 2, :inactivity_timeout => 4).post :body => message
+          http.callback {
+            LOGGER.debug("Message posted to #{ep} successfully")
+            #iter.next
+          }
+          http.errback {
+            LOGGER.debug("Something went wrong")
+            #iter.net
+          }
+          iter.next
+        end
+      }
+    else
+      logger.fatal("Must be inside a reactor!")
+    end
+  end
+
+  def watchers
+    @@watchers.size
+  end
+
+  def reread_watchers
+    @logger.debug("Found new watches")
+    @logger.debug("Current watch count: #{@@watchers.size}")
+    @@watchers = Noah::Watcher.watch_list
+    @logger.debug("New watch count: #{@@watchers.size}")
+    #@logger.debug(@@watchers)
+  end
+
+  def broker(msg)
+    # This is just for testing for now
+    @logger.warn(msg)
+    e,m = msg.split("|")
+    be = Base64.encode64(e).gsub("\n","")
+    @logger.info("Encoded event: #{be}")
+    @worker.notify e, m, @@watchers.clone
+  end
+end
+
+EventMachine.run do
+  logger = LOGGER
+  noah = EventMachine::NoahAgent.new
+  # Passing messages...like a boss
+  master_channel = EventMachine::Channel.new
+
+  r = EventMachine::Hiredis::Client.connect
+  logger.debug("Starting up")
+  r.psubscribe("//noah/*")
+  r.on(:pmessage) do |pattern, event, message|
+    noah.reread_watchers if event =~ /^\/\/noah\/watcher\/.*/
+    master_channel.push "#{event}|#{message}"
+    logger.debug("Saw[#{event}]")
+  end
+
+  sub = master_channel.subscribe {|msg|
+    # We short circuit if we have no watchers
+    noah.broker(msg) unless noah.watchers == 0
+  }
+end

data/config.ru

@@ -1,3 +1,6 @@
-require File.join(File.dirname(__FILE__), 'lib','noah')
-ENV['REDIS_URL'] = "redis://localhost:6379/0"
-run Noah::App
+require 'rubygems'
+require File.join('.', 'lib','noah')
+## Uncomment the following to hardcode a redis url
+ENV['REDIS_URL'] = "redis://localhost:6379/1"
+noah = Noah::App.new
+run noah

data/config/warble.rb

@@ -0,0 +1,18 @@
+# Disable automatic framework detection by uncommenting/setting to false
+# Warbler.framework_detection = false
+
+# Warbler web application assembly configuration file
+Warbler::Config.new do |config|
+  #config.features = %w(gemjar)
+  config.dirs = %w(config lib views)
+  config.includes = FileList["config.ru"]
+  config.excludes = FileList["noah.gemspec", "Gemfile", "Gemfile.lock"]
+  config.bundler = false
+  config.gems += ["json", "ohm", "ohm-contrib", "sinatra", "sinatra-namespace", "haml"]
+  config.gem_excludes = [/^(test|spec)\//]
+  config.public_html = FileList["views/**/*"]
+  config.webxml.booter = :rack
+  #config.webxml.rackup.path = 'WEB-INF/config.ru'
+  #config.webxml.rackup = %{require './lib/noah'; run Noah::App}
+  # config.webxml.rackup = require 'cgi' && CGI::escapeHTML(File.read("config.ru"))
+end

data/examples/README.md

@@ -0,0 +1,116 @@
+# Examples
+The following is a list of notes regarding the examples in this directory.
+
+## General Requirements
+
+You'll need a few additional gems
+
+* [em-hiredis](https://github.com/mloughran/em-hiredis)
+ You'll have to compile/install from source. Sorry. Should pull in the `hiredis` native ext.
+* [em-http-request](https://github.com/igrigorik/em-http-request)
+ Available via rubygems
+* [em-websocket](https://github.com/igrigorik/em-websocket)
+ Available via rubygems
+
+## custom-watcher.rb
+This is an idea I'm tossing around for allowing easy custom watchers to be written.
+Essentially the idea is that you tap into the Redis subscription with a defined pattern and a destination.
+
+### Example
+
+	require './watcher-idea.rb'
+
+	Noah::Watcher.watch do
+	  pattern "//noah/configuration/*"
+	  destination Proc.new {|x| something_with(x)}
+	  run!
+	end
+
+## logger.rb
+An example using logger as a watcher. Pretty straighforward.
+
+
+## httpclient.rb/httpclient-server.rb
+This is an example of how the Webhook system would work
+
+### Running
+
+To get the maximum effect, start with a clean Redis database
+
+* Start the webhook reciever
+
+	noah/examples$ ruby httpclient-server.rb 
+	== Sinatra/1.1.2 has taken the stage on 4567 for development with backup from Thin
+	>> Thin web server (v1.2.7 codename No Hup)
+	>> Maximum connections set to 1024
+	>> Listening on 0.0.0.0:4567, CTRL+C to stop
+
+* Start the webhook publisher
+
+	noah/examples$ ruby httpclient.rb
+
+* Run the rake sample script
+
+In the publisher window, you should see some messages like so:
+
+	Got message for noah.Host[localhost].create
+	Got message for noah.Host[localhost].save
+	Got message for noah.Host[localhost].save
+	Got message for noah.Host[localhost].update
+
+In the server window, you should see the following:
+
+	"{\"id\":\"1\",\"name\":\"localhost\",\"status\":\"up\",\"created_at\":\"2011-02-15 05:19:05 UTC\",\"updated_at\":\"2011-02-15 05:19:05 UTC\",\"services\":[]}"
+	127.0.0.1 - - [15/Feb/2011 00:19:05] "POST /webhook HTTP/1.1" 200 135 0.0024
+	"{\"id\":\"1\",\"name\":\"localhost\",\"status\":\"up\",\"created_at\":\"2011-02-15 05:19:05 UTC\",\"updated_at\":\"2011-02-15 05:19:05 UTC\",\"services\":[]}"
+	127.0.0.1 - - [15/Feb/2011 00:19:05] "POST /webhook HTTP/1.1" 200 135 0.0004
+	"{\"id\":\"1\",\"name\":\"localhost\",\"status\":\"up\",\"created_at\":\"2011-02-15 05:19:05 UTC\",\"updated_at\":\"2011-02-15 05:19:05 UTC\",\"services\":[]}"
+
+
+## websocket.rb
+This is an example of using Websockets, EventMachine and Redis PubSub to provide a "status" console of sorts.
+
+### Running
+
+To get the maximum effect, start with a clean Redis database.
+
+* Start the server:
+
+	~/development/noah/examples$ ./websocket.rb 
+	>> Thin web server (v1.2.7 codename No Hup)
+	>> Maximum connections set to 1024
+	>> Listening on 0.0.0.0:3000, CTRL+C to stop
+
+You should be able to load up the "normal" Noah sample page on [http://localhost:3000].
+
+* Load the "websocket" file
+
+In another browser window, open the `websocket.html` file.
+
+* Send a message
+
+From another terminal window send the following:
+
+	curl -X PUT -d '{"name":"testhost2","status":"down"}' http://localhost:3000/h/testhost2
+
+You should see the message come across in the browser window like so:
+
+	connected...
+
+	2 connected and waiting....
+
+	(noah.Host[testhost2].create) {"id":"1","name":"testhost2","status":"down","created_at":"2011-02-14 20:58:04 UTC","updated_at":"2011-02-14 20:58:04 UTC","services":[]}
+
+	(noah.Host[testhost2].save) {"id":"1","name":"testhost2","status":"down","created_at":"2011-02-14 20:58:04 UTC","updated_at":"2011-02-14 20:58:04 UTC","services":[]}
+
+	(noah.Host[testhost2].save) {"id":"1","name":"testhost2","status":"down","created_at":"2011-02-14 20:58:04 UTC","updated_at":"2011-02-14 20:58:04 UTC","services":[]}
+
+	(noah.Host[testhost2].update) {"id":"1","name":"testhost2","status":"down","created_at":"2011-02-14 20:58:04 UTC","updated_at":"2011-02-14 20:58:04 UTC","services":[]}
+
+You can see the Watcher pattern in the parenthesis and then the JSON message body.
+
+For fun, refresh the page to clear it and then run the sample data population rake task.
+
+### Known issues
+When I started working on the Watcher stuff, I realized that I'm sending A LOT of extranous messages. These are mostly the result of the way I'm creating new objects with Ohm (i.e. via `.create`).
+I'll be cleaning that up and trying to get down to a single message per operation.