octopus 0.0.1

data/.document

@@ -0,0 +1,5 @@
+README.rdoc
+lib/**/*.rb
+bin/*
+features/**/*.feature
+LICENSE

data/.gitignore

@@ -0,0 +1,22 @@
+## MAC OS
+.DS_Store
+
+## TEXTMATE
+*.tmproj
+tmtags
+
+## EMACS
+*~
+\#*
+.\#*
+
+## VIM
+*.swp
+
+## PROJECT::GENERAL
+coverage
+rdoc
+pkg
+
+## PROJECT::SPECIFIC
+*.sqlite3

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 dave@boomer
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,41 @@
+= octopus
+
+This thing is an experimental octopus implementation.
+
+== Installation
+
+
+ sudo apt-get install libsqlite3-ruby libsqlite3-dev ruby rubygems
+ sudo gem install octopus
+
+*Grab the code:*
+
+Anonymous checkout:
+
+ foo: put github stuff here
+
+Commit access:
+
+ git clone git@escapegoat.org:octopus.git
+
+Once you've got the code, just run the octopus like so:
+
+ ruby octopus.rb
+
+Then visit http://localhost:4567 and you can start adding resources for your octopus to grab. There is a resource creation form at http://localhost:4567/new
+
+
+== Note on Patches/Pull Requests
+
+* Fork the project.
+* Make your feature addition or bug fix.
+* Add tests for it. This is important so I don't break it in a
+  future version unintentionally.
+* Commit, do not mess with rakefile, version, or history.
+  (if you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull)
+* Send me a pull request. Bonus points for topic branches.
+
+== Copyright
+
+Copyright (c) 2009 dave@boomer See LICENSE for details.
+

data/Rakefile

@@ -0,0 +1,62 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "octopus"
+    gem.summary = %Q{An experimental octopus implementation.}
+    gem.description = %Q{Grabs stuff off the net and notifies interested subscribers.}
+    gem.email = "dave.hrycyszyn@headlondon.com"
+    gem.homepage = "http://github.com/futurechimp/octopus"
+    gem.authors = ["dave@boomer"]
+    gem.add_dependency "datamapper", ">= 0.10.1"
+    gem.add_dependency "do_sqlite3", ">= 0.10.0"
+    gem.add_dependency "sinatra", ">= 0.9.4"
+    gem.add_dependency "thin", ">= 1.2.5"
+    gem.add_dependency "rack-flash", ">= 0.1.1"
+    gem.add_dependency "rack", ">= 1.0.1"
+    gem.add_development_dependency "thoughtbot-shoulda", ">= 2.10.2"
+    gem.add_development_dependency "rack-test", ">= 0.5.2"
+    gem.add_development_dependency "notahat-machinist", ">= 1.0.3"
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
+end
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+begin
+  require 'rcov/rcovtask'
+  Rcov::RcovTask.new do |test|
+    test.libs << 'test'
+    test.pattern = 'test/**/test_*.rb'
+    test.verbose = true
+  end
+rescue LoadError
+  task :rcov do
+    abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
+  end
+end
+
+task :test => :check_dependencies
+
+task :default => :test
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "octopus #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+

data/VERSION

@@ -0,0 +1 @@
+0.0.1

data/bin/octopus

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+
+# Octopus command line interface script.
+# Run <tt>octopus -h</tt> to get more usage.
+require File.dirname(__FILE__) + '/../lib/octopus'
+require 'thin'
+
+rackup_file = "#{File.dirname(__FILE__)}/../lib/octopus/config.ru"
+
+argv = ARGV
+argv << ["-R", rackup_file] unless ARGV.include?("-R")
+argv << ["-p", "2001"] unless ARGV.include?("-p")
+argv << ["-l", "/home/dave"] unless ARGV.include?("-l")
+argv << ["-e", "production"] unless ARGV.include?("-e")
+Thin::Runner.new(argv.flatten).run!
+

data/lib/ext/array_ext.rb

@@ -0,0 +1,8 @@
+
+# Monkey patches
+class Array
+  def extract_options!
+    last.is_a?(::Hash) ? pop : {}
+  end
+end
+

data/lib/ext/partials.rb

@@ -0,0 +1,27 @@
+# An implementation of partials for Sinatra.
+#
+# Can be used like: <%= partial(:foo) %>
+#
+# Unlike Rails partials, the .erb files for these partials do not start with a
+# leading underscore, i.e. the file name should be foo.rb, not _foo.rb.
+#
+# Liberated from:
+# http://github.com/cschneid/irclogger/blob/master/lib/partials.rb
+#
+module Sinatra
+  module Partials
+    def partial(template, *args)
+      options = args.extract_options!
+      options.merge!(:layout => false)
+      if collection = options.delete(:collection) then
+        collection.inject([]) do |buffer, member|
+          buffer << erb(template, options.merge(:layout =>
+          false, :locals => {template.to_sym => member}))
+      end.join("\n")
+      else
+        erb(template, options)
+      end
+    end
+  end
+end
+

data/lib/ext/reloader.rb

@@ -0,0 +1,11 @@
+# Reload scripts and reset routes on change
+class Sinatra::Reloader < Rack::Reloader
+   def safe_load(file, mtime, stderr = $stderr)
+     if file == __FILE__
+       ::Sinatra::Application.reset!
+       stderr.puts "#{self.class}: resetting routes"
+     end
+     super
+   end
+end
+

data/lib/init.rb

@@ -0,0 +1,68 @@
+# Gems
+#
+require 'rubygems'
+require 'sinatra'
+require 'datamapper'
+require 'ruby-debug'
+require 'eventmachine'
+require 'rack-flash'
+require 'dm-validations'
+
+
+# Extensions to Sinatra
+#
+require File.dirname(__FILE__) + '/ext/partials'
+require File.dirname(__FILE__) + '/ext/array_ext'
+require File.dirname(__FILE__) + '/ext/reloader'
+
+# Octopus code
+#
+require File.dirname(__FILE__) + '/octopus'
+require File.dirname(__FILE__) + '/octopus/models/subscription'
+require File.dirname(__FILE__) + '/octopus/models/net_resource'
+require File.dirname(__FILE__) + '/octopus/grabbers/generic_http'
+
+configure :production do
+  db = "sqlite3:///#{Dir.pwd}/octopus.sqlite3"
+  DataMapper.setup(:default, db)
+end
+
+configure :development do
+  db = "sqlite3:///#{Dir.pwd}/octopus.sqlite3"
+  DataMapper.setup(:default, db)
+end
+
+configure :test do
+  db = "sqlite3::memory:"
+  DataMapper.setup(:default, db)
+end
+
+configure :production, :test, :development do
+  NetResource.auto_migrate! unless NetResource.storage_exists?
+  Subscription.auto_migrate! unless Subscription.storage_exists?
+  DataMapper.auto_upgrade!
+end
+
+# Set the views to the proper path inside the gem
+#
+set :views, File.dirname(__FILE__) + '/octopus/views'
+set :public, File.dirname(__FILE__) + '/octopus/public'
+
+# Register helpers
+#
+helpers do
+  include Sinatra::Partials
+  alias_method :h, :escape_html
+end
+
+# Set up Rack authentication
+#
+use Rack::Auth::Basic do |username, password|
+  [username, password] == ['admin', 'admin']
+end
+
+# Include flash notices
+#
+use Rack::Session::Cookie
+use Rack::Flash
+

data/lib/octopus/config.ru

@@ -0,0 +1,11 @@
+require 'sinatra'
+
+set  :run => false
+set  :environment => :production
+
+disable :reload
+
+run Sinatra::Application
+
+require File.dirname(__FILE__) + '/init'
+

data/lib/octopus/grabbers/generic_http.rb

@@ -0,0 +1,75 @@
+module Grabbers
+  class GenericHttp
+
+    include EM::Protocols
+
+    # Adds a periodic timer to the Eventmachine reactor loop and immediately
+    # starts grabbing expired resources and checking them.
+    #
+    def initialize
+      @@currently_encoding = false
+      puts "Initializing generic http grabber..."
+      EM.add_periodic_timer(5) {
+        check_expired_resources
+      }
+      check_expired_resources
+    end
+
+    # Gets all of the expired NetResources from the database and sends an HTTP
+    # GET requests for each one.  Subscribers to a NetResource will be notified
+    # if it has changed since the last time it was grabbed.
+    #
+    def check_expired_resources
+      net_resources = ::NetResource.expired
+      net_resources.each do |resource|
+        uri = URI.parse(resource.url)
+        conn = HttpClient2.connect uri.host, uri.port
+
+        req = conn.get(uri.path)
+        req.callback{ |response|
+          resource.set_next_update
+          if resource_changed?(resource, response)
+            notify_subscribers(resource)
+            update_changed_resource(resource, response)
+          end
+        }
+      end
+    end
+
+    # Notifies each of a NetResource's subscribers that the resource has changed
+    # by doing an HTTP GET request to the subscriber's callback url.
+    #
+    def notify_subscribers(resource)
+      resource.subscriptions.each do |subscription|
+        uri = URI.parse(subscription.url)
+        conn = HttpClient2.connect uri.host, uri.port
+        req = conn.get(uri.path)
+      end
+    end
+
+    # Determines whether a resource has changed by comparing its saved hash
+    # value with the hash value of the response content.
+    #
+    def resource_changed?(resource, response)
+      changed = false
+      if response.content.hash != resource.last_modified_hash
+        changed = true
+      end
+    end
+
+    # Updates the resource's fields when the resource has changed.  The
+    # last_modified_hash is set to the hash value of the response body, the
+    # response body itself is saved so that it can be sent to consumers during
+    # notifications, and the resource's last_updated time is set to the current
+    # time.
+    #
+    def update_changed_resource(resource, response)
+      resource.last_modified_hash = response.content.hash
+      resource.last_updated = Time.now
+      resource.body = response.content
+      resource.save
+    end
+
+  end
+end
+

data/lib/octopus/models/net_resource.rb

@@ -0,0 +1,68 @@
+# Some kind of resource which we're watching for changes out on the interweb.
+# Could be an RSS feed, a web page, etc.
+#
+class NetResource
+  include DataMapper::Resource
+
+  # Properties
+  #
+  property :id,         Serial
+  property :url,        String,  :required => true, :length => (1..254)
+  property :last_modified_hash, Integer
+  property :update_period, Integer, :required => true, :default => 1200
+  property :next_update, DateTime, :required => true, :default => Time.now
+  property :last_updated, DateTime
+  property :body, Text
+  property :created_at, DateTime
+  property :updated_at, DateTime
+
+  # Associations
+  #
+  has n, :subscriptions
+
+  # Validations
+  #
+  validates_with_method :url, :method => :validate_url
+  validates_with_method :update_period, :method => :validate_update_period
+
+  # Checks that the url property is formatted correctly.
+  #
+  def validate_url
+    begin
+      uri = ::URI.parse(self.url)
+      if uri && uri.scheme == "http" || uri.scheme == "https"
+        return true
+      else
+        return [false, "Url must be properly formatted"]
+      end
+    end
+    rescue ::URI::InvalidURIError
+  end
+
+  # Returns an array of all resources which have a next_update time which is
+  # less than or equal to Time.now.
+  #
+  def self.expired
+    all(:next_update.lte => Time.now)
+  end
+
+  # Sets the next_update time equal to the current time plus the update_period
+  # for this resource.
+  #
+  def set_next_update
+    self.next_update = Time.now + update_period
+    self.save
+  end
+
+  # Ensures that the update_period is not less than 20 seconds.
+  #
+  def validate_update_period
+    unless self.update_period.nil? || self.update_period < 20
+      return true
+    else
+      [false, "Update period must be more than 20 seconds"]
+    end
+  end
+
+end
+

data/lib/octopus/models/subscription.rb

@@ -0,0 +1,35 @@
+# A url callback pointing to a web application or messaging system which wants
+# to be notified about changes to a NetResource.
+#
+class Subscription
+  include DataMapper::Resource
+
+  # Properties
+  property :id,         Serial
+  property :url,        String,  :required => true, :length => (1..254)
+  property :created_at, DateTime
+  property :updated_at, DateTime
+
+  # Associations
+  belongs_to :net_resource
+
+  # Validations
+  validates_with_method :validate_url
+  validates_present :net_resource
+
+  # Checks that the url property is formatted correctly.
+  #
+  def validate_url
+    begin
+      uri = ::URI.parse(self.url)
+      if uri && uri.scheme == "http" || uri.scheme == "https"
+        return true
+      else
+        return [false, "Url must be properly formatted"]
+      end
+    end
+    rescue ::URI::InvalidURIError
+  end
+
+end
+