visionmedia-swarm 0.0.1

data/lib/swarm.rb

@@ -0,0 +1,31 @@
+# (The MIT License)
+# 
+# Copyright (c) 2009 TJ Holowaychuk <tj@vision-media.ca>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# 'Software'), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+# 
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+require 'json'
+require 'dm-core'
+require 'dm-validations'
+require 'sinatra/base'
+require 'user-agent'
+require 'swarm/swarm'
+require 'swarm/job'
+require 'swarm/agent'

data/lib/swarm/agent.rb

@@ -0,0 +1,78 @@
+
+class Agent
+  
+  #--
+  # Mixins
+  #++
+  
+  include DataMapper::Resource
+
+  #--
+  # Properties
+  #++
+  
+  property :id,      Serial
+  property :address, String,  :length => 5..128, :nullable => false
+  property :string,  String,  :index => true,    :length => 5..255, :nullable => false
+  property :busy,    Boolean, :default => false
+  
+  #--
+  # Associations
+  #++
+  
+  has n, :jobs, :through => Resource
+  
+  #--
+  # Instance methods
+  #++
+  
+  ##
+  # Initialize with _string_or_attrs_.
+  
+  def initialize string_or_attrs
+    self.attributes = String === string_or_attrs ?
+      { :string => string_or_attrs } :
+        string_or_attrs 
+  end
+  
+  ##
+  # Check if _other_ equals self. Supports
+  # DataMapper join support when checking AgentJob
+  # equality, as well as against other Agent instances via
+  # their #string value.
+  
+  def == other
+    AgentJob === other ?
+      id == other.agent_id :
+        super
+  end
+  
+  ##
+  # Check if the agent is busy.
+  
+  def busy?
+    busy
+  end
+  
+  #--
+  # Class methods
+  #++
+  
+  ##
+  # Unique agents.
+  
+  def self.unique
+    all.inject [] do |agents, agent|
+      agents << agent unless agents.include? agent
+      agents
+    end
+  end
+  
+  ##
+  # Unique available agents.
+  
+  def self.available
+    unique.delete_if { |agent| agent.busy? }
+  end
+
+end

data/lib/swarm/job.rb

@@ -0,0 +1,68 @@
+
+class Job
+  
+  #--
+  # Mixins
+  #++
+  
+  include DataMapper::Resource
+
+  #--
+  # Properties
+  #++
+  
+  property :id,      Serial
+  property :address, String,  :length => 5..128, :nullable => false
+  property :source,  Text,    :lazy => false
+  property :results, Object,  :lazy => false,    :default => []
+  property :active,  Boolean, :default => true
+  
+  #--
+  # Associations
+  #++
+  
+  has n, :agents, :through => Resource
+  
+  #--
+  # Instance methods
+  #++
+  
+  ##
+  # Check if the job is active.
+  
+  def active?
+    active
+  end
+  
+  ##
+  # Job JSON.
+  
+  def to_json
+    { :id => id, 
+      :source => source,
+      :results => results,
+      :address => address,
+      :active => active }.to_json
+  end
+    
+  #--
+  # Class methods
+  #++
+  
+  ##
+  # Return active job for _agent_.
+  
+  def self.for agent
+    active.find do |job|
+      not agent.jobs.include?(job) || job.agents.any? { |other| agent.string == other.string }
+    end
+  end
+  
+  ##
+  # Return array of active jobs.
+  
+  def self.active
+    all :active => true
+  end
+  
+end

data/lib/swarm/swarm.rb

@@ -0,0 +1,185 @@
+
+class Swarm < Sinatra::Base
+  
+  #--
+  # Constants
+  #++
+  
+  VERSION = '0.0.1'
+  
+  #--
+  # Settings
+  #++
+  
+  enable :methodoverride
+  disable :raise_errors
+  
+  #--
+  # Helpers
+  #++
+  
+  error do
+    env['ACCEPT'] =~ /json/ ?
+      json(:status => 0, :message => env['sinatra.error'].to_s) :
+        env['sinatra.error'].to_s
+  end
+  
+  ##
+  # Log _string_ to stderr.
+  
+  def log string
+    $stderr.puts string
+  end
+  
+  ##
+  # Halts with status of 200 and JSON _object_. 
+  # Sets content type to application/json and jsonifies the object.
+  
+  def json object
+    content_type :json
+    object[:status] ||= 1 if object.is_a? Hash
+    halt 200, object.to_json
+  end
+  
+  ##
+  # User agent.
+  
+  def user_agent
+    env['HTTP_USER_AGENT']
+  end
+  
+  ##
+  # Remote address.
+  
+  def address
+    env['REMOTE_ADDR']
+  end
+  
+  ##
+  # Path to public _file_.
+  
+  def public_path_for file
+    File.dirname(__FILE__) + '/../public/' + file
+  end
+  
+  #--
+  # Agent
+  #++
+  
+  ##
+  # Send public static files.
+  
+  get '/public/*' do |file|
+    send_file public_path_for(file)
+  end
+  
+  ##
+  # Bind a user agent. Reponds with swarm.html
+  # which then continues to poll for jobs.
+  
+  get '/bind/?' do
+    agent = Agent.new :string => user_agent, :address => address
+    raise 'invalid user agent' unless agent.save
+    log "bound: #{agent.inspect}"
+    log "       #{agent.string.inspect}"
+    send_file public_path_for('swarm.html')
+  end
+
+  ##
+  # Get a job. 
+  #
+  # * Loads the current agent
+  # * Finds an active job
+  # * Marks agent as busy
+  # * Responds with job json.
+  #
+  
+  get '/job/?' do
+    if agent = Agent.first(:string => user_agent, :address => address)
+      if job = Job.for(agent)
+        agent.update :busy => true
+        log "running job: #{job.id} with #{agent}"
+        json job
+      end
+    end
+    raise 'failed to find active job'
+  end
+  
+  ##
+  # Get job _id_ as json.
+
+  get '/job/:id.json' do |id|
+    if job = Job.get(id)
+      json job
+    end
+    raise "failed to load job #{id}"
+  end
+  
+  #--
+  # Client
+  #++
+  
+  ##
+  # Get DOM results for job _id_.
+  
+  get '/job/:id/?' do |id|
+    if job = Job.get(id)
+      p job
+      p job.results
+      'TODO: show table / update progress by polling job/:id.json'
+    end
+  end
+  
+  #--
+  # Server
+  #++
+  
+  ##
+  # Push job :source.
+  
+  post '/job/?' do
+    if js = params[:source]
+      job = Job.create :source => js, :address => address
+      log "pushed job: %d %0.3fkb %s" % [job.id, js.length / 1024, job.address]
+      json :message => "job #{job.id} created", :id => job.id
+    end
+    raise ':source required'
+  end
+  
+  ##
+  # Update job _id_ with results.
+  
+  put '/job/:id/?' do |id|
+    if agent = Agent.first(:string => user_agent, :address => address, :busy => true)
+      if job = Job.get(id)
+        job.results << params
+        agent.jobs << job
+        agent.update :busy => false
+        job.save
+        log "updated job: #{job.id} #{params.inspect} with #{agent}"
+        halt 200
+      end
+    end
+    raise "failed to update job #{id}"
+  end
+  
+  #--
+  # Class methods
+  #++
+  
+  ##
+  # Return array of agent instances for _names_.
+
+  def self.agents_for *names
+    unless names.empty?
+      Agent.available.select do |agent|
+        names.any? do |name|
+          agent.name.to_s.downcase == name.to_s.downcase if agent.name
+        end
+      end
+    else
+      Agent.available
+    end
+  end
+  
+end

data/spec/agent_spec.rb

@@ -0,0 +1,57 @@
+
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe Agent do
+  before :each do
+    DataMapper.auto_migrate!
+    @safari = Agent.create :string => 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en) AppleWebKit/526.9 (KHTML, like Gecko) Version/4.0dp1 Safari/526.8'
+    @chrome = Agent.create :string => 'Mozilla/5.0 (X11; U; Linux i686 (x86_64); en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.202.2 Safari/532.0'
+  end
+  
+  describe "#string" do
+    describe "should still work with DataMapper" do
+      it "when using .new" do
+        Agent.new('foo').string.should == 'foo'
+        Agent.new(:string => 'foo').string.should == 'foo'
+        Agent.new(:address => 'foo').address.should == 'foo'
+      end
+      
+      it "when using .create" do
+        Agent.create(:string => 'foo').string.should == 'foo'
+      end
+      
+      it "should #string=" do
+        agent = Agent.new 'foo'
+        agent.string = 'bar'
+        agent.string.should == 'bar'
+      end
+    end
+  end
+  
+  describe "#busy?" do
+    it "should default to false" do
+      Agent.create(:string => 'test').should_not be_busy  
+    end
+    
+    it "should check if the agent is busy" do
+      Agent.create(:string => 'test', :busy => true).should be_busy
+      Agent.create(:string => 'test', :busy => false).should_not be_busy
+    end
+  end
+  
+  describe ".unique" do
+    it "should return all unique agents available" do
+      Agent.stub!(:all).and_return [@chrome, @safari, @chrome, @chrome]
+      Agent.unique.should == [@chrome, @safari]
+    end
+  end
+  
+  describe ".available" do
+    it "should return all available / unique agents" do
+      Agent.stub!(:all).and_return [@chrome, @safari, @chrome, @chrome]
+      Agent.available.should == [@chrome, @safari]
+      @chrome.busy = true
+      Agent.available.should == [@safari]
+    end
+  end
+end

data/spec/job_spec.rb

@@ -0,0 +1,45 @@
+
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe Job do
+  before :each do
+    DataMapper.auto_migrate!
+    @active_job = Job.create :source => 'alert("foo")', :address => '127.0.0.1'
+    @active_job2 = Job.create :source => 'alert("bar")', :address => '127.0.0.1'
+    @inactive_job = Job.create :source => 'alert("baz")', :address => '127.0.0.1', :active => false
+    @safari  = Agent.create :string => 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en) AppleWebKit/526.9 (KHTML, like Gecko) Version/4.0dp1 Safari/526.8', :address => '1.1.1.1' 
+    @safari2 = Agent.create :string => 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en) AppleWebKit/526.9 (KHTML, like Gecko) Version/4.0dp1 Safari/526.8', :address => '2.2.2.2'  
+  end
+  
+  describe "#to_json" do
+    it "should convert a job to JSON" do
+      json = @active_job.to_json
+      json.should include('"source":"alert')
+      json.should include('"results":[]')
+      json.should include('"active":true')
+      json.should include('"address":"127.0.0.1"')
+    end
+  end
+  
+  describe ".active" do
+    it "should return active jobs" do
+      Job.active.should == [@active_job, @active_job2]
+    end
+  end
+  
+  describe ".for" do
+    it "should return an job for the agent passed" do
+      @safari.jobs << @active_job
+      Job.for(@safari).should == @active_job2
+      @safari.jobs << @active_job2
+      Job.for(@safari).should be_nil
+    end
+    
+    it "should only return a job that has not been provided to a similar agent" do
+      @safari.jobs << @active_job << @active_job2
+      @safari.save
+      Job.for(@safari).should be_nil
+      Job.for(@safari2).should be_nil
+    end
+  end
+end