cloudist 0.0.2 → 0.0.3

data/VERSION

@@ -1 +1 @@
-0.0.2
+0.0.3

data/cloudist.gemspec

@@ -0,0 +1,114 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{cloudist}
+  s.version = "0.0.3"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Ivan Vanderbyl"]
+  s.date = %q{2011-01-18}
+  s.description = %q{Cloudist is a simple, highly scalable job queue for Ruby applications, it can run within Rails, or on EC2, and does not load your entire Rails stack like delayed job does.}
+  s.email = %q{ivanvanderbyl@me.com}
+  s.extra_rdoc_files = [
+    "LICENSE.txt",
+    "README.md"
+  ]
+  s.files = [
+    ".document",
+    ".rspec",
+    "Gemfile",
+    "Gemfile.lock",
+    "LICENSE.txt",
+    "README.md",
+    "Rakefile",
+    "VERSION",
+    "cloudist.gemspec",
+    "examples/queue_message.rb",
+    "examples/sandwich_client.rb",
+    "examples/sandwich_worker.rb",
+    "lib/cloudist.rb",
+    "lib/cloudist/basic_queue.rb",
+    "lib/cloudist/core_ext/string.rb",
+    "lib/cloudist/errors.rb",
+    "lib/cloudist/job.rb",
+    "lib/cloudist/job_queue.rb",
+    "lib/cloudist/listener.rb",
+    "lib/cloudist/payload.rb",
+    "lib/cloudist/publisher.rb",
+    "lib/cloudist/reply_queue.rb",
+    "lib/cloudist/request.rb",
+    "lib/cloudist/utils.rb",
+    "lib/cloudist/worker.rb",
+    "spec/cloudist/basic_queue_spec.rb",
+    "spec/cloudist/job_spec.rb",
+    "spec/cloudist/payload_spec.rb",
+    "spec/cloudist/request_spec.rb",
+    "spec/cloudist_spec.rb",
+    "spec/core_ext/string_spec.rb",
+    "spec/spec_helper.rb"
+  ]
+  s.homepage = %q{http://github.com/ivanvanderbyl/cloudist}
+  s.licenses = ["MIT"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.7}
+  s.summary = %q{Super fast job queue using AMQP}
+  s.test_files = [
+    "examples/queue_message.rb",
+    "examples/sandwich_client.rb",
+    "examples/sandwich_worker.rb",
+    "spec/cloudist/basic_queue_spec.rb",
+    "spec/cloudist/job_spec.rb",
+    "spec/cloudist/payload_spec.rb",
+    "spec/cloudist/request_spec.rb",
+    "spec/cloudist_spec.rb",
+    "spec/core_ext/string_spec.rb",
+    "spec/spec_helper.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<amqp>, [">= 0"])
+      s.add_runtime_dependency(%q<json>, [">= 0"])
+      s.add_runtime_dependency(%q<activesupport>, [">= 0"])
+      s.add_development_dependency(%q<rspec>, ["~> 2.3.0"])
+      s.add_development_dependency(%q<moqueue>, [">= 0"])
+      s.add_development_dependency(%q<mocha>, [">= 0"])
+      s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
+      s.add_development_dependency(%q<jeweler>, ["~> 1.5.2"])
+      s.add_development_dependency(%q<rcov>, [">= 0"])
+      s.add_development_dependency(%q<reek>, ["~> 1.2.8"])
+      s.add_development_dependency(%q<roodi>, ["~> 2.1.0"])
+    else
+      s.add_dependency(%q<amqp>, [">= 0"])
+      s.add_dependency(%q<json>, [">= 0"])
+      s.add_dependency(%q<activesupport>, [">= 0"])
+      s.add_dependency(%q<rspec>, ["~> 2.3.0"])
+      s.add_dependency(%q<moqueue>, [">= 0"])
+      s.add_dependency(%q<mocha>, [">= 0"])
+      s.add_dependency(%q<bundler>, ["~> 1.0.0"])
+      s.add_dependency(%q<jeweler>, ["~> 1.5.2"])
+      s.add_dependency(%q<rcov>, [">= 0"])
+      s.add_dependency(%q<reek>, ["~> 1.2.8"])
+      s.add_dependency(%q<roodi>, ["~> 2.1.0"])
+    end
+  else
+    s.add_dependency(%q<amqp>, [">= 0"])
+    s.add_dependency(%q<json>, [">= 0"])
+    s.add_dependency(%q<activesupport>, [">= 0"])
+    s.add_dependency(%q<rspec>, ["~> 2.3.0"])
+    s.add_dependency(%q<moqueue>, [">= 0"])
+    s.add_dependency(%q<mocha>, [">= 0"])
+    s.add_dependency(%q<bundler>, ["~> 1.0.0"])
+    s.add_dependency(%q<jeweler>, ["~> 1.5.2"])
+    s.add_dependency(%q<rcov>, [">= 0"])
+    s.add_dependency(%q<reek>, ["~> 1.2.8"])
+    s.add_dependency(%q<roodi>, ["~> 2.1.0"])
+  end
+end
+

data/examples/queue_message.rb

@@ -2,8 +2,6 @@ $:.unshift File.dirname(__FILE__) + '/../lib'
 require "rubygems"
 require "cloudist"
 
-ENV["AMQP_URL"] = 'amqp://test_pilot:t35t_p1l0t!@ec2-50-16-134-211.compute-1.amazonaws.com:5672/'
-
 ::Signal.trap('INT') { Cloudist.stop }
 ::Signal.trap('TERM'){ Cloudist.stop }
 

data/examples/sandwich_client.rb

@@ -1,7 +1,20 @@
+# Cloudst Example: Sandwich Client
+# 
+# This example demonstrates dispatching a job to the worker and receiving event callbacks.
+# 
+# Be sure to update the Cloudist connection settings if they differ from defaults:
+# user: guest
+# pass: guest
+# port: 5672
+# host: localhost
+# vhost: /
+#
 $:.unshift File.dirname(__FILE__) + '/../lib'
 require "rubygems"
 require "cloudist"
 
+ENV['AMQP_URL'] = 'amqp://test_pilot:t35t_p1l0t!@ec2-50-16-8-137.compute-1.amazonaws.com:5672/ivan'
+
 Cloudist.signal_trap!
 
 Cloudist.start {
@@ -11,7 +24,8 @@ Cloudist.start {
   
   # Listen to all sandwich jobs
   listen('make.sandwich') {
-    Cloudist.log.info("Make sandwich event: #{id}")
+    # Cloudist.log.info("Make sandwich event: #{data[:event]}")
+    Cloudist.log.debug(queue_header.headers.inspect)
   }
   
 }

data/examples/sandwich_worker.rb

@@ -1,7 +1,21 @@
+# Cloudst Example: Sandwich Worker
+# 
+# This example demonstrates receiving a job and sending back events to the client to let it know we've started and finsihed
+# making a sandwich. From here you could dispatch an eat.sandwich event.
+# 
+# Be sure to update the Cloudist connection settings if they differ from defaults:
+# user: guest
+# pass: guest
+# port: 5672
+# host: localhost
+# vhost: /
+# 
 $:.unshift File.dirname(__FILE__) + '/../lib'
 require "rubygems"
 require "cloudist"
 
+ENV['AMQP_URL'] = 'amqp://test_pilot:t35t_p1l0t!@ec2-50-16-8-137.compute-1.amazonaws.com:5672/ivan'
+
 Cloudist.signal_trap!
 
 Cloudist.start {
@@ -10,12 +24,15 @@ Cloudist.start {
   worker {
     job('make.sandwich') {
       # Fire the started event
-      started!
       
       log.info("JOB (#{id}) Make sandwich with #{data[:bread]} bread")
       log.debug(data.inspect)
       
-      finished!
+      EM.defer {
+        started!
+        sleep(5)
+        finished!
+      }
     }    
   }
 }

data/lib/cloudist.rb

@@ -1,5 +1,5 @@
 require 'uri'
-require 'json'
+require 'json' unless defined? ActiveSupport::JSON
 require "active_support/hash_with_indifferent_access"
 require "amqp"
 require "mq"

data/lib/cloudist/basic_queue.rb

@@ -51,7 +51,7 @@ module Cloudist
           raise Cloudist::ExpiredMessage if request.expired?
           yield request if block_given?
           finished = Time.now.utc.to_i
-
+          
         rescue Cloudist::ExpiredMessage
           log.info "amqp_message action=timeout #{tag} ttl=#{request.ttl} age=#{request.age} #{request.inspect}"
           request.ack if amqp_opts[:ack]
@@ -76,6 +76,7 @@ module Cloudist
       body, headers = payload.formatted
       q.publish(body, headers)
       payload.publish
+      return headers
     end
     
     def teardown

data/lib/cloudist/job.rb

@@ -21,25 +21,38 @@ module Cloudist
       
     end
     
-    def reply(data, headers = {})
-      # headers.update(:message_id => payload.headers[:message_id])
+    def reply(data, headers = {}, options = {})
+      options = {
+        :echo => false
+      }.update(options)
+      
       headers = {
         :message_id => payload.headers[:message_id],
-        :reply_type => "reply"
+        :message_type => "reply"
       }.update(headers)
       
+      # Echo the payload back
+      data.merge!(payload.body) if options[:echo] == true
+      
       reply_payload = Payload.new(data, headers)
       
       reply_queue = ReplyQueue.new(payload.reply_to)
       reply_queue.setup
-      reply_queue.publish_to_q(reply_payload)
+      published_headers = reply_queue.publish_to_q(reply_payload)
       
-      # log.debug("Replying: #{data.inspect} - Payload: #{reply_payload.inspect}")
+      log.debug("Replying: #{data.inspect} - Headers: #{published_headers.inspect}")
+    end
+    
+    # Sends a progress update
+    # Inputs: percentage - Integer
+    # Optional description, this could be displayed to the user e.g. Resizing image
+    def progress(percentage, description = nil)
+      reply({:progress => percentage, :description => description}, {:message_type => 'progress'})
     end
     
-    def event(event_name, data = {})
-      data = {} unless data
-      reply({:event => event_name}.merge(payload.body), {:reply_type => "event"})
+    def event(event_name, event_data = {}, options = {})
+      event_data = {} if event_data.nil?
+      reply(event_data, {:event => event_name, :message_type => 'event'}, options)
     end
     
     def method_missing(meth, *args, &blk)

data/lib/cloudist/job_queue.rb

@@ -15,7 +15,7 @@ module Cloudist
     end
 
     def subscribe(amqp_opts={}, opts={})
-      amqp_opts[:ack] = true
+      amqp_opts[:ack] ||= true
       super(amqp_opts, opts) do |request|
         begin
           yield request if block_given?
@@ -25,4 +25,4 @@ module Cloudist
       end
     end
   end
-end
+end

data/lib/cloudist/listener.rb

@@ -19,10 +19,9 @@ module Cloudist
       reply_queue = Cloudist::ReplyQueue.new(job_queue_name)
       reply_queue.setup(job_id) if job_id
       
-      reply_queue.subscribe do |request|
-        job = Job.new(request.payload)
-        
-        job.instance_eval(&block)
+      reply_queue.subscribe do |request|        
+        # job = Job.new(request.payload)
+        request.instance_eval(&block)
       end
     end
     

data/lib/cloudist/payload.rb

@@ -4,22 +4,25 @@ module Cloudist
   class Payload
     include Utils
     
-    attr_accessor :body, :headers
+    attr_reader :body, :publish_opts, :headers
 
-    def initialize(data_hash_or_json, headers = {})
+    def initialize(data_hash_or_json, headers = {}, publish_opts = {})
+      @publish_opts, @headers = publish_opts, headers
+      @published = false
+      
       data_hash_or_json = parse_message(data_hash_or_json) if data_hash_or_json.is_a?(String)
       
       raise Cloudist::BadPayload, "Expected Hash for payload" unless data_hash_or_json.is_a?(Hash)
 
-      @body, @headers = HashWithIndifferentAccess.new(data_hash_or_json), headers
+      @body = HashWithIndifferentAccess.new(data_hash_or_json)
       update_headers
     end
 
+    # Return message formatted as JSON and headers ready for transport in array
     def formatted
-      body, headers = apply_custom_headers
-
-      # Return message formatted as JSON and headers ready for transport
-      [body.to_json, headers]
+      update_headers
+      
+      [body.to_json, publish_opts]
     end
     
     def id
@@ -41,8 +44,12 @@ module Cloudist
     end
     
     def update_headers
+      headers = extract_custom_headers
+      (publish_opts[:headers] ||= {}).merge!(headers)
+    end
+    
+    def extract_custom_headers
       raise StaleHeadersError, "Headers cannot be changed because payload has already been published" if published?
-      
       headers[:published_on] ||= body.delete('published_on') || Time.now.utc.to_i
       headers[:ttl] ||= body.delete('ttl') || Cloudist::DEFAULT_TTL
 
@@ -54,15 +61,17 @@ module Cloudist
       
       # We use JSON for message transport exclusively
       headers[:content_type] ||= 'application/json'
-
+      
+      # headers[:headers][:message_type] = 'event'
+       # ||= body.delete('message_type') || 'reply'
+      
+      # headers[:headers] = custom_headers
+      
       # some strange behavior with integers makes it better to
       # convert all amqp headers to strings to avoid any problems
       headers.each { |k,v| headers[k] = v.to_s }
-    end
-    
-    def apply_custom_headers
-      update_headers
-      [body, headers]
+      
+      headers
     end
 
     def parse_custom_headers
@@ -107,7 +116,7 @@ module Cloudist
     
     def parse_message(raw)
       return { } unless raw
-      JSON.parse(raw)
+      decode_json(raw)
     end
     
     def [](key)

data/lib/cloudist/reply_queue.rb

@@ -2,6 +2,7 @@ module Cloudist
   class ReplyQueue < Cloudist::BasicQueue
     def initialize(queue_name, opts={})
       opts[:auto_delete] = true
+      opts[:nowait] = false
       super
     end
 

data/lib/cloudist/request.rb

@@ -5,9 +5,9 @@ module Cloudist
     def initialize(queue, json_encoded_message, queue_header)
       @qobj, @queue_header = queue, queue_header
 
-      @payload = Cloudist::Payload.new(json_encoded_message.dup, queue_header.properties.dup)
+      @payload = Cloudist::Payload.new(json_encoded_message.dup, queue_header.headers.dup)
       @headers = @payload.parse_custom_headers
-      
+
       @start = Time.now.utc.to_i
     end
 

data/lib/cloudist/utils.rb

@@ -32,5 +32,14 @@ module Cloudist
       ]
       "%04x%04x%04x%04x%04x%06x%06x" % values
     end
+    
+    def decode_json(string)
+      if defined? ActiveSupport::JSON
+        ActiveSupport::JSON.decode string
+      else
+        JSON.load string
+      end
+    end
+    
   end
 end

data/spec/cloudist/payload_spec.rb

@@ -15,28 +15,25 @@ describe Cloudist::Payload do
   
   it "should prepare headers" do
     payload = Cloudist::Payload.new({:bread => 'white'})
-    body, headers = payload.apply_custom_headers
-    body.should == {"bread"=>"white"}
-    headers.has_key?(:ttl).should be_true
-    headers.has_key?(:content_type).should be_true
-    headers[:content_type].should == "application/json"
-    headers.has_key?(:published_on).should be_true
-    headers.has_key?(:event_hash).should be_true
-    headers.has_key?(:message_id).should be_true
+    payload.body.should == {"bread"=>"white"}
+    payload.headers.has_key?(:ttl).should be_true
+    payload.headers.has_key?(:content_type).should be_true
+    payload.headers[:content_type].should == "application/json"
+    payload.headers.has_key?(:published_on).should be_true
+    payload.headers.has_key?(:event_hash).should be_true
+    payload.headers.has_key?(:message_id).should be_true
   end
   
   it "should extract published_on from data" do
     payload = Cloudist::Payload.new({:bread => 'white', :published_on => 12345678})
-    body, headers = payload.apply_custom_headers
-    body.should == {"bread"=>"white"}
-    headers[:published_on].should == "12345678"
+    payload.body.should == {"bread"=>"white"}
+    payload.headers[:published_on].should == "12345678"
   end
   
   it "should extract custom event hash from data" do
     payload = Cloudist::Payload.new({:bread => 'white', :event_hash => 'foo'})
-    body, headers = payload.apply_custom_headers
-    body.should == {"bread"=>"white"}
-    headers[:event_hash].should == "foo"
+    payload.body.should == {"bread"=>"white"}
+    payload.headers[:event_hash].should == "foo"
   end
   
   it "should parse JSON message" do
@@ -67,10 +64,13 @@ describe Cloudist::Payload do
   end
   
   it "should format payload for sending" do
-    payload = Cloudist::Payload.new({:bread => 'white'}, {:event_hash => 'foo'})
-    json, headers = payload.formatted
+    payload = Cloudist::Payload.new({:bread => 'white'}, {:event_hash => 'foo', :message_type => 'reply'})
+    json, popts = payload.formatted
+    headers = popts[:headers]
+    
     json.should == "{\"bread\":\"white\"}"
     headers[:ttl].should == "300"
+    headers[:message_type].should == 'reply'
   end
   
   it "should generate a unique payload ID" do

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: cloudist
 version: !ruby/object:Gem::Version 
-  hash: 27
+  hash: 25
   prerelease: false
   segments: 
   - 0
   - 0
-  - 2
-  version: 0.0.2
+  - 3
+  version: 0.0.3
 platform: ruby
 authors: 
 - Ivan Vanderbyl
@@ -15,14 +15,11 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-01-16 00:00:00 +11:00
+date: 2011-01-18 00:00:00 +11:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
-  type: :runtime
-  prerelease: false
-  name: amqp
-  version_requirements: &id001 !ruby/object:Gem::Requirement 
+  requirement: &id001 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -31,12 +28,12 @@ dependencies:
         segments: 
         - 0
         version: "0"
-  requirement: *id001
-- !ruby/object:Gem::Dependency 
   type: :runtime
+  name: amqp
   prerelease: false
-  name: json
-  version_requirements: &id002 !ruby/object:Gem::Requirement 
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  requirement: &id002 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -45,12 +42,12 @@ dependencies:
         segments: 
         - 0
         version: "0"
-  requirement: *id002
-- !ruby/object:Gem::Dependency 
   type: :runtime
+  name: json
   prerelease: false
-  name: activesupport
-  version_requirements: &id003 !ruby/object:Gem::Requirement 
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  requirement: &id003 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -59,12 +56,12 @@ dependencies:
         segments: 
         - 0
         version: "0"
-  requirement: *id003
-- !ruby/object:Gem::Dependency 
-  type: :development
+  type: :runtime
+  name: activesupport
   prerelease: false
-  name: rspec
-  version_requirements: &id004 !ruby/object:Gem::Requirement 
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  requirement: &id004 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
@@ -75,12 +72,12 @@ dependencies:
         - 3
         - 0
         version: 2.3.0
-  requirement: *id004
-- !ruby/object:Gem::Dependency 
   type: :development
+  name: rspec
   prerelease: false
-  name: moqueue
-  version_requirements: &id005 !ruby/object:Gem::Requirement 
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  requirement: &id005 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -89,12 +86,12 @@ dependencies:
         segments: 
         - 0
         version: "0"
-  requirement: *id005
-- !ruby/object:Gem::Dependency 
   type: :development
+  name: moqueue
   prerelease: false
-  name: mocha
-  version_requirements: &id006 !ruby/object:Gem::Requirement 
+  version_requirements: *id005
+- !ruby/object:Gem::Dependency 
+  requirement: &id006 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -103,12 +100,12 @@ dependencies:
         segments: 
         - 0
         version: "0"
-  requirement: *id006
-- !ruby/object:Gem::Dependency 
   type: :development
+  name: mocha
   prerelease: false
-  name: bundler
-  version_requirements: &id007 !ruby/object:Gem::Requirement 
+  version_requirements: *id006
+- !ruby/object:Gem::Dependency 
+  requirement: &id007 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
@@ -119,12 +116,12 @@ dependencies:
         - 0
         - 0
         version: 1.0.0
-  requirement: *id007
-- !ruby/object:Gem::Dependency 
   type: :development
+  name: bundler
   prerelease: false
-  name: jeweler
-  version_requirements: &id008 !ruby/object:Gem::Requirement 
+  version_requirements: *id007
+- !ruby/object:Gem::Dependency 
+  requirement: &id008 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
@@ -135,12 +132,12 @@ dependencies:
         - 5
         - 2
         version: 1.5.2
-  requirement: *id008
-- !ruby/object:Gem::Dependency 
   type: :development
+  name: jeweler
   prerelease: false
-  name: rcov
-  version_requirements: &id009 !ruby/object:Gem::Requirement 
+  version_requirements: *id008
+- !ruby/object:Gem::Dependency 
+  requirement: &id009 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -149,12 +146,12 @@ dependencies:
         segments: 
         - 0
         version: "0"
-  requirement: *id009
-- !ruby/object:Gem::Dependency 
   type: :development
+  name: rcov
   prerelease: false
-  name: reek
-  version_requirements: &id010 !ruby/object:Gem::Requirement 
+  version_requirements: *id009
+- !ruby/object:Gem::Dependency 
+  requirement: &id010 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
@@ -165,12 +162,12 @@ dependencies:
         - 2
         - 8
         version: 1.2.8
-  requirement: *id010
-- !ruby/object:Gem::Dependency 
   type: :development
+  name: reek
   prerelease: false
-  name: roodi
-  version_requirements: &id011 !ruby/object:Gem::Requirement 
+  version_requirements: *id010
+- !ruby/object:Gem::Dependency 
+  requirement: &id011 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
@@ -181,7 +178,10 @@ dependencies:
         - 1
         - 0
         version: 2.1.0
-  requirement: *id011
+  type: :development
+  name: roodi
+  prerelease: false
+  version_requirements: *id011
 description: Cloudist is a simple, highly scalable job queue for Ruby applications, it can run within Rails, or on EC2, and does not load your entire Rails stack like delayed job does.
 email: ivanvanderbyl@me.com
 executables: []
@@ -200,6 +200,7 @@ files:
 - README.md
 - Rakefile
 - VERSION
+- cloudist.gemspec
 - examples/queue_message.rb
 - examples/sandwich_client.rb
 - examples/sandwich_worker.rb