maildir-queue 0.2.0 → 1.0.0

data/README.rdoc

@@ -1,9 +1,100 @@
 = maildir-queue
 
-Description goes here.
+A wrapper for the maildir library that gives it a queue-like interface.
+
+== Install
+
+  gem install maildir-queue
+
+== Usage
+
+Create a queue in /home/aaron/queue
+
+  require 'maildir/queue'
+  queue = Maildir::Queue.new("/home/aaron/queue")
+
+Push a new message onto the queue.
+
+  message = queue.push("Hello World!")
+
+List new messages
+
+  queue.list(:new) # => [message]
+
+List messages that are being processed
+
+  queue.list(:cur) # => []
+
+Move the first new message to 'cur' for processing
+
+  message = queue.shift # Returns nil if no new messages
+  message.data # => 'Hello World!'
+
+  queue.list(:new) # => []
+  queue.list(:cur) # => [message]
+
+Delete a message after it's processed
+
+  message.destroy
+
+  queue.list(:cur) # => []
+
+=== Dealing with stalled messages
+
+An exepected (but rare) behavior is for a client to fail while processing a message.
+Such messages will stay in 'cur' until retried.
+
+Get a list of messages in cur that have not been modified in 5 minutes
+
+  messages = queue.stalled_messages(Time.now - 300)
+
+Requeue them
+
+  count = queue.requeue_stalled_messages(Time.now - 300)
+
+Clients can update a message's timestamps to indicate it's still be processed.
+Then the retry interval can be shorter than the time it takes to process a message.
+
+  message.utime(Time.now, Time.now)
+
+== Maildir::WebQueue, the HTTP/JSON interface
+
+maildir-queue includes a simple Sinatra app providing an HTTP/JSON interface to a queue.
+
+Configure it like so:
+
+  require 'maildir/web_queue'
+  app = Maildir::WebQueue
+  app.path = '/home/aaron/queue'
+
+Push a message
+
+  POST /message?data=Hello World!
+
+Shift a message
+
+  GET /message
+
+  Response:
+  {"key": "cur/message_id", "data": "Hello world!"}
+  Or 404 Not Found if no new messages
+
+Delete a message
+
+  DELETE /message/key
+
+Update a message's timestamps
+
+  POST /touch/key
+
+Show count of messages in new and cur
+
+  GET /status
+  Response:
+  {"new": 2, "cur": 10}
 
 == Note on Patches/Pull Requests
- 
+
 * Fork the project.
 * Make your feature addition or bug fix.
 * Add tests for it. This is important so I don't break it in a

data/Rakefile

@@ -1,28 +1,4 @@
-require 'rubygems'
 require 'rake'
-
-begin
-  require 'jeweler'
-  Jeweler::Tasks.new do |gem|
-    gem.name = "maildir-queue"
-    gem.summary = %Q{A simple queue API with a maildir backend.}
-    gem.description = %Q{A simple queue API with a maildir backend. Also includes an HTTP API}
-    gem.email = "aaron@ktheory.com"
-    gem.homepage = "http://github.com/ktheory/maildir-queue"
-    gem.authors = ["Aaron Suggs"]
-    gem.add_development_dependency "shoulda", ">= 0"
-    gem.add_development_dependency "rack-test", ">= 0"
-    gem.add_development_dependency "ktheory-fakefs", ">= 0"
-    gem.add_dependency "maildir", ">= 0.3.0"
-    gem.add_dependency "sinatra", ">= 0.0.0"
-    gem.add_dependency "json", ">= 0.0.0"
-    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
-  end
-  Jeweler::GemcutterTasks.new
-rescue LoadError
-  puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
-end
-
 require 'rake/testtask'
 Rake::TestTask.new(:test) do |test|
   test.libs << 'lib' << 'test'
@@ -43,8 +19,6 @@ rescue LoadError
   end
 end
 
-task :test => :check_dependencies
-
 task :default => :test
 
 require 'rake/rdoctask'

data/lib/maildir/queue.rb

@@ -1,8 +1,5 @@
 require 'maildir'
 class Maildir::Queue < Maildir
-  # How many times to retry getting a key.
-  # Default is -1 (infinite retries)
-  SHIFT_RETRIES = -1
 
   # Adds a new message to the queue. Returns a Maildir::Message object
   def push(data)
@@ -11,29 +8,11 @@ class Maildir::Queue < Maildir
 
   # Finds a new message and marks it as being processed (i.e. moves message
   # from new to cur). Returns message if successful; nil if there are no
-  # pending messages.
+  # new messages.
   def shift
-    retries = 0
-    begin
-      # Get a pending message
-      message = list(:new, :limit => 1).first
-      return nil if message.nil?
-
-      # Move the message from new to cur
-      if message.process
-        return message
-      else
-        raise Errno::ENOENT
-      end
-    rescue Errno::ENOENT
-      # Either message.process failed. Retry.
-      if SHIFT_RETRIES < 0 || retries < SHIFT_RETRIES
-        retries += 1
-        retry
-      else
-        # After several failures, act as if there are no pending messages
-        return nil
-      end
+    loop do
+      message = do_shift
+      return message unless false == message
     end
   end
 
@@ -57,4 +36,15 @@ class Maildir::Queue < Maildir
       end
     end
   end
+
+  protected
+  # Called by shift. Returns the first message in the queue; nil if no
+  # messages are in the queue; or false if unable to process the message.
+  def do_shift
+    message = list(:new, :limit => 1).first
+    return nil if message.nil?
+
+    # Try to move the message from new to cur
+    message.process ? message : false
+  end
 end

data/lib/maildir/web_queue.rb

@@ -4,7 +4,10 @@ require "json"
 
 class Maildir::WebQueue < Sinatra::Base
 
-  KEY_VALIDATOR = /^cur\/\d{10}\.\w+(\.\w+)+:2,(\w+)?$/
+  KEY_VALIDATORS = [
+    /^cur\/\d{10}\.\w+(\.\w+)+:2,(\w+)?$/, # cur keys, with info
+    /^new\/\d{10}\.\w+(\.\w+)+$/ # new keys, no info
+  ]
 
   def self.path=(path)
     @@queue = Maildir::Queue.new(path)
@@ -17,7 +20,7 @@ class Maildir::WebQueue < Sinatra::Base
   # Test that key is well-formed. If not, return 403 Forbidden error.
   def sanitize(key)
     # E.g. cur/1263444769.M975543P58179Q11.gnt.local:2,
-    unless key.match(KEY_VALIDATOR)
+    unless KEY_VALIDATORS.any?{|validator| key.match(validator) }
       content_type "text/plain"
       halt 403, "Malformed key: #{key}"
     end
@@ -28,10 +31,16 @@ class Maildir::WebQueue < Sinatra::Base
     halt 204, ""
   end
 
+  # Set the content type to JSON and returns the body as JSON
+  def json(body)
+    content_type "application/json"
+    body.to_json
+  end
+
   # Check the server status
   get "/status" do
-    content_type "application/json"
-    {"new" => queue.list(:new).size,"cur" => queue.list(:cur).size}.to_json
+    body = {"new" => queue.list(:new).size,"cur" => queue.list(:cur).size}
+    json(body)
   end
 
   # Create a new message. Requires params[:data]
@@ -39,36 +48,38 @@ class Maildir::WebQueue < Sinatra::Base
   post "/message" do
     halt 400, "Must specify data parameter" unless params[:data]
     message = queue.push(params[:data])
-    content_type "application/json"
-    message.key.to_json
+    json(message.key)
   end
 
   # Shift a new message off the queue
   get "/message" do
     message = queue.shift
-    content_type "application/json"
+
     if message
-      {"key" => message.key, "data" => message.data}.to_json
+      json({"key" => message.key, "data" => message.data})
     else
-      not_found "No pending messages".to_json
+      not_found json("No new messages")
     end
   end
 
   # Delete a message from the queue
   delete "/message/*" do |key|
     sanitize(key)
-    queue.delete(key)
-    no_content
+    if queue.delete(key)
+      no_content
+    else
+      not_found json("Key #{key} does not exist")
+    end
   end
 
-  # # Update the timestamps on a message
-  # post "/message/touch/*" do |key|
-  #   sanitize(key)
-  #   if queue.get(key).utime(Time.now, Time.now)
-  #     no_content
-  #   else
-  #     not_found "Key #{key} does not exist"
-  #   end
-  # end
+  # Update the timestamps on a message
+  post "/touch/*" do |key|
+    sanitize(key)
+    if queue.get(key).utime(Time.now, Time.now)
+      no_content
+    else
+      not_found json("Key #{key} does not exist")
+    end
+  end
 end
 

data/test/helper.rb

@@ -14,6 +14,6 @@ require 'fakefs'
 
 
 # Create a reusable MaildirQueue that's cleaned up when the tests are done
-def temp_queue
+def queue
   Maildir::Queue.new("/tmp/maildir_queue_test")
 end

data/test/test_maildir_queue.rb

@@ -5,43 +5,47 @@ class TestMaildirQueue < Test::Unit::TestCase
 
     setup { FakeFS::FileSystem.clear }
 
+    should "return nil if no messages to shift" do
+      assert_equal nil, queue.shift
+    end
+
     should "shift messages" do
-      temp_queue.add("1")
-      temp_queue.add("2")
-      message = temp_queue.shift
+      queue.add("1")
+      queue.add("2")
+      message = queue.shift
       assert_equal "1", message.data
     end
 
     should "put shifted messages in cur" do
-      temp_queue.add("")
-      message = temp_queue.shift
+      queue.add("")
+      message = queue.shift
       assert_equal :cur, message.dir
     end
 
     should "list stalled messages" do
-      temp_queue.add("")
-      message = temp_queue.shift
+      queue.add("")
+      message = queue.shift
       mtime = Time.now - 100
       message.utime(mtime, mtime)
-      assert temp_queue.stalled_messages(Time.now - 50).include?(message)
+      assert queue.stalled_messages(Time.now - 50).include?(message)
     end
 
     should "requeue stalled messages" do
       data = "my data"
-      temp_queue.add(data)
-      stalled_message = temp_queue.shift
+      queue.add(data)
+      stalled_message = queue.shift
       mtime = Time.now - 100
       stalled_message.utime(mtime, mtime)
-      temp_queue.requeue_stalled_messages(Time.now - 50)
-      assert_equal data, temp_queue.list(:new).first.data
+      queue.requeue_stalled_messages(Time.now - 50)
+      assert_equal data, queue.list(:new).first.data
     end
 
     should "return the number of requeued stalled messages" do
-      temp_queue.add("")
-      stalled_message = temp_queue.shift
+      queue.add("")
+      stalled_message = queue.shift
       mtime = Time.now - 100
       stalled_message.utime(mtime, mtime)
-      assert_equal 1, temp_queue.requeue_stalled_messages(Time.now - 50)
+      assert_equal 1, queue.requeue_stalled_messages(Time.now - 50)
     end
   end
 end

data/test/test_maildir_web_queue.rb

@@ -3,11 +3,11 @@ class TestMaildirWebQueue < Test::Unit::TestCase
   include Rack::Test::Methods
 
   def app
-    Maildir::WebQueue.path = temp_queue.path
+    Maildir::WebQueue.path = queue.path
     Maildir::WebQueue
   end
 
-  context "The webcontroller" do
+  context "The WebQueue" do
 
     setup do
       FakeFS::FileSystem.clear
@@ -26,44 +26,71 @@ class TestMaildirWebQueue < Test::Unit::TestCase
 
     should "add posted messages to the queue" do
       post "/message", :data => @data
-      assert_equal 1, temp_queue.list(:new).size
+      assert_equal 1, queue.list(:new).size
     end
 
     should "save messages with the right data" do
       post "/message", :data => @data
-      assert_equal @data, temp_queue.list(:new).first.data
+      assert_equal @data, queue.list(:new).first.data
     end
 
-    should "return 404 when no pending messages" do
+    should "return 404 when there are no new messages" do
       get "/message"
       assert_equal 404, last_response.status
     end
 
-    should "return successfully when messages are pending" do
+    should "return successfully when a new message exists" do
       post "/message", :data => @data
       get "/message"
       assert last_response.ok?
     end
 
-    should "return a well-formed key when messages are pending" do
+    should "return a well-formed key when new a message exists" do
       post "/message", :data => @data
       get "/message"
-      assert_match Maildir::WebQueue::KEY_VALIDATOR, JSON.parse(last_response.body)["key"]
+      assert_match Maildir::WebQueue::KEY_VALIDATORS[0], JSON.parse(last_response.body)["key"]
     end
 
-    should "return message data when messages are pending" do
+    should "return message data when a new message exits" do
       post "/message", :data => @data
       get "/message"
       assert_equal @data, JSON.parse(last_response.body)["data"]
     end
 
+    should "return 404 when touching a missing message" do
+      key = "cur/1234567890.M975543P58179Q11.host:2,"
+      post "/touch/#{key}"
+      assert_equal 404, last_response.status
+    end
+
+    should "touch a message" do
+      message = queue.add(@data)
+      message.process
+
+      post "/touch/#{message.key}"
+      assert_equal 204, last_response.status
+    end
+
+    should "update a message's mtime when touched" do
+      message = queue.add(@data)
+      message.process
+
+      # Set mtime to 30 minutes ago
+      message.utime(Time.now, Time.now - 30*60)
+
+      post "/touch/#{message.key}"
+      assert_in_delta Time.now, message.mtime, 1
+    end
   end
 
-  # Test the Maildir::WebQueue::KEY_VALIDATOR
-  bad_keys = [ "/etc/passwd", "cur/../../etc/password",
+  # Test Maildir::WebQueue::KEY_VALIDATORS
+  bad_keys = [
+    "/etc/passwd", "cur/../../etc/password", "..",
     "cur/123456789.M975543P58179Q11.host:2,",
     "cur/1234567890.M975543P58179Q11:2,",
-    "cur/1234567890.M975543P58179Q11.host"
+    "cur/1234567890.M975543P58179Q11.host",
+    "new/1234567890.M975543P58179Q11.host:2,",
+    "tmp/1234567890.M975543P58179Q11.host:2,FRS"
   ]
   bad_keys.each_with_index do |key, i|
     define_method "test_bad_key_#{i}" do
@@ -72,14 +99,16 @@ class TestMaildirWebQueue < Test::Unit::TestCase
     end
   end
 
-  good_keys = [ "cur/1234567890.M975543P58179Q11.host:2,",
+  good_keys = [
+    "cur/1234567890.M975543P58179Q11.host:2,",
     "cur/1234567890.abc123.really.long.domain.co.uk:2,",
-    "cur/1234567890.M975543P58179Q11.host:2,FRS"
+    "cur/1234567890.M975543P58179Q11.host:2,FRS",
+    "new/1234567890.M975543P58179Q11.host"
   ]
   good_keys.each_with_index do |key, i|
     define_method "test_good_key_#{i}" do
       delete "/message/#{key}"
-      assert last_response.successful?
+      assert_not_equal 403, last_response.status, "Key: #{key}"
     end
   end
 end

metadata

@@ -1,7 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: maildir-queue
 version: !ruby/object:Gem::Version 
-  version: 0.2.0
+  hash: 23
+  prerelease: false
+  segments: 
+  - 1
+  - 0
+  - 0
+  version: 1.0.0
 platform: ruby
 authors: 
 - Aaron Suggs
@@ -9,89 +15,123 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-01-15 00:00:00 -05:00
+date: 2010-12-19 00:00:00 -05:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
-  name: shoulda
+  name: rake
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
   type: :development
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: shoulda
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
         version: "0"
-    version: 
+  type: :development
+  version_requirements: *id002
 - !ruby/object:Gem::Dependency 
   name: rack-test
-  type: :development
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
         version: "0"
-    version: 
+  type: :development
+  version_requirements: *id003
 - !ruby/object:Gem::Dependency 
   name: ktheory-fakefs
-  type: :development
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
         version: "0"
-    version: 
+  type: :development
+  version_requirements: *id004
 - !ruby/object:Gem::Dependency 
   name: maildir
-  type: :runtime
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
-    - - ">="
+    - - ~>
       - !ruby/object:Gem::Version 
-        version: 0.3.0
-    version: 
+        hash: 23
+        segments: 
+        - 1
+        - 0
+        - 0
+        version: 1.0.0
+  type: :runtime
+  version_requirements: *id005
 - !ruby/object:Gem::Dependency 
   name: sinatra
-  type: :runtime
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
+  prerelease: false
+  requirement: &id006 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        version: 0.0.0
-    version: 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id006
 - !ruby/object:Gem::Dependency 
   name: json
-  type: :runtime
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
+  prerelease: false
+  requirement: &id007 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        version: 0.0.0
-    version: 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id007
 description: A simple queue API with a maildir backend. Also includes an HTTP API
 email: aaron@ktheory.com
 executables: []
 
 extensions: []
 
-extra_rdoc_files: 
-- LICENSE
-- README.rdoc
+extra_rdoc_files: []
+
 files: 
-- .document
-- .gitignore
+- lib/maildir/queue.rb
+- lib/maildir/web_queue.rb
 - LICENSE
 - README.rdoc
 - Rakefile
-- VERSION
-- examples/config.ru
-- lib/maildir/queue.rb
-- lib/maildir/web_queue.rb
-- maildir-queue.gemspec
 - test/helper.rb
 - test/test_maildir_queue.rb
 - test/test_maildir_web_queue.rb
@@ -105,21 +145,29 @@ rdoc_options:
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
       version: "0"
-  version: 
 required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      version: "0"
-  version: 
+      hash: 17
+      segments: 
+      - 1
+      - 3
+      - 5
+      version: 1.3.5
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.3.5
+rubygems_version: 1.3.7
 signing_key: 
 specification_version: 3
 summary: A simple queue API with a maildir backend.

data/.document

@@ -1,5 +0,0 @@
-README.rdoc
-lib/**/*.rb
-bin/*
-features/**/*.feature
-LICENSE

data/.gitignore

@@ -1,21 +0,0 @@
-## MAC OS
-.DS_Store
-
-## TEXTMATE
-*.tmproj
-tmtags
-
-## EMACS
-*~
-\#*
-.\#*
-
-## VIM
-*.swp
-
-## PROJECT::GENERAL
-coverage
-rdoc
-pkg
-
-## PROJECT::SPECIFIC

data/VERSION

@@ -1 +0,0 @@
-0.2.0

data/examples/config.ru

@@ -1,5 +0,0 @@
-# A rackup file suitable for thin, passenger, unicorn, etc.
-
-require "maildir/web_queue"
-Maildir::WebQueue.path = "/tmp/my_maildir"
-run Maildir::WebQueue

data/maildir-queue.gemspec

@@ -1,73 +0,0 @@
-# Generated by jeweler
-# DO NOT EDIT THIS FILE DIRECTLY
-# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
-# -*- encoding: utf-8 -*-
-
-Gem::Specification.new do |s|
-  s.name = %q{maildir-queue}
-  s.version = "0.2.0"
-
-  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
-  s.authors = ["Aaron Suggs"]
-  s.date = %q{2010-01-15}
-  s.description = %q{A simple queue API with a maildir backend. Also includes an HTTP API}
-  s.email = %q{aaron@ktheory.com}
-  s.extra_rdoc_files = [
-    "LICENSE",
-     "README.rdoc"
-  ]
-  s.files = [
-    ".document",
-     ".gitignore",
-     "LICENSE",
-     "README.rdoc",
-     "Rakefile",
-     "VERSION",
-     "examples/config.ru",
-     "lib/maildir/queue.rb",
-     "lib/maildir/web_queue.rb",
-     "maildir-queue.gemspec",
-     "test/helper.rb",
-     "test/test_maildir_queue.rb",
-     "test/test_maildir_web_queue.rb"
-  ]
-  s.homepage = %q{http://github.com/ktheory/maildir-queue}
-  s.rdoc_options = ["--charset=UTF-8"]
-  s.require_paths = ["lib"]
-  s.rubygems_version = %q{1.3.5}
-  s.summary = %q{A simple queue API with a maildir backend.}
-  s.test_files = [
-    "test/helper.rb",
-     "test/test_maildir_queue.rb",
-     "test/test_maildir_web_queue.rb"
-  ]
-
-  if s.respond_to? :specification_version then
-    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
-    s.specification_version = 3
-
-    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
-      s.add_development_dependency(%q<shoulda>, [">= 0"])
-      s.add_development_dependency(%q<rack-test>, [">= 0"])
-      s.add_development_dependency(%q<ktheory-fakefs>, [">= 0"])
-      s.add_runtime_dependency(%q<maildir>, [">= 0.3.0"])
-      s.add_runtime_dependency(%q<sinatra>, [">= 0.0.0"])
-      s.add_runtime_dependency(%q<json>, [">= 0.0.0"])
-    else
-      s.add_dependency(%q<shoulda>, [">= 0"])
-      s.add_dependency(%q<rack-test>, [">= 0"])
-      s.add_dependency(%q<ktheory-fakefs>, [">= 0"])
-      s.add_dependency(%q<maildir>, [">= 0.3.0"])
-      s.add_dependency(%q<sinatra>, [">= 0.0.0"])
-      s.add_dependency(%q<json>, [">= 0.0.0"])
-    end
-  else
-    s.add_dependency(%q<shoulda>, [">= 0"])
-    s.add_dependency(%q<rack-test>, [">= 0"])
-    s.add_dependency(%q<ktheory-fakefs>, [">= 0"])
-    s.add_dependency(%q<maildir>, [">= 0.3.0"])
-    s.add_dependency(%q<sinatra>, [">= 0.0.0"])
-    s.add_dependency(%q<json>, [">= 0.0.0"])
-  end
-end
-