beehive 0.1

data/MANIFEST

@@ -0,0 +1,21 @@
+.gemtest
+MANIFEST
+README.md
+Rakefile
+example/client.rb
+example/multiple-workers.rb
+example/worker.rb
+lib/.gitkeep
+lib/beehive/.gitkeep
+lib/beehive/client.rb
+lib/beehive/version.rb
+lib/beehive/worker.rb
+lib/beehive.rb
+license.txt
+spec/.gitkeep
+spec/.rspec
+spec/beehive/client.rb
+spec/beehive/worker.rb
+spec/helper.rb
+spec/tmp/.gitkeep
+task/build.rake

data/README.md

@@ -0,0 +1,42 @@
+# Beehive
+
+Beehive is a super lightweight queue system that uses Redis as it's storage engine.
+Beehive was created because I got fed up with Resque's large memory footprint and not
+being able to find a decent alternative that wasn't broken. 
+
+## Requirements
+
+* Redis and the redis Gem - install with `gem install redis`
+* The JSON gem - install with `gem install json`
+* Ruby >= 1.9.2
+
+## Installation & Usage
+
+Installing Beehive is done as following:
+
+    $ gem install beehive
+
+Once it's installed you can use it as following:
+
+    require 'beehive'
+
+    client = Beehive::Client.new
+    client.queue('email.send', :to => 'info@yorickpeterse.com', :subject => 'Hello, world!')
+
+Your worker would look like the following:
+
+    require 'beehive'
+
+    Beehive.job('email.send') do |params|
+      # Do something with Net::IMAP, Net::POP, etc
+    end
+
+    worker = Beehive::Worker.new({}, {:jobs => ['email.send']})
+    worker.work
+
+For more examples see the "example" directory.
+
+## License
+
+Beehive is licensed under the MIT license, a copy of this license can be found in the file
+"license.txt".

data/Rakefile

@@ -0,0 +1,13 @@
+require File.expand_path('../lib/beehive', __FILE__)
+require 'rake/testtask'
+
+# Load all the tasks
+Dir.glob(File.expand_path("../task/*.rake", __FILE__)).each do |f|
+  import(f)
+end
+
+# Set the tests so they can be run with $ rake test
+Rake::TestTask.new do |t|
+  t.test_files = Dir.glob(File.expand_path('../spec/beehive/**/*.rb', __FILE__))
+  t.verbose    = true
+end

data/example/client.rb

@@ -0,0 +1,7 @@
+require File.expand_path('../../lib/beehive', __FILE__)
+
+# Initialize the client
+client = Beehive::Client.new
+
+# Send the job to the queue
+client.queue('example', :amount => 2)

data/example/multiple-workers.rb

@@ -0,0 +1,30 @@
+require File.expand_path('../../lib/beehive', __FILE__)
+
+# A more "complex" job
+Beehive.job('example') do |params|
+  if params.key?('amount')
+    amount = params['amount'].to_i
+  else
+    amount = 2
+  end
+
+  amount.times do |num|
+    puts "Iteration: #{num}"
+  end
+end
+
+# The amount of workers to start
+if ARGV.empty?
+  worker_amount = 5
+else
+  worker_amount = ARGV[0].to_i
+end
+
+# Start 5 workers
+worker_amount.times do
+  Process.fork do
+    # Start the worker and wait for a job to do
+    worker = Beehive::Worker.new({}, {:jobs => ['example'], :log_level => Logger::INFO, :daemonize => true})
+    worker.work
+  end
+end

data/example/worker.rb

@@ -0,0 +1,12 @@
+require File.expand_path('../../lib/beehive', __FILE__)
+
+# Define the job. Due to the forking behavior the output won't be actually visible.
+Beehive.job('example') do |params|
+  puts "Hello, world!"
+end
+
+# Create a new instance of a worker and customize the settings a bit.
+worker = Beehive::Worker.new({}, {:jobs => ['example'], :log_level => Logger::INFO})
+
+# And start the worker
+worker.work

data/lib/beehive/client.rb

@@ -0,0 +1,70 @@
+module Beehive
+  ##
+  # The client class that can be used to add and manage jobs in the queue.
+  #
+  # @author Yorick Peterse
+  # @since  0.1
+  #
+  class Client
+    # Instance of the class "Redis"
+    attr_reader :connection
+
+    ##
+    # Creates a new instance of the client and sends the specified options to Redis.
+    #
+    # @author Yorick Peterse
+    # @since  0.1
+    # @param  [Hash] options A hash containing options to use for Redis. See 
+    # Redis#initialize for more information.
+    #
+    def initialize(options = {})
+      @connection = ::Redis.new(options)
+    end
+
+    ##
+    # Queues a given job in the storage.
+    #
+    # @example
+    #  client = Beehive::Client.new
+    #  client.queue('video', :title => 'Hello world')
+    #
+    # @author Yorick Peterse
+    # @since  0.1
+    # @param  [String] job The name of the job to queue.
+    # @param  [Hash] params The parameters to send to the job.
+    #
+    def queue(job, params = {})
+      job    = "beehive.jobs.#{job}"
+      params = JSON.dump(params)
+
+      @connection.rpush(job, params)
+    end
+
+    ##
+    # Retrieves the last job and removes it from the storage.
+    #
+    # @author Yorick Peterse
+    # @since  0.1
+    # @param  [String] job The name of the type of job to retrieve.
+    # @return [Array]
+    #
+    def get(job)
+      job = "beehive.jobs.#{job}"
+      job = @connection.rpop(job)
+      
+      if !job.nil?
+        return JSON.load(job)
+      end
+    end
+
+    ##
+    # Closes the Redis connection.
+    #
+    # @author Yorick Peterse
+    # @since  0.1
+    #
+    def disconnect
+      @connection.quit
+    end
+  end # Client
+end # Beehive

data/lib/beehive/version.rb

@@ -0,0 +1,9 @@
+module Beehive
+  ##
+  # The current version of Beehive.
+  #
+  # @author Yorick Peterse
+  # @since  0.1
+  #
+  Version = '0.1'
+end

data/lib/beehive/worker.rb

@@ -0,0 +1,159 @@
+module Beehive
+  ##
+  # The Worker class is used to retrieve and process jobs (in the background). Whenever a
+  # job is received the worker will fork itself and execute the job in that process. This
+  # is useful because any errors won't crash the worker itself plus it will reduce the
+  # memory usage as once Ruby allocates memory to a process it's never released unless
+  # that process exits.
+  #
+  # @author Yorick Peterse
+  # @since  0.1
+  #
+  class Worker
+    ##
+    # Hash containing the default worker options.
+    #
+    # @author Yorick Peterse
+    # @since  0.1
+    #
+    Options = {
+      :logger    => ::Logger.new(STDOUT),
+      :daemonize => false,
+      :jobs      => [],
+      :wait      => 5,
+      :log_level => Logger::WARN
+    }
+
+    # Instance of Beehive::Client, used for communicating with the Redis server
+    attr_reader :connection
+
+    # Hash containing all the custom configuration options
+    attr_accessor :options
+
+    ##
+    # Creates a new instance of the class, sets all the options and connects to the Redis
+    # database.
+    #
+    # @example
+    #  worker = Beehive::Worker.new({}, {:jobs => ['video'], :wait => 2})
+    #
+    # @author Yorick Peterse
+    # @since  0.1
+    # @param  [Hash] redis_options Hash containing all the options to use for Redis. See
+    # Redis#new for more information.
+    # @param  [Hash] worker_options Hash containing worker specific options.
+    # @option worker_options :logger The logger that should be used, has to be compatible
+    # with Logger of the standard library.
+    # @option worker_options :daemonize Tells the worker to run in the background.
+    # @option worker_options :jobs An array of jobs the current worker has to process.
+    # @option worker_options :wait The amount of seconds to wait after each iteration,
+    # reduces CPU and network usage.
+    # @option worker_options :log_level The log even to use for the :logger option, set to
+    # Logger::WARN by default.
+    #
+    def initialize(redis_options = {}, worker_options = {})
+      @connection             = ::Beehive::Client.new(redis_options)
+      @options                = Options.merge(worker_options)
+      @options[:logger].level = @options[:log_level]
+      @shutdown               = false
+
+      # Check if the given jobs are valid
+      @options[:jobs].each do |job|
+        if !::Beehive::Jobs.key?(job)
+          raise("The job \"#{job}\" is invalid as it could not be found in Beehive::Jobs")
+        end
+      end
+
+      trap_signals
+    end
+
+    ##
+    # Waits for available jobs and execute a job whenever one is available.
+    #
+    # @example
+    #  worker = Beehive::Worker.new
+    #  worker.work
+    #
+    # @author Yorick Peterse
+    # @since  0.1
+    #
+    def work
+      # Daemonize the process?
+      if @options[:daemonize] === true
+        Process.daemon(true)
+      end
+
+      @options[:logger].info("Starting main worker, PID: #{Process.pid}")
+
+      loop do
+        if @shutdown === true
+          @options[:logger].info('The party has ended, time to shut down')
+          @connection.disconnect
+          exit
+        end
+
+        # Reset the child PID
+        @child_pid = nil
+
+        # Check if there are any jobs available
+        @options[:jobs].each do |job|
+          params = @connection.get(job)
+
+          if params
+            @options[:logger].info(
+              "Received the job \"#{job}\" with the following data: #{params.inspect}"
+            )
+
+            # Fork the process and run the job
+            @child_pid = Process.fork do
+              @options[:logger].info('Process forked, executing job')
+
+              begin
+                ::Beehive::Jobs[job].call(params)
+
+                @options[:logger].info('Job successfully processed')
+                exit
+              rescue => e
+                @options[:logger].error("Failed to execute the job: #{e.inspect}")
+              end
+            end
+
+            # Wait for the job to finish. This prevents this worker from spawning a worker
+            # for each job it has to process (which could result in hundreds of processes
+            # being spawned.
+            Process.waitpid(@child_pid)
+          end
+        end
+
+        # Reduces CPU load and network traffic
+        sleep(@options[:wait])
+      end
+    end
+
+    private
+
+    ##
+    # Registers all the signals that trigger a shutdown.
+    #
+    # @author Yorick Peterse
+    # @since  0.1
+    #
+    def trap_signals
+      # Shut down gracefully
+      ['TERM', 'INT'].each do |signal|
+        Signal.trap(signal) { @shutdown = true }
+      end
+
+      # QUIT will trigger the setup to quit immediately
+      Signal.trap('QUIT') do
+        @shutdown = true
+
+        # No point in killing a child process if it isn't there in the first place
+        if !@child_pid.nil?
+          @options[:logger].info("Shutting down the worker with PID #{@child_pid}")
+          Process.kill('INT', @child_pid)
+        end
+      end
+    end
+  end # Worker
+end # Beehive

data/lib/beehive.rb

@@ -0,0 +1,51 @@
+require 'redis'
+require 'json'
+require 'logger'
+
+require_relative('beehive/version')
+require_relative('beehive/client')
+require_relative('beehive/worker')
+
+##
+# Beehive is a lightweight queue system that uses Redis to store all jobs. It's inspired
+# by existing systems such as Minion and Stalker. Beehive does not offer a plugin system
+# or any other fancy features, it merely allows you to queue a job and have it processed
+# in the background.
+#
+# ## Features
+#
+# * Uses Redis as it's storage mechanism.
+# * Lightweight: workers don't waste resources on loading large amounts of libraries.
+# * Easy to use
+#
+# Fore more information see the README file.
+#
+# @author Yorick Peterse
+# @since  0.1
+#
+module Beehive
+  ##
+  # Hash containing all the defined jobs.
+  #
+  # @author Yorick Peterse
+  # @since  0.1
+  #
+  Jobs = {}
+
+  ##
+  # Adds a new job to the list.
+  #
+  # @example
+  #  Beehive.job('video') do |params|
+  #    puts "Hello!"
+  #  end
+  #
+  # @author Yorick Peterse
+  # @since  0.1
+  # @param  [String] job The name of the job.
+  # @param  [Proc] block The block to call whenever the job has to be processed.
+  #
+  def self.job(job, &block)
+    Jobs[job.to_s] = block
+  end
+end

data/license.txt

@@ -0,0 +1,19 @@
+Copyright (c) 2011, Yorick Peterse
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/spec/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--colour

data/spec/beehive/client.rb

@@ -0,0 +1,17 @@
+require File.expand_path('../../helper', __FILE__)
+
+describe('Beehive::Client') do
+
+  it('Queue a new job') do
+    client = Beehive::Client.new
+    client.queue('video', :title => 'hello')
+  end
+
+  it('Get the latest job') do
+    client = Beehive::Client.new
+    job    = client.get('video')
+
+    job['title'].should === 'hello'
+  end
+
+end

data/spec/beehive/worker.rb

@@ -0,0 +1,43 @@
+require File.expand_path('../../helper', __FILE__)
+
+describe('Beehive::Worker') do
+  it('Create a new instance of Beehive::Worker') do
+    worker = Beehive::Worker.new
+
+    worker.options[:logger].level.should === Logger::WARN
+    worker.options[:jobs].size.should    === 0
+  end
+
+  it('Start a worker and queue a job') do
+    path = File.expand_path('../../tmp/output', __FILE__)
+    
+    if File.exist?(path)
+      File.unlink(path)
+    end
+
+    Beehive.job('spec') do |params|
+      File.open(path, 'w') do |handle|
+        handle.write('Hello, world!')
+        handle.close
+      end
+    end
+
+    # Queue the job
+    client = Beehive::Client.new
+    client.queue('spec')
+
+    pid = Process.fork do
+      worker = Beehive::Worker.new({}, {:jobs => ['spec']})
+      worker.work
+    end
+
+    # Wait for the process to end
+    Process.kill('INT', pid)
+    Process.waitpid(pid)
+
+    path = File.read(path, File.size(path)).strip
+
+    path.should === 'Hello, world!'
+  end
+
+end

data/spec/helper.rb

@@ -0,0 +1,3 @@
+require File.expand_path('../../lib/beehive', __FILE__)
+require 'rspec'
+require 'rspec/autorun'

data/task/build.rake

@@ -0,0 +1,55 @@
+require 'find'
+
+namespace :build do
+
+  desc 'Builds the documentation using YARD'
+  task :doc do
+    gem_path = File.expand_path('../../', __FILE__)
+    command  = "yard doc #{gem_path}/lib -m markdown -M rdiscount -o #{gem_path}/doc "
+    command += "-r #{gem_path}/README.md --private --protected "
+    command += "--files #{gem_path}/license.txt"
+
+    sh(command)
+  end
+
+  desc 'Builds a new Gem'
+  task :gem do
+    gem_path = File.expand_path('../../', __FILE__)
+
+    # Build and install the gem
+    sh("gem build #{gem_path}/beehive.gemspec")
+    sh("mv #{gem_path}/beehive-#{Beehive::Version}.gem #{gem_path}/pkg")
+    sh("gem install #{gem_path}/pkg/beehive-#{Beehive::Version}.gem")
+  end
+
+  desc 'Builds the MANIFEST file'
+  task :manifest do
+    gem_path     = File.expand_path('../../', __FILE__)
+    ignore_exts  = ['.gem', '.gemspec', '.swp']
+    ignore_files = ['.DS_Store', '.gitignore', 'output']
+    ignore_dirs  = ['.git', '.yardoc', 'pkg', 'doc']
+    files        = ''
+    
+    Find.find(gem_path) do |f|
+      f[gem_path] = ''
+      f.gsub!(/^\//, '')
+
+      # Ignore directories
+      if !File.directory?(f) and !ignore_exts.include?(File.extname(f)) and !ignore_files.include?(File.basename(f))
+        files += "#{f}\n"
+      else
+        Find.prune if ignore_dirs.include?(f)
+      end
+    end
+    
+    # Time to write the MANIFEST file
+    begin
+      handle = File.open 'MANIFEST', 'w'
+      handle.write files.strip
+      puts "The MANIFEST file has been updated."
+    rescue
+      abort "The MANIFEST file could not be written."
+    end
+  end
+
+end

metadata

@@ -0,0 +1,119 @@
+--- !ruby/object:Gem::Specification 
+name: beehive
+version: !ruby/object:Gem::Version 
+  prerelease: 
+  version: "0.1"
+platform: ruby
+authors: 
+- Yorick Peterse
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-05-20 00:00:00 +02:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: redis
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 2.2.0
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: json
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.5.1
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: rake
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.8.7
+  type: :development
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 2.6.0
+  type: :development
+  version_requirements: *id004
+description: Beehive is a lightweight queue system that uses Redis.
+email: info@yorickpeterse.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- .gemtest
+- MANIFEST
+- README.md
+- Rakefile
+- example/client.rb
+- example/multiple-workers.rb
+- example/worker.rb
+- lib/.gitkeep
+- lib/beehive/.gitkeep
+- lib/beehive/client.rb
+- lib/beehive/version.rb
+- lib/beehive/worker.rb
+- lib/beehive.rb
+- license.txt
+- spec/.gitkeep
+- spec/.rspec
+- spec/beehive/client.rb
+- spec/beehive/worker.rb
+- spec/helper.rb
+- spec/tmp/.gitkeep
+- task/build.rake
+has_rdoc: yard
+homepage: https://github.com/yorickpeterse/beehive/
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.6.2
+signing_key: 
+specification_version: 3
+summary: Beehive is a lightweight queue system that uses Redis.
+test_files: []
+