havanna 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 083b1457c1f1af6aa4327822bbb00daba957324d
+  data.tar.gz: c5118e5d44add3623f7457caab9d8d1fda81a399
+SHA512:
+  metadata.gz: f0e1927e6176f9fa7bb90be4524a52d136bdf8fb3c7221baadb861221fb0eb03a0030d9e5df429bc8f54baa872d23c1df4a0134fe08278577684d1effba8b78f
+  data.tar.gz: 55d2ebf43d9fdfcce39a20fd848c8b72fe3e9393133a8eacf56a003ddc44c87a4ca6f5847b5b495f61a1f6c4f9e57c8b372eeedff89c880439c6680c60893969

data/.gitignore

@@ -0,0 +1,3 @@
+/.env
+/.gs
+/test/**/*.log

data/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Damian Janowski
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,118 @@
+Havanna
+=======
+
+Ruby workers with [Disque][disque].
+
+Usage
+-----
+
+Create a worker:
+
+```ruby
+class Mailer
+  def call(item)
+    puts "Emailing #{item}..."
+
+    # Actually do it.
+  end
+end
+```
+
+Havanna doesn't perform any kind of magic autodiscovery of
+your workers. Similar to Rack's `config.ru`, Havanna has an
+entry point file where you explicitly declare your workers.
+This would be a valid `Havannafile`:
+
+```ruby
+require "app"
+
+Havanna.run(Mailer)
+```
+
+Now on the command line, start your workers:
+
+```
+$ havanna start
+```
+
+In a different window, try queuing a job using Disque's
+built-in client:
+
+```
+$ disque addjob Mailer foo@example.com 5000
+```
+
+Once you're up and running, deploy your workers with `-d`
+for daemonization:
+
+```
+$ havanna start -d
+```
+
+Stop the worker pool by issuing a `stop` command:
+
+```
+$ havanna stop
+```
+
+This will wait for all workers to exit gracefully.
+
+For more information, run:
+
+```
+$ havanna -h
+```
+
+
+Design notes
+------------
+
+Havanna assumes that your workers perform a fair amount of I/O (probably one
+of the most common reasons to send jobs to the background). We will optimize
+Havanna for this use case.
+
+Currently, Havanna runs multiple threads per worker. However, we may `fork(2)`
+if we find that's better for multiple-core utilization under MRI.
+
+
+Alternatives
+------------
+
+It's very likely that Havanna is not for you. While I use it in production,
+it's small and doesn't do much.
+
+These are the alternatives I know of in Rubyland:
+
+- [Disc][disc]: By my friend [pote][pote]. It supports more customization of
+  workers and queues, takes configuration from environment variables and can
+  take advantage of Celluloid if you're using it.
+
+- [DisqueJockey][disque_jockey]: I don't know much about this one, but
+  apparently it's even more configurable, has a DSL and (naturally) might be
+  a better fit if you use/like Rails.
+
+
+About the name
+--------------
+
+Havanna is inspired by [Ost][ost] and [ost(1)][ost-bin]. [soveran][soveran]
+named Ost after a café, and I happened to be sitting at another café when I
+started to work on this library. Its name: [Havanna][havanna].
+
+By the way, before becoming a café, Havanna produced the best
+*[alfajores][alfajores]* in Argentina. They only had one store in Mar del
+Plata (~400km away from Buenos Aires), so it became a tradition to bring these
+exquisite *alfajores* when you returned from a trip to the beach.  Several
+years later they opened stores in Buenos Aires and elsewhere and became a
+coffee shop.
+
+
+[alfajores]:     https://www.google.com.ar/search?q=alfajor+argentino&tbm=isch
+[disc]:          https://github.com/pote/disc
+[disque]:        https://github.com/antirez/disque
+[disque_jockey]: https://github.com/DevinRiley/disque_jockey
+[havanna]:       https://www.google.com.ar/search?q=havanna+cafe&tbm=isch
+[ost-bin]:       https://github.com/djanowski/ost-bin
+[ost]:           https://github.com/soveran/ost
+[pote]:          https://twitter.com/poteland
+[soveran]:       https://twitter.com/soveran

data/bin/havanna

@@ -0,0 +1,137 @@
+#! /usr/bin/env ruby
+
+stop = proc do
+  if defined?(Havanna)
+    Havanna.stop
+  else
+    exit 0
+  end
+end
+
+trap(:INT,  &stop)
+trap(:TERM, &stop)
+
+usage = <<-EOS
+Usage:
+
+  havanna start [-r <require>] [-d] [-p <pid-path>] [-l <log-path>]
+  havanna stop [-p <pid-path>]
+
+EOS
+
+require "clap"
+require_relative "../lib/havanna"
+
+opts = {
+  requires: []
+}
+
+command, _ = Clap.run ARGV,
+  "-d" => -> {
+    opts[:daemonize] = true
+    opts[:log_path] = File.expand_path("havanna.log") unless opts.include?(:log_path)
+  },
+  "-l" => -> path {
+    opts[:log_path] = path
+  },
+  "-p" => -> path {
+    opts[:pid_path] = path
+  },
+  "-s" => -> size {
+    opts[:pool_size] = Integer(size)
+  },
+  "-r" => -> file {
+    opts[:requires] << file
+  },
+  "-v" => -> {
+    require_relative "../lib/havanna/version"
+
+    puts Havanna::VERSION
+
+    exit 0
+  },
+  "-h" => -> {
+    puts(usage)
+    exit 0
+  }
+
+opts[:pid_path] = File.expand_path("havanna.pid") unless opts.include?(:pid_path)
+
+opts[:requires].each do |file|
+  require(file)
+end
+
+module Havanna
+  def self.run(worker)
+    workers << worker
+  end
+
+  def self.workers
+    @workers ||= []
+  end
+end
+
+case command
+when "start"
+  if opts[:daemonize]
+    Process.daemon(true)
+
+    File.open(opts[:pid_path], File::RDWR|File::EXCL|File::CREAT, 0600) do |io|
+      io.write(Process.pid)
+    end
+
+    at_exit do
+      File.delete(opts[:pid_path]) if File.exists?(opts[:pid_path])
+    end
+  end
+
+  if opts[:log_path]
+    $stdout.reopen(opts[:log_path], "a")
+    $stderr.reopen(opts[:log_path], "a")
+  end
+
+  load "./Havannafile"
+
+  opts[:pool_size] = Havanna.workers.size unless opts.include?(:pool_size)
+
+  threads_per_worker = opts[:pool_size] / Havanna.workers.size
+
+  if threads_per_worker == 0
+    abort("Not enough threads for your workers (found #{Havanna.workers.size} workers).")
+  end
+
+  pool = Havanna.workers.each_with_object([]) do |worker, accum|
+    accum.concat(Array.new(threads_per_worker) do
+      Thread.new(worker) do |worker|
+        Thread.current.abort_on_exception = true
+        Havanna.start(worker)
+      end
+    end)
+  end
+
+  pool.each(&:join)
+
+when "stop"
+  if File.exist?(opts[:pid_path])
+    pid = Integer(File.read(opts[:pid_path]).chomp)
+
+    running = true
+
+    Process.kill(:TERM, pid)
+
+    while running
+      begin
+        Process.kill(0, pid)
+        running = true
+      rescue Errno::ESRCH
+        running = false
+      end
+    end
+  end
+
+else
+  $stderr.puts("Unkown command #{command.inspect}.")
+  $stderr.puts(usage)
+
+  exit 2
+end

data/havanna.gemspec

@@ -0,0 +1,19 @@
+require_relative "lib/havanna"
+
+Gem::Specification.new do |s|
+  s.name     = "havanna"
+  s.version  = Havanna::VERSION
+  s.summary  = "Ruby workers for Disque."
+  s.authors  = ["Damian Janowski"]
+  s.email    = ["damian.janowski@gmail.com"]
+  s.homepage = "https://github.com/djanowski/havanna"
+
+  s.files = `git ls-files`.split("\n")
+
+  s.executables << "havanna"
+
+  s.add_dependency "disque"
+  s.add_dependency "clap"
+
+  s.add_development_dependency "cutest"
+end

data/lib/havanna.rb

@@ -0,0 +1,38 @@
+require "disque"
+
+module Havanna
+  VERSION = "1.0.0"
+
+  def self.connect(*args)
+    @connect = args
+    @disque = Disque.new(*args)
+  end
+
+  def self.start(worker)
+    instance = worker.new
+
+    begin
+      disque = Disque.new(*@connect)
+
+      printf("Started worker %s\n", worker)
+
+      loop do
+        disque.fetch(from: [worker.name], timeout: 5000) do |job|
+          instance.call(job)
+        end
+
+        break if @stop
+      end
+    ensure
+      disque.quit
+    end
+  end
+
+  def self.push(*args)
+    @disque.push(*args)
+  end
+
+  def self.stop
+    @stop = true
+  end
+end

data/makefile

@@ -0,0 +1,4 @@
+test:
+	cutest test/havanna_test.rb
+
+.PHONY: test

data/test/havanna_test.rb

@@ -0,0 +1,228 @@
+require "cutest"
+require "timeout"
+require "disque"
+
+at_exit {
+  Process.waitall
+}
+
+def wait_for_pid(pid)
+  wait_for { !running?(pid) }
+end
+
+def wait_for_child(pid)
+  Timeout.timeout(5) do
+    Process.wait(pid)
+  end
+end
+
+def wait_for
+  Timeout.timeout(10) do
+    until value = yield
+      sleep(0.1)
+    end
+
+    return value
+  end
+end
+
+def running?(pid)
+  begin
+    Process.kill(0, pid)
+    true
+  rescue Errno::ESRCH
+    false
+  end
+end
+
+def read_pid_file(path)
+  wait_for { File.exist?(path) && File.size(path) > 0 }
+
+  Integer(File.read(path))
+end
+
+def root(path)
+  File.expand_path("../#{path}", File.dirname(__FILE__))
+end
+
+disque = Disque.new("127.0.0.1:7711")
+
+prepare do
+  disque.call("DEBUG", "FLUSHALL")
+  Dir["test/workers/**/*.pid"].each { |file| File.delete(file) }
+end
+
+test "start" do
+  pid = nil
+
+  begin
+    pid = spawn("#{root("bin/havanna")} start", chdir: "test/workers/echo")
+
+    disque.push("Echo", 2, 5000)
+
+    job = wait_for { disque.fetch(from: ["Echo:result"]) }
+
+    assert_equal "2", job[0][-1]
+  ensure
+    Process.kill(:INT, pid) if pid
+  end
+end
+
+test "gracefully handles TERM signals" do
+  disque.push("Slow", 3, 5000)
+
+  begin
+    spawn("#{root("bin/havanna")} -d start", chdir: "test/workers/slow")
+
+    pid = read_pid_file("./test/workers/slow/havanna.pid")
+
+    assert wait_for { disque.call("QLEN", "Slow:result") == 0 }
+  ensure
+    Process.kill(:TERM, pid) if pid
+  end
+
+  wait_for_pid(pid)
+
+  assert_equal "3", disque.fetch(from: ["Slow:result"])[0][-1]
+end
+
+test "stop waits for workers to be done" do
+  spawn("#{root("bin/havanna")} start -d", chdir: "test/workers/slow")
+
+  pid = read_pid_file("./test/workers/slow/havanna.pid")
+
+  stopper = spawn("#{root("bin/havanna")} stop", chdir: "test/workers/slow")
+
+  # Let the stop command start.
+  wait_for { running?(stopper) }
+
+  # Let the stop command end.
+  wait_for_child(stopper)
+
+  # Immediately after the stop command exits,
+  # havanna(1) shouldn't be running and the pid file
+  # should be gone.
+
+  assert !running?(pid)
+  assert !File.exist?("./test/workers/slow/havanna.pid")
+end
+
+test "use a specific path for the pid file" do
+  pid = nil
+  pid_path = "./test/workers/echo/foo.pid"
+
+  begin
+    spawn("#{root("bin/havanna")} -d start -p foo.pid", chdir: "test/workers/echo")
+
+    pid = read_pid_file(pid_path)
+
+    assert pid
+  ensure
+    Process.kill(:INT, pid) if pid
+  end
+
+  wait_for_pid(pid)
+
+  assert !File.exist?(pid_path)
+end
+
+test "load Havannafile" do
+  pid = nil
+
+  begin
+    pid = spawn("#{root("bin/havanna")} start", chdir: "test/workers/echo")
+
+    disque.push("Echo", 2, 5000)
+
+    value = wait_for { disque.fetch(from: ["Echo:result"]) }
+
+    assert_equal "2", value
+  ensure
+    Process.kill(:INT, pid) if pid
+  end
+end
+
+test "redirect stdout and stderr to a log file when daemonizing" do
+  pid, detached_pid = nil
+
+  pid_path = "./test/workers/logger/havanna.pid"
+
+  log_path = "test/workers/logger/havanna.log"
+
+  File.delete(log_path) if File.exist?(log_path)
+
+  begin
+    pid = spawn("#{root("bin/havanna")} -d start", chdir: "test/workers/logger")
+
+    assert wait_for {
+      `ps -p #{pid} -o state`.lines.to_a.last[/(\w+)/, 1] == "Z"
+    }
+
+    redis.lpush("Logger", 1)
+  ensure
+    detached_pid = read_pid_file(pid_path)
+
+    Process.kill(:INT, pid) if pid
+    Process.kill(:INT, detached_pid) if detached_pid
+  end
+
+  wait_for_pid(detached_pid)
+
+  assert_equal "out: 1\nerr: 1\n", File.read(log_path)
+end
+
+test "redirect stdout and stderr to a different log file when daemonizing" do
+  pid, detached_pid = nil
+
+  pid_path = "./test/workers/logger/havanna.pid"
+
+  log_path = "test/workers/logger/foo.log"
+
+  File.delete(log_path) if File.exist?(log_path)
+
+  begin
+    pid = spawn("#{root("bin/havanna")} -d -l foo.log start", chdir: "test/workers/logger")
+
+    assert wait_for {
+      `ps -p #{pid} -o state`.lines.to_a.last[/(\w+)/, 1] == "Z"
+    }
+
+    redis.lpush("Logger", 1)
+  ensure
+    detached_pid = read_pid_file(pid_path)
+
+    Process.kill(:INT, pid) if pid
+    Process.kill(:INT, detached_pid) if detached_pid
+  end
+
+  wait_for_pid(detached_pid)
+
+  assert_equal "out: 1\nerr: 1\n", File.read(log_path)
+end
+
+test "daemonizes" do
+  pid, detached_pid = nil
+
+  pid_path = "./test/workers/echo/havanna.pid"
+
+  begin
+    pid = spawn("#{root("bin/havanna")} -d start", chdir: "test/workers/echo")
+
+    assert wait_for {
+      `ps -p #{pid} -o state`.lines.to_a.last[/(\w+)/, 1] == "Z"
+    }
+
+    detached_pid = read_pid_file(pid_path)
+
+    ppid = `ps -p #{detached_pid} -o ppid`.lines.to_a.last[/(\d+)/, 1]
+
+    assert_equal "1", ppid
+  ensure
+    Process.kill(:INT, pid) if pid
+    Process.kill(:INT, detached_pid) if detached_pid
+  end
+
+  wait_for_pid(detached_pid)
+
+  assert !File.exist?(pid_path)
+end

data/test/workers/echo/Havannafile

@@ -0,0 +1,3 @@
+require_relative "echo"
+
+Havanna.run(Echo)

data/test/workers/echo/echo.rb

@@ -0,0 +1,7 @@
+Havanna.connect("127.0.0.1:7711")
+
+class Echo
+  def call(job)
+    Havanna.push("Echo:result", job, 5000)
+  end
+end

data/test/workers/logger/Havannafile

@@ -0,0 +1,3 @@
+require_relative "logger"
+
+Havanna.run(Logger)

data/test/workers/logger/logger.rb

@@ -0,0 +1,8 @@
+Havanna.connect("127.0.0.1:7711")
+
+class Logger
+  def call(id)
+    $stdout.puts("out: #{id}")
+    $stderr.puts("err: #{id}")
+  end
+end

data/test/workers/slow/Havannafile

@@ -0,0 +1,3 @@
+require_relative "slow"
+
+Havanna.run(Slow)

data/test/workers/slow/slow.rb

@@ -0,0 +1,8 @@
+Havanna.connect("127.0.0.1:7711")
+
+class Slow
+  def call(n)
+    sleep(n.to_i)
+    Havanna.push("Slow:result", n, 5000)
+  end
+end

metadata

@@ -0,0 +1,100 @@
+--- !ruby/object:Gem::Specification
+name: havanna
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Damian Janowski
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-07-11 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: disque
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: clap
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: cutest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- damian.janowski@gmail.com
+executables:
+- havanna
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- LICENSE
+- README.md
+- bin/havanna
+- havanna.gemspec
+- lib/havanna.rb
+- makefile
+- test/havanna_test.rb
+- test/workers/echo/Havannafile
+- test/workers/echo/echo.rb
+- test/workers/logger/Havannafile
+- test/workers/logger/logger.rb
+- test/workers/slow/Havannafile
+- test/workers/slow/slow.rb
+homepage: https://github.com/djanowski/havanna
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.6
+signing_key: 
+specification_version: 4
+summary: Ruby workers for Disque.
+test_files: []