qs 0.0.1 → 0.1.0

data/.gitignore

@@ -1,6 +1,7 @@
 *.gem
 *.log
 *.rbc
+.rbx/
 .bundle
 .config
 .yardoc

data/Gemfile

@@ -1,5 +1,10 @@
-source "https://rubygems.org"
+source 'https://rubygems.org'
 
 gemspec
 
 gem 'rake'
+gem 'pry', "~> 0.9.0"
+
+platform :ruby_18 do
+  gem 'json', '~> 1.8'
+end

data/LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2010-Present Kelly Redding and Collin Redding
+Copyright (c) 2013-Present Kelly Redding and Collin Redding
 
 MIT License
 

data/bench/config.qs

@@ -0,0 +1,46 @@
+require 'qs'
+require 'bench/queue'
+
+ROOT_PATH = Pathname.new(File.expand_path('../..', __FILE__))
+
+LOGGER = if ENV['BENCH_REPORT']
+  Logger.new(ROOT_PATH.join('log/bench_daemon.log').to_s)
+else
+  Logger.new(STDOUT)
+end
+
+PROGRESS_IO = if ENV['BENCH_PROGRESS_IO']
+  ::IO.for_fd(ENV['BENCH_PROGRESS_IO'].to_i)
+else
+  File.open('/dev/null', 'w')
+end
+
+class BenchDaemon
+  include Qs::Daemon
+
+  name     'bench'
+  pid_file ROOT_PATH.join('tmp/bench_daemon.pid').to_s
+
+  logger LOGGER
+  verbose_logging false
+
+  queue BenchQueue
+
+  # if jobs fail notify the bench report so it doesn't hang forever on IO.select
+  error do |exception, daemon_data, job|
+    PROGRESS_IO.write_nonblock('F')
+  end
+
+  class Multiply
+    include Qs::JobHandler
+
+    after{ PROGRESS_IO.write_nonblock('.') }
+
+    def run!
+      'a' * params['size']
+    end
+  end
+
+end
+
+run BenchDaemon.new

data/bench/queue.rb

@@ -0,0 +1,8 @@
+require 'qs'
+require 'json'
+
+BenchQueue = Qs::Queue.new do
+  name 'bench'
+
+  job 'multiply', 'BenchDaemon::Multiply'
+end

data/bench/report.rb

@@ -0,0 +1,114 @@
+require 'benchmark'
+require 'scmd'
+require 'bench/queue'
+
+class BenchRunner
+
+  TIME_MODIFIER = 10 ** 4 # 4 decimal places
+
+  def initialize
+    output_file_path = if ENV['OUTPUT_FILE']
+      File.expand_path(ENV['OUTPUT_FILE'])
+    else
+      File.expand_path('../report.txt', __FILE__)
+    end
+    @output_file = File.open(output_file_path, 'w')
+
+    @number_of_jobs = ENV['NUM_JOBS'] || 10_000
+    @job_name       = 'multiply'
+    @job_params     = { 'size' => 100_000 }
+
+    @progress_reader, @progress_writer = IO.pipe
+
+    @results = {}
+
+    Qs.init
+  end
+
+  def run
+    output "Running benchmark report..."
+    output("\n", false)
+
+    benchmark_adding_jobs
+    benchmark_running_jobs
+
+    size = @results.values.map(&:size).max
+    output "Adding #{@number_of_jobs} Jobs Time:  #{@results[:adding_jobs].rjust(size)}s"
+    output "Running #{@number_of_jobs} Jobs Time: #{@results[:running_jobs].rjust(size)}s"
+
+    output "\n"
+    output "Done running benchmark report"
+  end
+
+  private
+
+  def benchmark_adding_jobs
+    output "Adding jobs"
+    benchmark = Benchmark.measure do
+      (1..@number_of_jobs).each do |n|
+        BenchQueue.add(@job_name, @job_params)
+        output('.', false) if ((n - 1) % 100 == 0)
+      end
+    end
+    @results[:adding_jobs] = round_and_display(benchmark.real)
+    output("\n", false)
+  end
+
+  def benchmark_running_jobs
+    cmd_str = "bundle exec ./bin/qs bench/config.qs"
+    cmd = Scmd.new(cmd_str, {
+      'BENCH_REPORT'      => 'yes',
+      'BENCH_PROGRESS_IO' => @progress_writer.fileno
+    })
+
+    output "Running jobs"
+    begin
+      benchmark = Benchmark.measure do
+        cmd.start
+        if !cmd.running?
+          raise "failed to start qs process: #{cmd_str.inspect}"
+        end
+
+        progress = 0
+        while progress < @number_of_jobs
+          ::IO.select([@progress_reader])
+          result = @progress_reader.read_nonblock(1)
+          progress += 1
+          output(result, false) if ((progress - 1) % 100 == 0)
+        end
+      end
+      @results[:running_jobs] = round_and_display(benchmark.real)
+      output("\n", false)
+    ensure
+      cmd.kill('TERM')
+      cmd.wait(5)
+    end
+
+    output("\n", false)
+  end
+
+  private
+
+  def output(message, puts = true)
+    method = puts ? :puts : :print
+    self.send(method, message)
+    @output_file.send(method, message)
+    STDOUT.flush if method == :print
+  end
+
+  def round_and_display(time_in_ms)
+    display_time(round_time(time_in_ms))
+  end
+
+  def round_time(time_in_ms)
+    (time_in_ms * TIME_MODIFIER).to_i / TIME_MODIFIER.to_f
+  end
+
+  def display_time(time)
+    integer, fractional = time.to_s.split('.')
+    [ integer, fractional.ljust(4, '0') ].join('.')
+  end
+
+end
+
+BenchRunner.new.run

data/bench/report.txt

@@ -0,0 +1,11 @@
+Running benchmark report...
+
+Adding jobs
+....................................................................................................
+Running jobs
+....................................................................................................
+
+Adding 10000 Jobs Time:   1.6041s
+Running 10000 Jobs Time: 12.3159s
+
+Done running benchmark report

data/bin/qs

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+#
+# Copyright (c) 2013 Collin Redding & Kelly Redding
+#
+
+require 'qs/cli'
+Qs::CLI.run ARGV

data/lib/qs/cli.rb

@@ -0,0 +1,124 @@
+require 'qs'
+require 'qs/config_file'
+require 'qs/process'
+require 'qs/process_signal'
+require 'qs/version'
+
+module Qs
+
+  class CLI
+
+    def self.run(args)
+      self.new.run(*args)
+    end
+
+    def initialize(kernel = nil)
+      @kernel = kernel || Kernel
+      @cli = CLIRB.new
+    end
+
+    def run(*args)
+      begin
+        run!(*args)
+      rescue CLIRB::HelpExit
+        @kernel.puts help
+      rescue CLIRB::VersionExit
+        @kernel.puts Qs::VERSION
+      rescue CLIRB::Error, Qs::ConfigFile::InvalidError => exception
+        @kernel.puts "#{exception.message}\n\n"
+        @kernel.puts help
+        @kernel.exit 1
+      rescue StandardError => exception
+        @kernel.puts "#{exception.class}: #{exception.message}"
+        @kernel.puts exception.backtrace.join("\n")
+        @kernel.exit 1
+      end
+      @kernel.exit 0
+    end
+
+    private
+
+    def run!(*args)
+      @cli.parse!(args)
+      config_file_path, command = @cli.args
+      config_file_path ||= 'config.qs'
+      command ||= 'run'
+      daemon = Qs::ConfigFile.new(config_file_path).daemon
+      case(command)
+      when 'run'
+        Qs::Process.new(daemon, :daemonize => false).run
+      when 'start'
+        Qs::Process.new(daemon, :daemonize => true).run
+      when 'stop'
+        Qs::ProcessSignal.new(daemon, 'TERM').send
+      when 'restart'
+        Qs::ProcessSignal.new(daemon, 'USR2').send
+      else
+        raise CLIRB::Error, "#{command.inspect} is not a valid command"
+      end
+    end
+
+    def help
+      "Usage: qs [CONFIG_FILE] [COMMAND]\n\n" \
+      "Commands: run, start, stop, restart\n" \
+      "Options: #{@cli}"
+    end
+
+  end
+
+  class CLIRB  # Version 1.0.0, https://github.com/redding/cli.rb
+    Error    = Class.new(RuntimeError);
+    HelpExit = Class.new(RuntimeError); VersionExit = Class.new(RuntimeError)
+    attr_reader :argv, :args, :opts, :data
+
+    def initialize(&block)
+      @options = []; instance_eval(&block) if block
+      require 'optparse'
+      @data, @args, @opts = [], [], {}; @parser = OptionParser.new do |p|
+        p.banner = ''; @options.each do |o|
+          @opts[o.name] = o.value; p.on(*o.parser_args){ |v| @opts[o.name] = v }
+        end
+        p.on_tail('--version', ''){ |v| raise VersionExit, v.to_s }
+        p.on_tail('--help',    ''){ |v| raise HelpExit,    v.to_s }
+      end
+    end
+
+    def option(*args); @options << Option.new(*args); end
+    def parse!(argv)
+      @args = (argv || []).dup.tap do |args_list|
+        begin; @parser.parse!(args_list)
+        rescue OptionParser::ParseError => err; raise Error, err.message; end
+      end; @data = @args + [@opts]
+    end
+    def to_s; @parser.to_s; end
+    def inspect
+      "#<#{self.class}:#{'0x0%x' % (object_id << 1)} @data=#{@data.inspect}>"
+    end
+
+    class Option
+      attr_reader :name, :opt_name, :desc, :abbrev, :value, :klass, :parser_args
+
+      def initialize(name, *args)
+        settings, @desc = args.last.kind_of?(::Hash) ? args.pop : {}, args.pop || ''
+        @name, @opt_name, @abbrev = parse_name_values(name, settings[:abbrev])
+        @value, @klass = gvalinfo(settings[:value])
+        @parser_args = if [TrueClass, FalseClass, NilClass].include?(@klass)
+          ["-#{@abbrev}", "--[no-]#{@opt_name}", @desc]
+        else
+          ["-#{@abbrev}", "--#{@opt_name} #{@opt_name.upcase}", @klass, @desc]
+        end
+      end
+
+      private
+
+      def parse_name_values(name, custom_abbrev)
+        [ (processed_name = name.to_s.strip.downcase), processed_name.gsub('_', '-'),
+          custom_abbrev || processed_name.gsub(/[^a-z]/, '').chars.first || 'a'
+        ]
+      end
+      def gvalinfo(v); v.kind_of?(Class) ? [nil,gklass(v)] : [v,gklass(v.class)]; end
+      def gklass(k); k == Fixnum ? Integer : k; end
+    end
+  end
+
+end

data/lib/qs/client.rb

@@ -0,0 +1,121 @@
+require 'hella-redis'
+require 'qs'
+require 'qs/job'
+require 'qs/queue'
+
+module Qs
+
+  module Client
+
+    def self.new(*args)
+      if !ENV['QS_TEST_MODE']
+        QsClient.new(*args)
+      else
+        TestClient.new(*args)
+      end
+    end
+
+    def self.included(klass)
+      klass.class_eval do
+        include InstanceMethods
+      end
+    end
+
+    module InstanceMethods
+
+      attr_reader :redis_config, :redis
+
+      def initialize(redis_config)
+        @redis_config = redis_config
+      end
+
+      def enqueue(queue, job_name, params = nil)
+        job = Qs::Job.new(job_name, params || {})
+        enqueue!(queue, job)
+        job
+      end
+
+      def push(queue_name, payload)
+        raise NotImplementedError
+      end
+
+      def block_dequeue(*args)
+        self.redis.with{ |c| c.brpop(*args) }
+      end
+
+      def append(queue_redis_key, serialized_payload)
+        self.redis.with{ |c| c.lpush(queue_redis_key, serialized_payload) }
+      end
+
+      def prepend(queue_redis_key, serialized_payload)
+        self.redis.with{ |c| c.rpush(queue_redis_key, serialized_payload) }
+      end
+
+      def clear(redis_key)
+        self.redis.with{ |c| c.del(redis_key) }
+      end
+
+    end
+
+  end
+
+  class QsClient
+    include Client
+
+    def initialize(*args)
+      super
+      @redis = HellaRedis::Connection.new(self.redis_config)
+    end
+
+    def push(queue_name, payload)
+      queue_redis_key    = Queue::RedisKey.new(queue_name)
+      serialized_payload = Qs.serialize(payload)
+      self.append(queue_redis_key, serialized_payload)
+    end
+
+    private
+
+    def enqueue!(queue, job)
+      serialized_payload = Qs.serialize(job.to_payload)
+      self.append(queue.redis_key, serialized_payload)
+    end
+
+  end
+
+  class TestClient
+    include Client
+
+    attr_reader :pushed_items
+
+    def initialize(*args)
+      super
+      require 'hella-redis/connection_spy'
+      @redis = HellaRedis::ConnectionSpy.new(self.redis_config)
+      @pushed_items = []
+    end
+
+    def push(queue_name, payload)
+      # attempt to serialize (and then throw away) the payload, this will error
+      # on the developer if it can't be serialized
+      Qs.serialize(payload)
+      @pushed_items << PushedItem.new(queue_name, payload)
+    end
+
+    def reset!
+      @pushed_items.clear
+    end
+
+    private
+
+    def enqueue!(queue, job)
+      # attempt to serialize (and then throw away) the job payload, this will
+      # error on the developer if it can't serialize the job
+      Qs.serialize(job.to_payload)
+      queue.enqueued_jobs << job
+    end
+
+    PushedItem = Struct.new(:queue_name, :payload)
+
+  end
+
+end

data/lib/qs/config_file.rb

@@ -0,0 +1,79 @@
+require 'qs/daemon'
+
+module Qs
+
+  class ConfigFile
+
+    attr_reader :daemon
+
+    def initialize(file_path)
+      @file_path = build_file_path(file_path)
+      @daemon = nil
+      evaluate_file(@file_path)
+      validate!
+    end
+
+    def run(daemon)
+      @daemon = daemon
+    end
+
+    private
+
+    def validate!
+      if !@daemon.kind_of?(Qs::Daemon)
+        raise NoDaemonError.new(@daemon, @file_path)
+      end
+    end
+
+    def build_file_path(path)
+      full_path = File.expand_path(path)
+      raise NoConfigFileError.new(full_path) unless File.exists?(full_path)
+      full_path
+    rescue NoConfigFileError
+      full_path_with_sanford = "#{full_path}.qs"
+      raise unless File.exists?(full_path_with_sanford)
+      full_path_with_sanford
+    end
+
+    # This evaluates the file and creates a proc using it's contents. This is
+    # a trick borrowed from Rack. It is essentially converting a file into a
+    # proc and then instance eval'ing it. This has a couple benefits:
+    # * The obvious benefit is the file is evaluated in the context of this
+    #   class. This allows the file to call `run`, setting the daemon that
+    #   will be used.
+    # * The other benefit is that the file's contents behave like they were a
+    #   proc defined by the user. Instance eval'ing the file directly, makes
+    #   any constants (modules/classes) defined in it namespaced by the
+    #   instance of the config (not namespaced by `Qs::ConfigFile`,
+    #   they are actually namespaced by an instance of this class, its like
+    #   accessing it via `ConfigFile.new::MyDaemon`), which is very confusing.
+    #   Thus, the proc is created and eval'd using the `TOPLEVEL_BINDING`,
+    #   which defines the constants at the top-level, as would be expected.
+    def evaluate_file(file_path)
+      config_file_code = "proc{ #{File.read(file_path)} }"
+      config_file_proc = eval(config_file_code, TOPLEVEL_BINDING, file_path, 0)
+      self.instance_eval(&config_file_proc)
+    end
+
+    InvalidError = Class.new(StandardError)
+
+    class NoConfigFileError < InvalidError
+      def initialize(path)
+        super "A configuration file couldn't be found at: #{path.to_s.inspect}"
+      end
+    end
+
+    class NoDaemonError < InvalidError
+      def initialize(daemon, path)
+        prefix = "Configuration file #{path.to_s.inspect}"
+        if daemon
+          super "#{prefix} called `run` without a Qs::Daemon"
+        else
+          super "#{prefix} didn't call `run` with a Qs::Daemon"
+        end
+      end
+    end
+
+  end
+
+end