async-jobs 0.0.3

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ad313be63e58bd780e2b68297719ec64cb7f46e6
+  data.tar.gz: 4f341fc468bf497f4266185e930351c3a1c6ce1b
+SHA512:
+  metadata.gz: eff272909fab9a45836846e6576762ef7e9da0bee6442e47aa769a3bc3be2d8eafdf0e33bed032dd37332ec69b410b0a1fe71d2e13ba25093585e571a8600352
+  data.tar.gz: 0fe48659523783bd0314e8c5b1e6425fd99bdba12f721c245bc36589d745c90466e73003e51388a8d1e573f4eaca3624aa6241f55446960856f8fe3f4ef3daee

data/.gitignore

@@ -0,0 +1 @@
+async*.gem

data/Gemfile

@@ -0,0 +1,12 @@
+source 'https://rubygems.org'
+
+gemspec
+
+group :test do
+  gem 'redis'
+  gem 'cubbyhole', :require => false
+  gem 'qu'
+  gem 'rspec'
+  gem 'resque'
+  gem 'sidekiq'
+end

data/Gemfile.lock

@@ -0,0 +1,66 @@
+PATH
+  remote: .
+  specs:
+    async-jobs (0.0.3)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    celluloid (0.12.4)
+      facter (>= 1.6.12)
+      timers (>= 1.0.0)
+    connection_pool (1.0.0)
+    cubbyhole (0.2.1)
+    diff-lcs (1.1.3)
+    facter (1.6.18)
+    multi_json (1.7.0)
+    qu (0.2.0)
+      multi_json
+    rack (1.4.5)
+    rack-protection (1.4.0)
+      rack
+    redis (3.0.2)
+    redis-namespace (1.2.1)
+      redis (~> 3.0.0)
+    resque (1.23.0)
+      multi_json (~> 1.0)
+      redis-namespace (~> 1.0)
+      sinatra (>= 0.9.2)
+      vegas (~> 0.1.2)
+    rspec (2.11.0)
+      rspec-core (~> 2.11.0)
+      rspec-expectations (~> 2.11.0)
+      rspec-mocks (~> 2.11.0)
+    rspec-core (2.11.1)
+    rspec-expectations (2.11.2)
+      diff-lcs (~> 1.1.3)
+    rspec-mocks (2.11.1)
+    sidekiq (2.8.0)
+      celluloid (~> 0.12.0)
+      connection_pool (~> 1.0)
+      multi_json (~> 1)
+      redis (~> 3)
+      redis-namespace
+    sinatra (1.3.5)
+      rack (~> 1.4)
+      rack-protection (~> 1.3)
+      tilt (~> 1.3, >= 1.3.3)
+    tilt (1.3.4)
+    timers (1.1.0)
+    vegas (0.1.11)
+      rack (>= 1.0.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  async-jobs!
+  cubbyhole
+  qu
+  redis
+  resque
+  rspec
+  sidekiq
+
+BUNDLED WITH
+   1.12.5

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) Engine Yard. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,28 @@
+#Yet another background processing abstraction layer
+
+Assuming that every time you want to do something in a background job, it's defined in a method on an active record object.
+
+Zero explicit dependencies. (just respond to `id` and `find` like AR does)
+
+##Example
+
+gem is called `async-jobs`
+
+```ruby
+  require 'async'
+  require 'async/resque'
+  Async.backend = Async::ResqueBackend
+
+  class Invoice < ActiveRecord::Base
+    def process(arg)
+      Async.run{ process_now(arg)}
+    end
+    def process_now(arg)
+      #actually do it
+    end
+  end
+
+  invoice.process 1
+```
+
+Will enqueue a Resque job that runs `invoice.process_now 1`

data/async.gemspec

@@ -0,0 +1,17 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "async/version"
+
+Gem::Specification.new do |s|
+  s.name = "async-jobs"
+  s.version = Async::VERSION
+  s.author = "Jacob"
+  s.email = "jacob@engineyard.com"
+  s.homepage = "https://github.com/engineyard/async"
+  s.summary = "abstraction over background job systems"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.require_paths = ['lib']
+
+end

data/lib/async.rb

@@ -0,0 +1,119 @@
+class Async
+  class << self
+    attr_accessor :backend
+  end
+
+  class MethodCatcher
+    attr_reader :_method_name, :_args
+    def method_missing(method_name, *args)
+      @_method_name = method_name
+      @_args = args
+    end
+  end
+
+  def self.ensure_backend!
+    unless Async.backend
+      raise "Please configure the background processing system of choice by setting: Async.backend="
+    end
+  end
+
+  def self.run(&block)
+    ensure_backend!
+    receiver = block.binding.eval("self")
+    mc = MethodCatcher.new
+    mc.instance_eval(&block)
+    run_later self.to_s, receiver, mc._method_name, *mc._args
+  end
+
+  def self.run_later(job_class, receiver, method_name, *args)
+    if receiver.is_a?(Class)
+      receiver_class, receiver_id = receiver.to_s, nil
+    else
+      receiver_class, receiver_id = receiver.class.to_s, receiver.id
+    end
+    Async.backend.enqueue Async.backend.job_class, job_class, receiver_class, receiver_id, method_name, Job.transform_args(args)
+  end
+
+  def self.run_now(receiver, method_name, args)
+    Notifications.notify_job("run", receiver, method_name, args)
+    receiver.send(method_name, *args)
+  ensure
+    Notifications.notify_job("finish", receiver, method_name, args)
+  end
+
+  #TODO: test this
+  class Notifications
+    class << self
+      attr_accessor :handler
+    end
+    def self.notify_lock(thing, lock_name)
+      handler && handler.call(thing, {:lock_name => lock_name})
+    end
+    def self.notify_job(thing, receiver, method_name, args)
+      handler && handler.call(thing, {
+        :receiver => receiver,
+        :method_name => method_name.to_sym,
+        :args => args
+      })
+    end
+  end
+
+  #TODO: test this
+  class ErrorReporting
+    class << self
+      attr_accessor :handler
+    end
+    def self.notify_exception(e, job_args)
+      handler && handler.call(e, job_args)
+    end
+  end
+
+  class Job
+
+    def self.perform(wrapper, receiver_class_str, receiver_id, method, args)
+      receiver_class = constantize(receiver_class_str)
+      receiver = receiver_id ? receiver_class.find(receiver_id) : receiver_class
+      untransform_args = untransform_args(args)
+      constantize(wrapper).run_now(receiver, method, untransform_args)
+    rescue => e
+      ErrorReporting.notify_exception(e, 
+        receiver_class_str: receiver_class_str, receiver_id: receiver_id, method: method, args: args)
+    end
+
+    def self.transform_args(args)
+      args.map do |x|
+        if x.class.respond_to?(:find)
+          {'_transform_arg' => true, 'class' => x.class.to_s, 'id' => x.id}
+        elsif x.is_a?(Array)
+          transform_args(x)
+        else
+          x
+        end
+      end
+    end
+
+    def self.untransform_args(args)
+      args.map do |x|
+        if x.is_a?(Hash) && x['_transform_arg']
+          constantize(x['class']).find(x['id'])
+        elsif x.is_a?(Array)
+          untransform_args(x)
+        else
+          x
+        end
+      end
+    end
+
+    private
+
+    def self.constantize(str)
+      if str.respond_to?(:constantize)
+        str.constantize
+      else
+        eval(str)
+      end
+    end
+
+  end
+
+end

data/lib/async/locked.rb

@@ -0,0 +1,141 @@
+class Async
+  class << self
+    attr_accessor :redis, :lock_time
+  end
+
+  class Locked < Async
+
+    def self.run_now(receiver, method_name, args)
+      Notifications.notify_job("consider", receiver, method_name, args)
+      if Lock.is_lock_arg?(args.first)
+        lock_arg = args.shift
+        lock = Lock.claim(make_lock_name(receiver)) || Lock.create(make_lock_name(receiver))
+      else
+        lock = Lock.create(make_lock_name(receiver))
+      end
+      if lock
+        super
+      else
+        run_later(self.to_s, receiver, method_name, *args)
+      end
+    ensure
+      lock && lock.release
+    end
+
+    def self.run_later(job_class, receiver, method_name, *args)
+      if lock = Lock.pass_on(make_lock_name(receiver))
+        super(job_class, receiver, method_name, lock.as_job_arg, *args)
+      else
+        super
+      end
+    end
+
+    def self.make_lock_name(receiver)
+      if receiver.is_a?(Class)
+        Lock.make_name(receiver.to_s, nil)
+      else
+        Lock.make_name(receiver.class.to_s, receiver.id)
+      end
+    end
+
+  end
+
+  class Lock
+    def self.make_name(*names)
+      "lock:"+names.join(":")
+    end
+
+    def initialize(lock_name)
+      @lock_name = lock_name
+      @passed_on = false
+    end
+
+    def self.is_lock_arg?(arg)
+      arg.is_a?(Hash) && arg["_lock_arg"]
+    end
+
+    def as_job_arg
+      {"_lock_arg" => true, 'lock_name' => @lock_name}
+    end
+
+    def claim
+      Notifications.notify_lock("claim", @lock_name)
+      if redis.get(@lock_name) #still locked
+        refresh!
+        return true
+      else
+        lock
+      end
+    end
+
+    def lock
+      if redis.setnx(@lock_name, "locked")
+        refresh!
+        Notifications.notify_lock("lock", @lock_name)
+        return true
+      else
+        return false
+      end
+    end
+
+    def refresh!
+      redis.expire(@lock_name, Async::Locked.lock_time || 15)
+    end
+
+    def pass_on
+      if @passed_on
+        #already passed on, can't pass again
+        false
+      else
+        @passed_on = true
+        true
+      end
+    end
+
+    def release
+      if @passed_on
+        return false
+      end
+      redis.del(@lock_name)
+      Notifications.notify_lock("release", @lock_name)
+      Thread.current["Async::Lock.named"][@lock_name] = nil
+      return true
+    end
+
+    def redis
+      Async::Locked.redis
+    end
+
+    def self.thread_kill_lock(lock_name)
+      Thread.current["Async::Lock.named"] ||= {}
+      Thread.current["Async::Lock.named"][lock_name] = nil
+    end
+
+    def self.thread_save_lock(lock_name, lock)
+      Thread.current["Async::Lock.named"] ||= {}
+      Thread.current["Async::Lock.named"][lock_name] = lock
+    end
+
+    def self.thread_fetch_lock(lock_name)
+      Thread.current["Async::Lock.named"] ||= {}
+      Thread.current["Async::Lock.named"][lock_name]
+    end
+
+    def self.claim(lock_name)
+      lock = Lock.new(lock_name)
+      lock.claim && thread_save_lock(lock_name, lock) && lock
+    end
+
+    def self.pass_on(lock_name)
+      lock = thread_fetch_lock(lock_name)
+      lock && lock.pass_on && lock
+    end
+
+    def self.create(lock_name)
+      lock = Lock.new(lock_name)
+      lock.lock && thread_save_lock(lock_name, lock) && lock
+    end
+
+  end
+
+end

data/lib/async/qu.rb

@@ -0,0 +1,15 @@
+class Async
+
+  class QuBackend
+
+    def self.enqueue(job_class, *args)
+      Qu.enqueue(job_class, *args)
+    end
+
+    def self.job_class
+      Async::Job
+    end
+
+  end
+
+end

data/lib/async/resque.rb

@@ -0,0 +1,19 @@
+class Async
+
+  class ResqueBackend
+
+    def self.enqueue(job_class, *args)
+      Resque.enqueue(job_class, *args)
+    end
+
+    def self.job_class
+      Async::ResqueBackend::Job
+    end
+
+    class Job < Async::Job
+      @queue = :default
+    end
+
+  end
+
+end

data/lib/async/sidekiq.rb

@@ -0,0 +1,25 @@
+require 'sidekiq'
+class Async
+
+  class SidekiqBackend
+
+    def self.enqueue(job_class, *args)
+      job_class.perform_async(*args)
+    end
+
+    def self.job_class
+      Async::SidekiqBackend::Job
+    end
+
+    class Job < Async::Job
+      include Sidekiq::Worker
+
+      def perform(*args)
+        self.class.perform(*args)
+      end
+
+    end
+
+  end
+
+end

data/lib/async/version.rb

@@ -0,0 +1,3 @@
+class Async
+  VERSION = "0.0.3"
+end

data/spec/async_and_lock_spec.rb

@@ -0,0 +1,121 @@
+require 'async'
+require 'async/qu'
+require 'async/locked'
+require 'qu'
+require 'redis'
+require 'shared/models'
+
+require 'qu-immediate'
+class PoppableQu < Qu::Backend::Immediate
+
+  def self.queue
+    @queue ||= []
+  end
+
+  def enqueue(payload)
+    PoppableQu.queue.unshift(payload)
+  end
+
+end
+
+describe Async::Locked do
+  before(:all) do
+    Async.backend = Async::QuBackend
+    Async::Locked.redis = Redis.new
+  end
+  before(:each) do
+    @qubackend = Qu.backend
+    Qu.backend = PoppableQu.new
+  end
+  after(:each) do
+    Qu.backend = @qubackend
+  end
+
+  it "works" do
+    Async::Locked.redis.flushdb
+
+    order_of_ops = []
+    Async::Notifications.handler = Proc.new do |name, hash|
+      # puts [name, (hash[:method_name] || hash[:lock_name]).to_s, hash[:args]].inspect
+      order_of_ops << [name, (hash[:method_name] || hash[:lock_name]).to_s]
+    end
+
+    Thread.current["Async::Lock.named"] = nil
+    y = Yard.new
+    y.save
+    y.do_all_the_work("front")
+    y.conflict
+
+    PoppableQu.queue.size.should eq 2
+    next_job = PoppableQu.queue.pop
+    next_job.args.should eq ["Async::Locked", "Yard", y.id, :now_do_all_the_work, ["front"]]
+
+    # puts ""
+    Thread.current["Async::Lock.named"] = nil
+    next_job.perform
+
+    PoppableQu.queue.size.should eq 3
+    next_job = PoppableQu.queue.pop
+    next_job.args.should eq ["Async::Locked", "Yard", 0, :now_conflict, []]
+
+    # puts ""
+    Thread.current["Async::Lock.named"] = nil
+    next_job.perform
+
+    PoppableQu.queue.size.should eq 3
+    next_job = PoppableQu.queue.pop
+    next_job.args.should eq ["Async::Locked", "Yard", y.id, :now_trim, [{"_lock_arg"=>true, "lock_name"=>"lock:Yard:0"}, "front"]]
+
+    # puts ""
+    Thread.current["Async::Lock.named"] = nil
+    next_job.perform
+
+    PoppableQu.queue.size.should eq 2
+    next_job = PoppableQu.queue.pop
+    next_job.args.should eq ["Async", "Yard", y.id, :now_mow, ["front"]]
+
+    # puts ""
+    Thread.current["Async::Lock.named"] = nil
+    next_job.perform
+
+    PoppableQu.queue.size.should eq 2
+    next_job = PoppableQu.queue.pop
+    next_job.args.should eq ["Async::Locked", "Yard", 0, :now_conflict, []]
+
+    # puts ""
+    Thread.current["Async::Lock.named"] = nil
+    next_job.perform
+
+    PoppableQu.queue.size.should eq 1
+    next_job = PoppableQu.queue.pop
+    next_job.args.should eq ["Async", "Yard", nil, :now_burn, []]
+
+    # puts ""
+    Thread.current["Async::Lock.named"] = nil
+    next_job.perform
+
+    PoppableQu.queue.size.should eq 0
+
+    order_of_ops.should eq [
+      ["consider", "now_do_all_the_work"],
+      ["lock", "lock:Yard:0"],
+      ["run", "now_do_all_the_work"],
+      ["finish", "now_do_all_the_work"],
+      ["consider", "now_conflict"],
+      ["consider", "now_trim"],
+      ["claim", "lock:Yard:0"],
+      ["run", "now_trim"],
+      ["finish", "now_trim"],
+      ["release", "lock:Yard:0"],
+      ["run", "now_mow"],
+      ["finish", "now_mow"],
+      ["consider", "now_conflict"],
+      ["lock", "lock:Yard:0"],
+      ["run", "now_conflict"],
+      ["finish", "now_conflict"],
+      ["release", "lock:Yard:0"],
+      ["run", "now_burn"],
+      ["finish", "now_burn"]]
+  end
+
+end

data/spec/basics_spec.rb

@@ -0,0 +1,40 @@
+require 'async'
+require 'shared/models'
+
+class TestBackend
+  def initialize
+    @jobs = []
+  end
+  def enqueue(job_class, *args)
+    @jobs << [job_class, args]
+  end
+  def run_all_jobs!
+    while(job = @jobs.pop)
+      job_class, args = job
+      job_class.perform(*args)
+    end
+  end
+  def job_class
+    Async::Job
+  end
+end
+
+describe "Async basics" do
+  before(:all) do
+    @test_backend = TestBackend.new
+    Async.backend = @test_backend
+  end
+
+  it "works" do
+    y = Yard.new
+    y.save
+    y.mowed.should be_nil
+    y.mow("front")
+
+    @test_backend.run_all_jobs!
+
+    y.reload
+    y.mowed.should eq "front"
+  end
+
+end

data/spec/qu_spec.rb

@@ -0,0 +1,23 @@
+require 'async'
+require 'async/qu'
+require 'shared/models'
+require 'qu'
+require 'qu-immediate'
+
+describe "Async qu" do
+  before(:all) do
+    Qu.backend = Qu::Backend::Immediate.new
+    Async.backend = Async::QuBackend
+  end
+
+  it "works" do
+    y = Yard.new
+    y.save
+    y.mowed.should be_nil
+    y.mow("front")
+
+    y.reload
+    y.mowed.should eq "front"
+  end
+
+end

data/spec/resque_spec.rb

@@ -0,0 +1,28 @@
+require 'async'
+require 'async/resque'
+require 'shared/models'
+require 'redis'
+require 'resque'
+
+describe "Async resque" do
+  before(:all) do
+    Resque.redis = Redis.new
+    Async.backend = Async::ResqueBackend
+  end
+
+  it "works" do
+    y = Yard.new
+    y.save
+    y.mowed.should be_nil
+    y.mow("front")
+
+    worker = Resque::Worker.new("*")
+    while job = worker.reserve
+      job.perform
+    end
+
+    y.reload
+    y.mowed.should eq "front"
+  end
+
+end

data/spec/shared/models.rb

@@ -0,0 +1,54 @@
+require 'cubbyhole/base'
+class Yard < Cubbyhole::Base
+
+  def self.backend
+    @backend ||= Hash.new
+  end
+
+  def self.find(id)
+    get(id)
+  end
+
+  def do_all_the_work(which)
+    Async::Locked.run{ now_do_all_the_work(which) }
+  end
+
+  def now_do_all_the_work(which)
+    # puts "do_all_the_work"
+    trim(which)
+    mow(which)
+  end
+
+  def trim(which)
+    Async::Locked.run{ now_trim(which) }
+  end
+
+  def now_trim(which)
+    # puts "trim #{which}"
+  end
+
+  def mow(which)
+    Async.run{ now_mow(which) }
+  end
+  def now_mow(which)
+    # puts "mow #{which}"
+    self.mowed = which
+    self.save
+    Yard.burn
+  end
+
+  def self.burn
+    Async.run{ now_burn }
+  end
+  def self.now_burn
+    # puts "burning..."
+  end
+
+  def conflict
+    Async::Locked.run{ now_conflict }
+  end
+  def now_conflict
+    # puts "conflict"
+  end
+
+end

data/spec/sidekiq_spec.rb

@@ -0,0 +1,23 @@
+require 'async'
+require 'async/sidekiq'
+require 'shared/models'
+require 'sidekiq/testing'
+
+describe "Async sidekiq" do
+  before(:all) do
+    Async.backend = Async::SidekiqBackend
+  end
+
+  it "works" do
+    y = Yard.new
+    y.save
+    y.mowed.should be_nil
+    y.mow("front")
+
+    Async::SidekiqBackend::Job.drain
+
+    y.reload
+    y.mowed.should eq "front"
+  end
+
+end

metadata

@@ -0,0 +1,67 @@
+--- !ruby/object:Gem::Specification
+name: async-jobs
+version: !ruby/object:Gem::Version
+  version: 0.0.3
+platform: ruby
+authors:
+- Jacob
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-03-30 00:00:00.000000000 Z
+dependencies: []
+description: 
+email: jacob@engineyard.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- Gemfile.lock
+- MIT-LICENSE
+- README.md
+- async.gemspec
+- lib/async.rb
+- lib/async/locked.rb
+- lib/async/qu.rb
+- lib/async/resque.rb
+- lib/async/sidekiq.rb
+- lib/async/version.rb
+- spec/async_and_lock_spec.rb
+- spec/basics_spec.rb
+- spec/qu_spec.rb
+- spec/resque_spec.rb
+- spec/shared/models.rb
+- spec/sidekiq_spec.rb
+homepage: https://github.com/engineyard/async
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: abstraction over background job systems
+test_files:
+- spec/async_and_lock_spec.rb
+- spec/basics_spec.rb
+- spec/qu_spec.rb
+- spec/resque_spec.rb
+- spec/shared/models.rb
+- spec/sidekiq_spec.rb
+has_rdoc: