delayed_job_memento 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 288931fe637b0eba836c6403575b0440c8a2a16b
+  data.tar.gz: ec5b88eb644bf4e01bf22ba92c6ee82d8ee22c6e
+SHA512:
+  metadata.gz: 3607f7798b3a30b9a128ed0f51f3b7390cf5c061df4303d3b7a3f52b9502ed15dc5c3c1ca6edc5758f145272d280b713dbd74ebb0187337452960e6aa24c2745
+  data.tar.gz: fcbf4e84387eb8e440f7befbe81161ad03469b23a134b5ca075cbf58d2c0291fd595f82a7a3929f4579eb2432a5cdc1f71cf1d356eb9a02ca7edb82737147a1c

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+.idea/

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in delayed_job_memento.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Joseph Quadrino
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,72 @@
+# Delayed Job Memento [![Gem Version](https://badge.fury.io/rb/delayed_job_memento.png)](http://badge.fury.io/rb/delayed_job_memento)
+
+Delayed Jobs can be sentimental, save them as mementos so you don't have to forget.
+
+Also, set quotas for delayed jobs. 
+
+## Installation
+
+Add this line to your Gemfile:
+
+    gem 'delayed_job_memento'
+    
+and run `bundle`
+
+or without bundler:
+
+    $ gem install delayed_job_memento
+    
+####Then    
+
+Create migration for mementos:
+
+    $ rails g delayed_job_memento:install
+    $ rake db:migrate
+    
+## Usage
+
+####You probably shouldn't hold onto mementos forever:
+
+    # config/initializers
+    
+    Delayed::Memento.expiration = 1.week
+    
+    ## keep quotas in mind, see below
+    
+####To set Quotas:
+    
+    # config/initializers
+    
+    Delayed::Quota.new(queue: 'cat emails', interval: 1.day, size: 100) # Don't send more than 100 cat emails per day
+    ...
+    
+    ## Don't let any of your quota intervals exceed memento expiration time or that quota wont work
+When a Quota is reached its queue is rebalanced so that remaining queue jobs run at the end of the quotas current lifetime.
+
+
+####Get mementos
+
+    Delayed::Memento.all
+
+### Note
+
+Only works with ActiveRecord as your delayed job backend.
+    
+### TODO
+    
+1. Tests
+2. More Backends
+3. More things to do with Mementos
+
+## Contributing
+
+1. Fork it ( http://github.com/<my-github-username>/delayed_job_memento/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+
+
+<a href='https://github.com/jquadrin/delayed_job_memento/blob/master/LICENSE.txt'>LICENSE</a> 
+

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/delayed_job_memento.gemspec

@@ -0,0 +1,25 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'delayed_job_memento/version'
+
+Gem::Specification.new do |spec|
+  spec.add_dependency   'activerecord', ['>= 3.0', '< 4.1']
+  spec.add_dependency   'delayed_job',  ['>= 3.0', '< 4.1']
+  spec.name          = "delayed_job_memento"
+  spec.version       = DelayedJobMemento::VERSION
+  spec.authors       = ["Joseph Quadrino"]
+  spec.email         = ["jquadrin@gmail.com"]
+  spec.summary       = %q{Delayed jobs can be sentimental, you don't have to let them go.}
+  spec.description   = %q{Save mementos of your delayed jobs and also set queue quotas.}
+  spec.homepage      = "https://github.com/jquadrin/delayed_job_memento"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.5"
+  spec.add_development_dependency "rake"
+end

data/lib/delayed_job_memento.rb

@@ -0,0 +1,5 @@
+require 'active_record'
+require 'delayed_job_memento/version'
+require 'delayed_job_memento/delayed/worker'
+require 'delayed_job_memento/delayed/memento'
+require 'delayed_job_memento/delayed/quota'

data/lib/delayed_job_memento/delayed/memento.rb

@@ -0,0 +1,26 @@
+require 'active_record/version'
+
+module Delayed
+  class Memento < ActiveRecord::Base
+
+    self.table_name = :delayed_jobs_history
+
+    if ActiveRecord::VERSION::MAJOR < 4 || defined?(ActiveRecord::MassAssignmentSecurity)
+      attr_accessible :run_at, :priority, :handler, :queue, :locked_at,
+                      :locked_by, :attempts, :failed_at, :last_error
+    end
+
+    cattr_accessor :expiration, :last_clean_up
+
+    after_save :run_clean_up
+
+    def run_clean_up
+      current_time = Time.zone.now
+      if @@expiration != nil && (@@last_clean_up.nil? || current_time - @@last_clean_up > @@expiration)
+        Delayed::Memento.delete_all(['created_at < ?', current_time - @@expiration])
+        @@last_clean_up = current_time
+      end
+    end
+
+  end
+end

data/lib/delayed_job_memento/delayed/quota.rb

@@ -0,0 +1,59 @@
+module Delayed
+  class Quota
+    attr_accessor :queue, :interval, :size, :started, :reached
+    cattr_accessor :instances
+
+    self.instances = {}
+
+    def initialize(options)
+      self.queue = options[:queue]
+      self.interval = options[:interval]
+      self.size = options[:size]
+      self.started, self.reached = nil, nil
+      self.instances[self.queue] = self
+    end
+
+    # fix class variables to instances
+    def quota_reached?
+      current_time = Time.zone.now
+      if self.reached != nil
+        time_left = self.interval - (self.reached - self.started)
+        if self.reached + time_left < current_time
+          self.reached = nil
+          self.started = current_time
+          false
+        else
+          true
+        end
+      else
+        if self.started != nil
+          conditions = {
+              queue: self.queue,
+              created_at: self.started..current_time,
+              locked_at: Time.new(0)..current_time
+          }
+          finished_jobs = Delayed::Memento.where(queue: self.queue,created_at: self.started..current_time,locked_at: Time.new(0)..current_time).count
+          if finished_jobs >= self.size
+            self.reached = current_time
+            true
+          else
+            false
+          end
+        else
+          self.started = current_time
+          false
+        end
+      end
+    end
+
+    def rebalance_queue
+      run_at = self.interval - (self.reached - self.started)
+      conditions = {queue: self.queue, locked_at: nil}
+      Delayed::Job.where(conditions).find_in_batches(batch_size: self.size) do |batch|
+        batch.each { |job| job.run_at = run_at }
+        Delayed::Job.import batch, validate: false, on_duplicate_key_update: [:run_at]
+        run_at= run_at + self.interval
+      end
+    end
+  end
+end

data/lib/delayed_job_memento/delayed/worker.rb

@@ -0,0 +1,65 @@
+require 'delayed_job'
+module Delayed
+  class Worker
+    extend Delayed
+
+    DEFAULT_LOG_LEVEL = Logger::INFO
+
+    def self.backend=(backend)
+      if backend.is_a? Symbol
+        require "delayed/serialization/#{backend}"
+        require "delayed/backend/#{backend}"
+        backend = "Delayed::Backend::#{backend.to_s.classify}::Job".constantize
+      end
+      @@backend = backend
+      silence_warnings { ::Delayed.const_set(:Job, backend) }
+    end
+
+    def save_as_memento(job)
+      params = job.attributes
+      attrs = ['run_at', 'priority', 'handler', 'queue', 'locked_at', 'locked_by', 'attempts', 'failed_at', 'last_error']
+      params.delete_if { |k| !attrs.include? k }
+      Delayed::Memento.create(params)
+    end
+
+    def run(job)
+      instances = Delayed::Quota.instances
+      if job.queue != nil && instances.has_key?(job.queue) && instances[job.queue].quota_reached?
+        instances[job.queue].rebalance_queue
+        return false
+      else
+        job_say job, 'RUNNING'
+        runtime = Benchmark.realtime do
+          Timeout.timeout(self.class.max_run_time.to_i, WorkerTimeout) { job.invoke_job }
+          save_as_memento(job)
+          job.destroy
+        end
+        job_say job, 'COMPLETED after %.4f' % runtime
+        return true # did work
+      end
+    rescue DeserializationError => error
+      job.last_error = "#{error.message}\n#{error.backtrace.join("\n")}"
+      failed(job)
+    rescue Exception => error
+      self.class.lifecycle.run_callbacks(:error, self, job) { handle_failed_job(job, error) }
+      return false # work failed
+    end
+
+    def failed(job)
+      self.class.lifecycle.run_callbacks(:failure, self, job) do
+        job.hook(:failure)
+        if self.class.destroy_failed_jobs
+          save_as_memento(job)
+          job.destroy
+        else
+          job.fail!
+        end
+      end
+    end
+
+    def job_say(job, text, level = DEFAULT_LOG_LEVEL)
+      text = "Job #{job.name} (id=#{job.id}) #{text}"
+      say text, level
+    end
+  end
+end

data/lib/delayed_job_memento/version.rb

@@ -0,0 +1,3 @@
+module DelayedJobMemento
+  VERSION = '0.1.0'
+end

data/lib/generators/delayed_job_memento/install_generator.rb

@@ -0,0 +1,23 @@
+require 'generators/delayed_job_memento/next_migration_version'
+require 'rails/generators/migration'
+require 'rails/generators/active_record'
+
+module DelayedJobMemento
+  module Generators
+    class InstallGenerator < Rails::Generators::Base
+      include Rails::Generators::Migration
+      extend DelayedJobMemento::NextMigrationVersion
+
+      desc "Adds delayed job history migration"
+      self.source_paths << File.join(File.dirname(__FILE__), 'templates')
+
+      def create_migration_file
+        migration_template 'migration.rb', 'db/migrate/create_delayed_jobs_history.rb'
+      end
+
+      def self.next_migration_number dirname
+        ActiveRecord::Generators::Base.next_migration_number dirname
+      end
+    end
+  end
+end

data/lib/generators/delayed_job_memento/next_migration_version.rb

@@ -0,0 +1,12 @@
+module DelayedJobMemento
+  module NextMigrationVersion
+    def next_migration_number(dirname)
+      next_migration_number = current_migration_number(dirname) + 1
+      if ActiveRecord::Base.timestamped_migrations
+        [Time.now.utc.strftime("%Y%m%d%H%M%S"), "%.14d" % next_migration_number].max
+      else
+        "%.3d" % next_migration_number
+      end
+    end
+  end
+end

data/lib/generators/delayed_job_memento/templates/migration.rb

@@ -0,0 +1,23 @@
+class CreateDelayedJobsHistory < ActiveRecord::Migration
+  def self.up
+    create_table :delayed_jobs_history, force: true do |t|
+      t.datetime :created_at                                  # when job was completed
+      t.text     :handler, null: false
+      t.string   :queue
+      t.datetime :locked_at
+      t.string   :locked_by
+      t.datetime :failed_at
+      t.integer  :priority, default: 0, null: false
+      t.datetime :run_at
+      t.integer  :attempts, default: 0, null: false
+      t.text     :last_error
+      t.datetime :updated_at
+    end
+
+    add_index :delayed_jobs_history, :created_at, name: 'delayed_jobs_completed'
+  end
+
+  def self.down
+    drop_table :delayed_jobs_history
+  end
+end

data/spec/memento_spec.rb

@@ -0,0 +1,9 @@
+describe Delayed::Memento do
+  # TODO a ton of tests
+  before :all do
+    Delayed::Memento.expiration = 2.seconds
+  end
+
+  Delayed::Memento.create(queue: 'whatever')
+
+end

data/spec/quota_spec.rb

@@ -0,0 +1,3 @@
+describe Delayed::Quota do
+# TODO a ton of tests
+end

metadata

@@ -0,0 +1,131 @@
+--- !ruby/object:Gem::Specification
+name: delayed_job_memento
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Joseph Quadrino
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-02-18 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '3.0'
+    - - <
+      - !ruby/object:Gem::Version
+        version: '4.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '3.0'
+    - - <
+      - !ruby/object:Gem::Version
+        version: '4.1'
+- !ruby/object:Gem::Dependency
+  name: delayed_job
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '3.0'
+    - - <
+      - !ruby/object:Gem::Version
+        version: '4.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '3.0'
+    - - <
+      - !ruby/object:Gem::Version
+        version: '4.1'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Save mementos of your delayed jobs and also set queue quotas.
+email:
+- jquadrin@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- delayed_job_memento.gemspec
+- lib/delayed_job_memento.rb
+- lib/delayed_job_memento/delayed/memento.rb
+- lib/delayed_job_memento/delayed/quota.rb
+- lib/delayed_job_memento/delayed/worker.rb
+- lib/delayed_job_memento/version.rb
+- lib/generators/delayed_job_memento/install_generator.rb
+- lib/generators/delayed_job_memento/next_migration_version.rb
+- lib/generators/delayed_job_memento/templates/migration.rb
+- spec/memento_spec.rb
+- spec/quota_spec.rb
+homepage: https://github.com/jquadrin/delayed_job_memento
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.1
+signing_key: 
+specification_version: 4
+summary: Delayed jobs can be sentimental, you don't have to let them go.
+test_files:
+- spec/memento_spec.rb
+- spec/quota_spec.rb
+has_rdoc: