chronofage 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 708bb7f6a4836dbe1089bc0a2a699fe8fad2740c
+  data.tar.gz: 2a717b066b6454e01a5c88ac38c8da5e0eda1f77
+SHA512:
+  metadata.gz: 170565c678b3c0d277b523657e198ebf1d89e4dd2612955c2550c60166d475088a85a4cd0f81645fbc20bfd870fdc9b4fd7b8f9ab59a12d415260ec7e49e900d
+  data.tar.gz: 8a39277b4cf3572751ef1a1e2e47b86e6cdcd1530b72319ced3bb604bca8c33d6727a663ed1f0e34e53b7a14549a2f2a8352b8baa0470fb2f4942c3289e970e8

data/.gitignore

@@ -0,0 +1 @@
+*.gem

data/Gemfile

@@ -0,0 +1,3 @@
+source "https://rubygems.org"
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,19 @@
+PATH
+  remote: .
+  specs:
+    chronofage (0.0.1)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    byebug (9.1.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  byebug
+  chronofage!
+
+BUNDLED WITH
+   1.15.1

data/README.md

@@ -0,0 +1,70 @@
+# Chronofage
+
+Chronofage is a Cron (or whatever scheduler you want) and ActiveRecord based ActiveJob backend. It is
+well suited for long-running tasks (heavy video processing, render, etc). Every job
+is run in its own process and you can spread the work on multiple hosts.
+
+## Installation
+
+Copy the migration file and run it to create the `chronofage_jobs` and `chronofage_runners` tables.
+
+```
+rake chronofage_engine:install:migrations db:migrate
+```
+
+The `chronofage_jobs` table hold informations about all the jobs ready to be executed, failed or done.
+
+```
+create_table "chronofage_jobs", force: :cascade do |t|
+  t.string   "job_class"
+  t.string   "job_id"
+  t.string   "queue_name"
+  t.text     "arguments"
+  t.integer  "priority"
+  t.datetime "started_at"      # set when a job is started
+  t.datetime "completed_at"    # set when a job is completed successfuly
+  t.datetime "failed_at"       # set when a job failed
+  t.datetime "created_at",   null: false
+  t.datetime "updated_at",   null: false
+end
+```
+
+The `chronofage_runners` table hold informations about the process currently running.
+
+```
+create_table "chronofage_runners", force: :cascade do |t|
+  t.string   "queue_name"
+  t.string   "host"
+  t.datetime "created_at", null: false
+  t.datetime "updated_at", null: false
+end
+```
+
+## Usage
+
+You can selectively override the ActiveJob backend for your very long-runny task.
+
+```
+class BuildCompilationJob < ApplicationJob
+  self.queue_adapter = :chronofage
+  queue_as :heavy
+
+  def perform(compilation_settings)
+    # long-running stuff
+  end
+end
+```
+
+Then jobs can be run with a simple Rake task taking a queue name and a concurrency argument.
+
+```
+rake chronofage_engine:jobs:execute[heavy,2]
+```
+
+The task can be run from Cron or any task scheduler you like (the Windows task scheduler, the Heroku scheduler plugin, etc).
+The concurrency argument is host-based, so a Cron config like the following one, spread on 3 hosts, will execute a maximum
+of 6 jobs, 2 for each host, and check for new jobs every 5 minutes.
+
+```
+*/5 * * * * cd /var/www/my_app && rake chronofage_engine:jobs:execute[heavy,2]
+```

data/Rakefile

@@ -0,0 +1,6 @@
+require 'rubygems'
+require 'bundler/setup'
+
+import "./lib/tasks/chronofage.rake"
+
+Bundler::GemHelper.install_tasks

data/chronofage.gemspec

@@ -0,0 +1,22 @@
+# -*- encoding: utf-8 -*-
+
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "chronofage/version"
+
+Gem::Specification.new do |gem|
+  gem.name          = "chronofage"
+  gem.version       = Chronofage::VERSION
+  gem.authors       = ["Victor Goya", "Amuse Animation"]
+  gem.email         = ["v.goya@millimages.com"]
+  gem.description   = "Cron based job scheduler"
+  gem.summary       = "Cron based job scheduler"
+  gem.homepage      = "https://www.amusenetwork.com/"
+
+  gem.files         = `git ls-files -z`.split("\x0")
+  gem.require_paths = ["lib"]
+
+  gem.licenses      = ["MIT"]
+
+  gem.required_ruby_version = "~> 2.0"
+end

data/db/migrate/1_create_job_table.rb

@@ -0,0 +1,16 @@
+class CreateJobTable < ActiveRecord::Migration[5.0]
+  def change
+    create_table :chronofage_jobs do |t|
+      t.string   :job_class
+      t.string   :job_id
+      t.string   :queue_name
+      t.text     :arguments
+      t.integer  :priority
+
+      t.datetime :started_at
+      t.datetime :completed_at
+      t.datetime :failed_at
+      t.timestamps
+    end
+  end
+end

data/db/migrate/2_create_runner_table.rb

@@ -0,0 +1,10 @@
+class CreateRunnerTable < ActiveRecord::Migration[5.0]
+  def change
+    create_table :chronofage_runners do |t|
+      t.string   :queue_name
+      t.string   :host
+
+      t.timestamps
+    end
+  end
+end

data/lib/active_job/queue_adapters/chronofage_adapter.rb

@@ -0,0 +1,19 @@
+module ActiveJob
+  module QueueAdapters
+    class ChronofageAdapter
+      def enqueue(job)
+        Chronofage::Job.create!({
+          job_class: job.class,
+          arguments: ActiveJob::Arguments.serialize(job.arguments).to_json,
+          job_id: job.job_id,
+          queue_name: job.queue_name,
+          priority: job.priority
+        })
+      end
+
+      def enqueue_at(job)
+        raise NotImplementedError, "Chronofage doesn't support enqueue_at."
+      end
+    end
+  end
+end

data/lib/chronofage.rb

@@ -0,0 +1,16 @@
+require 'active_record'
+require 'active_record/version'
+require 'active_support/core_ext/module'
+require 'active_job/queue_adapters/chronofage_adapter'
+require 'chronofage/job'
+require 'chronofage/runner'
+
+begin
+  require 'rails/engine'
+  require 'chronofage/engine'
+  rescue LoadError
+end
+
+module Chronofage
+  extend ActiveSupport::Autoload
+end

data/lib/chronofage/engine.rb

@@ -0,0 +1,4 @@
+module Chronofage
+  class Engine < Rails::Engine
+  end
+end

data/lib/chronofage/job.rb

@@ -0,0 +1,33 @@
+module Chronofage
+  class Job < ::ActiveRecord::Base
+    self.table_name = "chronofage_jobs"
+
+    scope :ready, -> { where(started_at: nil) }
+
+    def execute!
+      start!
+      job_class.constantize.perform_now(*deserialized_arguments)
+      done!
+    rescue
+      failed!
+      raise
+    end
+
+    def start!
+      update(started_at: Time.now)
+    end
+
+    def done!
+      update(completed_at: Time.now)
+    end
+
+    def failed!
+      update(failed_at: Time.now)
+    end
+
+    def deserialized_arguments
+      ActiveJob::Arguments.deserialize(JSON.parse(arguments))
+    end
+
+  end
+end

data/lib/chronofage/runner.rb

@@ -0,0 +1,33 @@
+require 'socket'
+
+module Chronofage
+  class Runner < ::ActiveRecord::Base
+    self.table_name = "chronofage_runners"
+
+    class MaxConcurrencyReached < StandardError
+    end
+
+    def self.register!(queue_name, concurrency)
+      if concurrent_runnners(queue_name).count >= concurrency
+        raise MaxConcurrencyReached
+      else
+        create!(queue_name: queue_name, host: host)
+      end
+    end
+
+    def self.concurrent_runnners(queue_name)
+      where(queue_name: queue_name, host: host)
+    end
+
+    def unregister!
+      destroy!
+    end
+
+    private
+
+    def self.host
+      Socket.gethostname
+    end
+
+  end
+end

data/lib/chronofage/version.rb

@@ -0,0 +1,3 @@
+module Chronofage
+  VERSION = "0.0.1"
+end

data/lib/tasks/chronofage.rake

@@ -0,0 +1,18 @@
+namespace :chronofage_engine do
+  namespace :jobs do
+    task :execute, [:queue_name, :concurrency] => :environment do |t, args|
+      job = Chronofage::Job.ready.where(queue_name: args.queue_name).order(priority: :asc).first
+
+      if job.nil?
+        Rails.logger.info "chronofage[#{args.queue_name}]: no job to execute."
+      else
+        runner = Chronofage::Runner.register!(args.queue_name, args.concurrency.to_i)
+        begin
+          job.execute!
+        ensure
+          runner.unregister!
+        end
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,60 @@
+--- !ruby/object:Gem::Specification
+name: chronofage
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Victor Goya
+- Amuse Animation
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-11-27 00:00:00.000000000 Z
+dependencies: []
+description: Cron based job scheduler
+email:
+- v.goya@millimages.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- Gemfile.lock
+- README.md
+- Rakefile
+- chronofage.gemspec
+- db/migrate/1_create_job_table.rb
+- db/migrate/2_create_runner_table.rb
+- lib/active_job/queue_adapters/chronofage_adapter.rb
+- lib/chronofage.rb
+- lib/chronofage/engine.rb
+- lib/chronofage/job.rb
+- lib/chronofage/runner.rb
+- lib/chronofage/version.rb
+- lib/tasks/chronofage.rake
+homepage: https://www.amusenetwork.com/
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - "~>"
+    - !ruby/object:Gem::Version
+      version: '2.0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.2
+signing_key: 
+specification_version: 4
+summary: Cron based job scheduler
+test_files: []