chronofage 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +1 -0
- data/Gemfile +3 -0
- data/Gemfile.lock +19 -0
- data/README.md +70 -0
- data/Rakefile +6 -0
- data/chronofage.gemspec +22 -0
- data/db/migrate/1_create_job_table.rb +16 -0
- data/db/migrate/2_create_runner_table.rb +10 -0
- data/lib/active_job/queue_adapters/chronofage_adapter.rb +19 -0
- data/lib/chronofage.rb +16 -0
- data/lib/chronofage/engine.rb +4 -0
- data/lib/chronofage/job.rb +33 -0
- data/lib/chronofage/runner.rb +33 -0
- data/lib/chronofage/version.rb +3 -0
- data/lib/tasks/chronofage.rake +18 -0
- metadata +60 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 708bb7f6a4836dbe1089bc0a2a699fe8fad2740c
|
4
|
+
data.tar.gz: 2a717b066b6454e01a5c88ac38c8da5e0eda1f77
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 170565c678b3c0d277b523657e198ebf1d89e4dd2612955c2550c60166d475088a85a4cd0f81645fbc20bfd870fdc9b4fd7b8f9ab59a12d415260ec7e49e900d
|
7
|
+
data.tar.gz: 8a39277b4cf3572751ef1a1e2e47b86e6cdcd1530b72319ced3bb604bca8c33d6727a663ed1f0e34e53b7a14549a2f2a8352b8baa0470fb2f4942c3289e970e8
|
data/.gitignore
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
*.gem
|
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
data/README.md
ADDED
@@ -0,0 +1,70 @@
|
|
1
|
+
# Chronofage
|
2
|
+
|
3
|
+
Chronofage is a Cron (or whatever scheduler you want) and ActiveRecord based ActiveJob backend. It is
|
4
|
+
well suited for long-running tasks (heavy video processing, render, etc). Every job
|
5
|
+
is run in its own process and you can spread the work on multiple hosts.
|
6
|
+
|
7
|
+
## Installation
|
8
|
+
|
9
|
+
Copy the migration file and run it to create the `chronofage_jobs` and `chronofage_runners` tables.
|
10
|
+
|
11
|
+
```
|
12
|
+
rake chronofage_engine:install:migrations db:migrate
|
13
|
+
```
|
14
|
+
|
15
|
+
The `chronofage_jobs` table hold informations about all the jobs ready to be executed, failed or done.
|
16
|
+
|
17
|
+
```
|
18
|
+
create_table "chronofage_jobs", force: :cascade do |t|
|
19
|
+
t.string "job_class"
|
20
|
+
t.string "job_id"
|
21
|
+
t.string "queue_name"
|
22
|
+
t.text "arguments"
|
23
|
+
t.integer "priority"
|
24
|
+
t.datetime "started_at" # set when a job is started
|
25
|
+
t.datetime "completed_at" # set when a job is completed successfuly
|
26
|
+
t.datetime "failed_at" # set when a job failed
|
27
|
+
t.datetime "created_at", null: false
|
28
|
+
t.datetime "updated_at", null: false
|
29
|
+
end
|
30
|
+
```
|
31
|
+
|
32
|
+
The `chronofage_runners` table hold informations about the process currently running.
|
33
|
+
|
34
|
+
```
|
35
|
+
create_table "chronofage_runners", force: :cascade do |t|
|
36
|
+
t.string "queue_name"
|
37
|
+
t.string "host"
|
38
|
+
t.datetime "created_at", null: false
|
39
|
+
t.datetime "updated_at", null: false
|
40
|
+
end
|
41
|
+
```
|
42
|
+
|
43
|
+
## Usage
|
44
|
+
|
45
|
+
You can selectively override the ActiveJob backend for your very long-runny task.
|
46
|
+
|
47
|
+
```
|
48
|
+
class BuildCompilationJob < ApplicationJob
|
49
|
+
self.queue_adapter = :chronofage
|
50
|
+
queue_as :heavy
|
51
|
+
|
52
|
+
def perform(compilation_settings)
|
53
|
+
# long-running stuff
|
54
|
+
end
|
55
|
+
end
|
56
|
+
```
|
57
|
+
|
58
|
+
Then jobs can be run with a simple Rake task taking a queue name and a concurrency argument.
|
59
|
+
|
60
|
+
```
|
61
|
+
rake chronofage_engine:jobs:execute[heavy,2]
|
62
|
+
```
|
63
|
+
|
64
|
+
The task can be run from Cron or any task scheduler you like (the Windows task scheduler, the Heroku scheduler plugin, etc).
|
65
|
+
The concurrency argument is host-based, so a Cron config like the following one, spread on 3 hosts, will execute a maximum
|
66
|
+
of 6 jobs, 2 for each host, and check for new jobs every 5 minutes.
|
67
|
+
|
68
|
+
```
|
69
|
+
*/5 * * * * cd /var/www/my_app && rake chronofage_engine:jobs:execute[heavy,2]
|
70
|
+
```
|
data/Rakefile
ADDED
data/chronofage.gemspec
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
|
3
|
+
lib = File.expand_path("../lib", __FILE__)
|
4
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
5
|
+
require "chronofage/version"
|
6
|
+
|
7
|
+
Gem::Specification.new do |gem|
|
8
|
+
gem.name = "chronofage"
|
9
|
+
gem.version = Chronofage::VERSION
|
10
|
+
gem.authors = ["Victor Goya", "Amuse Animation"]
|
11
|
+
gem.email = ["v.goya@millimages.com"]
|
12
|
+
gem.description = "Cron based job scheduler"
|
13
|
+
gem.summary = "Cron based job scheduler"
|
14
|
+
gem.homepage = "https://www.amusenetwork.com/"
|
15
|
+
|
16
|
+
gem.files = `git ls-files -z`.split("\x0")
|
17
|
+
gem.require_paths = ["lib"]
|
18
|
+
|
19
|
+
gem.licenses = ["MIT"]
|
20
|
+
|
21
|
+
gem.required_ruby_version = "~> 2.0"
|
22
|
+
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
class CreateJobTable < ActiveRecord::Migration[5.0]
|
2
|
+
def change
|
3
|
+
create_table :chronofage_jobs do |t|
|
4
|
+
t.string :job_class
|
5
|
+
t.string :job_id
|
6
|
+
t.string :queue_name
|
7
|
+
t.text :arguments
|
8
|
+
t.integer :priority
|
9
|
+
|
10
|
+
t.datetime :started_at
|
11
|
+
t.datetime :completed_at
|
12
|
+
t.datetime :failed_at
|
13
|
+
t.timestamps
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
module ActiveJob
|
2
|
+
module QueueAdapters
|
3
|
+
class ChronofageAdapter
|
4
|
+
def enqueue(job)
|
5
|
+
Chronofage::Job.create!({
|
6
|
+
job_class: job.class,
|
7
|
+
arguments: ActiveJob::Arguments.serialize(job.arguments).to_json,
|
8
|
+
job_id: job.job_id,
|
9
|
+
queue_name: job.queue_name,
|
10
|
+
priority: job.priority
|
11
|
+
})
|
12
|
+
end
|
13
|
+
|
14
|
+
def enqueue_at(job)
|
15
|
+
raise NotImplementedError, "Chronofage doesn't support enqueue_at."
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
data/lib/chronofage.rb
ADDED
@@ -0,0 +1,16 @@
|
|
1
|
+
require 'active_record'
|
2
|
+
require 'active_record/version'
|
3
|
+
require 'active_support/core_ext/module'
|
4
|
+
require 'active_job/queue_adapters/chronofage_adapter'
|
5
|
+
require 'chronofage/job'
|
6
|
+
require 'chronofage/runner'
|
7
|
+
|
8
|
+
begin
|
9
|
+
require 'rails/engine'
|
10
|
+
require 'chronofage/engine'
|
11
|
+
rescue LoadError
|
12
|
+
end
|
13
|
+
|
14
|
+
module Chronofage
|
15
|
+
extend ActiveSupport::Autoload
|
16
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
module Chronofage
|
2
|
+
class Job < ::ActiveRecord::Base
|
3
|
+
self.table_name = "chronofage_jobs"
|
4
|
+
|
5
|
+
scope :ready, -> { where(started_at: nil) }
|
6
|
+
|
7
|
+
def execute!
|
8
|
+
start!
|
9
|
+
job_class.constantize.perform_now(*deserialized_arguments)
|
10
|
+
done!
|
11
|
+
rescue
|
12
|
+
failed!
|
13
|
+
raise
|
14
|
+
end
|
15
|
+
|
16
|
+
def start!
|
17
|
+
update(started_at: Time.now)
|
18
|
+
end
|
19
|
+
|
20
|
+
def done!
|
21
|
+
update(completed_at: Time.now)
|
22
|
+
end
|
23
|
+
|
24
|
+
def failed!
|
25
|
+
update(failed_at: Time.now)
|
26
|
+
end
|
27
|
+
|
28
|
+
def deserialized_arguments
|
29
|
+
ActiveJob::Arguments.deserialize(JSON.parse(arguments))
|
30
|
+
end
|
31
|
+
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
require 'socket'
|
2
|
+
|
3
|
+
module Chronofage
|
4
|
+
class Runner < ::ActiveRecord::Base
|
5
|
+
self.table_name = "chronofage_runners"
|
6
|
+
|
7
|
+
class MaxConcurrencyReached < StandardError
|
8
|
+
end
|
9
|
+
|
10
|
+
def self.register!(queue_name, concurrency)
|
11
|
+
if concurrent_runnners(queue_name).count >= concurrency
|
12
|
+
raise MaxConcurrencyReached
|
13
|
+
else
|
14
|
+
create!(queue_name: queue_name, host: host)
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def self.concurrent_runnners(queue_name)
|
19
|
+
where(queue_name: queue_name, host: host)
|
20
|
+
end
|
21
|
+
|
22
|
+
def unregister!
|
23
|
+
destroy!
|
24
|
+
end
|
25
|
+
|
26
|
+
private
|
27
|
+
|
28
|
+
def self.host
|
29
|
+
Socket.gethostname
|
30
|
+
end
|
31
|
+
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
namespace :chronofage_engine do
|
2
|
+
namespace :jobs do
|
3
|
+
task :execute, [:queue_name, :concurrency] => :environment do |t, args|
|
4
|
+
job = Chronofage::Job.ready.where(queue_name: args.queue_name).order(priority: :asc).first
|
5
|
+
|
6
|
+
if job.nil?
|
7
|
+
Rails.logger.info "chronofage[#{args.queue_name}]: no job to execute."
|
8
|
+
else
|
9
|
+
runner = Chronofage::Runner.register!(args.queue_name, args.concurrency.to_i)
|
10
|
+
begin
|
11
|
+
job.execute!
|
12
|
+
ensure
|
13
|
+
runner.unregister!
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
metadata
ADDED
@@ -0,0 +1,60 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: chronofage
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Victor Goya
|
8
|
+
- Amuse Animation
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2017-11-27 00:00:00.000000000 Z
|
13
|
+
dependencies: []
|
14
|
+
description: Cron based job scheduler
|
15
|
+
email:
|
16
|
+
- v.goya@millimages.com
|
17
|
+
executables: []
|
18
|
+
extensions: []
|
19
|
+
extra_rdoc_files: []
|
20
|
+
files:
|
21
|
+
- ".gitignore"
|
22
|
+
- Gemfile
|
23
|
+
- Gemfile.lock
|
24
|
+
- README.md
|
25
|
+
- Rakefile
|
26
|
+
- chronofage.gemspec
|
27
|
+
- db/migrate/1_create_job_table.rb
|
28
|
+
- db/migrate/2_create_runner_table.rb
|
29
|
+
- lib/active_job/queue_adapters/chronofage_adapter.rb
|
30
|
+
- lib/chronofage.rb
|
31
|
+
- lib/chronofage/engine.rb
|
32
|
+
- lib/chronofage/job.rb
|
33
|
+
- lib/chronofage/runner.rb
|
34
|
+
- lib/chronofage/version.rb
|
35
|
+
- lib/tasks/chronofage.rake
|
36
|
+
homepage: https://www.amusenetwork.com/
|
37
|
+
licenses:
|
38
|
+
- MIT
|
39
|
+
metadata: {}
|
40
|
+
post_install_message:
|
41
|
+
rdoc_options: []
|
42
|
+
require_paths:
|
43
|
+
- lib
|
44
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
45
|
+
requirements:
|
46
|
+
- - "~>"
|
47
|
+
- !ruby/object:Gem::Version
|
48
|
+
version: '2.0'
|
49
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
50
|
+
requirements:
|
51
|
+
- - ">="
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: '0'
|
54
|
+
requirements: []
|
55
|
+
rubyforge_project:
|
56
|
+
rubygems_version: 2.5.2
|
57
|
+
signing_key:
|
58
|
+
specification_version: 4
|
59
|
+
summary: Cron based job scheduler
|
60
|
+
test_files: []
|