chronofage 0.0.1

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 708bb7f6a4836dbe1089bc0a2a699fe8fad2740c
4
+ data.tar.gz: 2a717b066b6454e01a5c88ac38c8da5e0eda1f77
5
+ SHA512:
6
+ metadata.gz: 170565c678b3c0d277b523657e198ebf1d89e4dd2612955c2550c60166d475088a85a4cd0f81645fbc20bfd870fdc9b4fd7b8f9ab59a12d415260ec7e49e900d
7
+ data.tar.gz: 8a39277b4cf3572751ef1a1e2e47b86e6cdcd1530b72319ced3bb604bca8c33d6727a663ed1f0e34e53b7a14549a2f2a8352b8baa0470fb2f4942c3289e970e8
data/.gitignore ADDED
@@ -0,0 +1 @@
1
+ *.gem
data/Gemfile ADDED
@@ -0,0 +1,3 @@
1
+ source "https://rubygems.org"
2
+
3
+ gemspec
data/Gemfile.lock ADDED
@@ -0,0 +1,19 @@
1
+ PATH
2
+ remote: .
3
+ specs:
4
+ chronofage (0.0.1)
5
+
6
+ GEM
7
+ remote: https://rubygems.org/
8
+ specs:
9
+ byebug (9.1.0)
10
+
11
+ PLATFORMS
12
+ ruby
13
+
14
+ DEPENDENCIES
15
+ byebug
16
+ chronofage!
17
+
18
+ BUNDLED WITH
19
+ 1.15.1
data/README.md ADDED
@@ -0,0 +1,70 @@
1
+ # Chronofage
2
+
3
+ Chronofage is a Cron (or whatever scheduler you want) and ActiveRecord based ActiveJob backend. It is
4
+ well suited for long-running tasks (heavy video processing, render, etc). Every job
5
+ is run in its own process and you can spread the work on multiple hosts.
6
+
7
+ ## Installation
8
+
9
+ Copy the migration file and run it to create the `chronofage_jobs` and `chronofage_runners` tables.
10
+
11
+ ```
12
+ rake chronofage_engine:install:migrations db:migrate
13
+ ```
14
+
15
+ The `chronofage_jobs` table hold informations about all the jobs ready to be executed, failed or done.
16
+
17
+ ```
18
+ create_table "chronofage_jobs", force: :cascade do |t|
19
+ t.string "job_class"
20
+ t.string "job_id"
21
+ t.string "queue_name"
22
+ t.text "arguments"
23
+ t.integer "priority"
24
+ t.datetime "started_at" # set when a job is started
25
+ t.datetime "completed_at" # set when a job is completed successfuly
26
+ t.datetime "failed_at" # set when a job failed
27
+ t.datetime "created_at", null: false
28
+ t.datetime "updated_at", null: false
29
+ end
30
+ ```
31
+
32
+ The `chronofage_runners` table hold informations about the process currently running.
33
+
34
+ ```
35
+ create_table "chronofage_runners", force: :cascade do |t|
36
+ t.string "queue_name"
37
+ t.string "host"
38
+ t.datetime "created_at", null: false
39
+ t.datetime "updated_at", null: false
40
+ end
41
+ ```
42
+
43
+ ## Usage
44
+
45
+ You can selectively override the ActiveJob backend for your very long-runny task.
46
+
47
+ ```
48
+ class BuildCompilationJob < ApplicationJob
49
+ self.queue_adapter = :chronofage
50
+ queue_as :heavy
51
+
52
+ def perform(compilation_settings)
53
+ # long-running stuff
54
+ end
55
+ end
56
+ ```
57
+
58
+ Then jobs can be run with a simple Rake task taking a queue name and a concurrency argument.
59
+
60
+ ```
61
+ rake chronofage_engine:jobs:execute[heavy,2]
62
+ ```
63
+
64
+ The task can be run from Cron or any task scheduler you like (the Windows task scheduler, the Heroku scheduler plugin, etc).
65
+ The concurrency argument is host-based, so a Cron config like the following one, spread on 3 hosts, will execute a maximum
66
+ of 6 jobs, 2 for each host, and check for new jobs every 5 minutes.
67
+
68
+ ```
69
+ */5 * * * * cd /var/www/my_app && rake chronofage_engine:jobs:execute[heavy,2]
70
+ ```
data/Rakefile ADDED
@@ -0,0 +1,6 @@
1
+ require 'rubygems'
2
+ require 'bundler/setup'
3
+
4
+ import "./lib/tasks/chronofage.rake"
5
+
6
+ Bundler::GemHelper.install_tasks
@@ -0,0 +1,22 @@
1
+ # -*- encoding: utf-8 -*-
2
+
3
+ lib = File.expand_path("../lib", __FILE__)
4
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
5
+ require "chronofage/version"
6
+
7
+ Gem::Specification.new do |gem|
8
+ gem.name = "chronofage"
9
+ gem.version = Chronofage::VERSION
10
+ gem.authors = ["Victor Goya", "Amuse Animation"]
11
+ gem.email = ["v.goya@millimages.com"]
12
+ gem.description = "Cron based job scheduler"
13
+ gem.summary = "Cron based job scheduler"
14
+ gem.homepage = "https://www.amusenetwork.com/"
15
+
16
+ gem.files = `git ls-files -z`.split("\x0")
17
+ gem.require_paths = ["lib"]
18
+
19
+ gem.licenses = ["MIT"]
20
+
21
+ gem.required_ruby_version = "~> 2.0"
22
+ end
@@ -0,0 +1,16 @@
1
+ class CreateJobTable < ActiveRecord::Migration[5.0]
2
+ def change
3
+ create_table :chronofage_jobs do |t|
4
+ t.string :job_class
5
+ t.string :job_id
6
+ t.string :queue_name
7
+ t.text :arguments
8
+ t.integer :priority
9
+
10
+ t.datetime :started_at
11
+ t.datetime :completed_at
12
+ t.datetime :failed_at
13
+ t.timestamps
14
+ end
15
+ end
16
+ end
@@ -0,0 +1,10 @@
1
+ class CreateRunnerTable < ActiveRecord::Migration[5.0]
2
+ def change
3
+ create_table :chronofage_runners do |t|
4
+ t.string :queue_name
5
+ t.string :host
6
+
7
+ t.timestamps
8
+ end
9
+ end
10
+ end
@@ -0,0 +1,19 @@
1
+ module ActiveJob
2
+ module QueueAdapters
3
+ class ChronofageAdapter
4
+ def enqueue(job)
5
+ Chronofage::Job.create!({
6
+ job_class: job.class,
7
+ arguments: ActiveJob::Arguments.serialize(job.arguments).to_json,
8
+ job_id: job.job_id,
9
+ queue_name: job.queue_name,
10
+ priority: job.priority
11
+ })
12
+ end
13
+
14
+ def enqueue_at(job)
15
+ raise NotImplementedError, "Chronofage doesn't support enqueue_at."
16
+ end
17
+ end
18
+ end
19
+ end
data/lib/chronofage.rb ADDED
@@ -0,0 +1,16 @@
1
+ require 'active_record'
2
+ require 'active_record/version'
3
+ require 'active_support/core_ext/module'
4
+ require 'active_job/queue_adapters/chronofage_adapter'
5
+ require 'chronofage/job'
6
+ require 'chronofage/runner'
7
+
8
+ begin
9
+ require 'rails/engine'
10
+ require 'chronofage/engine'
11
+ rescue LoadError
12
+ end
13
+
14
+ module Chronofage
15
+ extend ActiveSupport::Autoload
16
+ end
@@ -0,0 +1,4 @@
1
+ module Chronofage
2
+ class Engine < Rails::Engine
3
+ end
4
+ end
@@ -0,0 +1,33 @@
1
+ module Chronofage
2
+ class Job < ::ActiveRecord::Base
3
+ self.table_name = "chronofage_jobs"
4
+
5
+ scope :ready, -> { where(started_at: nil) }
6
+
7
+ def execute!
8
+ start!
9
+ job_class.constantize.perform_now(*deserialized_arguments)
10
+ done!
11
+ rescue
12
+ failed!
13
+ raise
14
+ end
15
+
16
+ def start!
17
+ update(started_at: Time.now)
18
+ end
19
+
20
+ def done!
21
+ update(completed_at: Time.now)
22
+ end
23
+
24
+ def failed!
25
+ update(failed_at: Time.now)
26
+ end
27
+
28
+ def deserialized_arguments
29
+ ActiveJob::Arguments.deserialize(JSON.parse(arguments))
30
+ end
31
+
32
+ end
33
+ end
@@ -0,0 +1,33 @@
1
+ require 'socket'
2
+
3
+ module Chronofage
4
+ class Runner < ::ActiveRecord::Base
5
+ self.table_name = "chronofage_runners"
6
+
7
+ class MaxConcurrencyReached < StandardError
8
+ end
9
+
10
+ def self.register!(queue_name, concurrency)
11
+ if concurrent_runnners(queue_name).count >= concurrency
12
+ raise MaxConcurrencyReached
13
+ else
14
+ create!(queue_name: queue_name, host: host)
15
+ end
16
+ end
17
+
18
+ def self.concurrent_runnners(queue_name)
19
+ where(queue_name: queue_name, host: host)
20
+ end
21
+
22
+ def unregister!
23
+ destroy!
24
+ end
25
+
26
+ private
27
+
28
+ def self.host
29
+ Socket.gethostname
30
+ end
31
+
32
+ end
33
+ end
@@ -0,0 +1,3 @@
1
+ module Chronofage
2
+ VERSION = "0.0.1"
3
+ end
@@ -0,0 +1,18 @@
1
+ namespace :chronofage_engine do
2
+ namespace :jobs do
3
+ task :execute, [:queue_name, :concurrency] => :environment do |t, args|
4
+ job = Chronofage::Job.ready.where(queue_name: args.queue_name).order(priority: :asc).first
5
+
6
+ if job.nil?
7
+ Rails.logger.info "chronofage[#{args.queue_name}]: no job to execute."
8
+ else
9
+ runner = Chronofage::Runner.register!(args.queue_name, args.concurrency.to_i)
10
+ begin
11
+ job.execute!
12
+ ensure
13
+ runner.unregister!
14
+ end
15
+ end
16
+ end
17
+ end
18
+ end
metadata ADDED
@@ -0,0 +1,60 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: chronofage
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ platform: ruby
6
+ authors:
7
+ - Victor Goya
8
+ - Amuse Animation
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+ date: 2017-11-27 00:00:00.000000000 Z
13
+ dependencies: []
14
+ description: Cron based job scheduler
15
+ email:
16
+ - v.goya@millimages.com
17
+ executables: []
18
+ extensions: []
19
+ extra_rdoc_files: []
20
+ files:
21
+ - ".gitignore"
22
+ - Gemfile
23
+ - Gemfile.lock
24
+ - README.md
25
+ - Rakefile
26
+ - chronofage.gemspec
27
+ - db/migrate/1_create_job_table.rb
28
+ - db/migrate/2_create_runner_table.rb
29
+ - lib/active_job/queue_adapters/chronofage_adapter.rb
30
+ - lib/chronofage.rb
31
+ - lib/chronofage/engine.rb
32
+ - lib/chronofage/job.rb
33
+ - lib/chronofage/runner.rb
34
+ - lib/chronofage/version.rb
35
+ - lib/tasks/chronofage.rake
36
+ homepage: https://www.amusenetwork.com/
37
+ licenses:
38
+ - MIT
39
+ metadata: {}
40
+ post_install_message:
41
+ rdoc_options: []
42
+ require_paths:
43
+ - lib
44
+ required_ruby_version: !ruby/object:Gem::Requirement
45
+ requirements:
46
+ - - "~>"
47
+ - !ruby/object:Gem::Version
48
+ version: '2.0'
49
+ required_rubygems_version: !ruby/object:Gem::Requirement
50
+ requirements:
51
+ - - ">="
52
+ - !ruby/object:Gem::Version
53
+ version: '0'
54
+ requirements: []
55
+ rubyforge_project:
56
+ rubygems_version: 2.5.2
57
+ signing_key:
58
+ specification_version: 4
59
+ summary: Cron based job scheduler
60
+ test_files: []