inst-jobs 0.11.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (86) hide show
  1. checksums.yaml +7 -0
  2. data/bin/inst_job +4 -0
  3. data/db/migrate/20101216224513_create_delayed_jobs.rb +40 -0
  4. data/db/migrate/20110208031356_add_delayed_jobs_tag.rb +14 -0
  5. data/db/migrate/20110426161613_add_delayed_jobs_max_attempts.rb +13 -0
  6. data/db/migrate/20110516225834_add_delayed_jobs_strand.rb +14 -0
  7. data/db/migrate/20110531144916_cleanup_delayed_jobs_indexes.rb +26 -0
  8. data/db/migrate/20110610213249_optimize_delayed_jobs.rb +40 -0
  9. data/db/migrate/20110831210257_add_delayed_jobs_next_in_strand.rb +52 -0
  10. data/db/migrate/20120510004759_delayed_jobs_delete_trigger_lock_for_update.rb +31 -0
  11. data/db/migrate/20120531150712_drop_psql_jobs_pop_fn.rb +15 -0
  12. data/db/migrate/20120607164022_delayed_jobs_use_advisory_locks.rb +80 -0
  13. data/db/migrate/20120607181141_index_jobs_on_locked_by.rb +15 -0
  14. data/db/migrate/20120608191051_add_jobs_run_at_index.rb +15 -0
  15. data/db/migrate/20120927184213_change_delayed_jobs_handler_to_text.rb +13 -0
  16. data/db/migrate/20140505215131_add_failed_jobs_original_job_id.rb +13 -0
  17. data/db/migrate/20140505215510_copy_failed_jobs_original_id.rb +13 -0
  18. data/db/migrate/20140505223637_drop_failed_jobs_original_id.rb +13 -0
  19. data/db/migrate/20140512213941_add_source_to_jobs.rb +15 -0
  20. data/db/migrate/20150807133223_add_max_concurrent_to_jobs.rb +70 -0
  21. data/db/migrate/20151123210429_add_expires_at_to_jobs.rb +15 -0
  22. data/db/migrate/20151210162949_improve_max_concurrent.rb +50 -0
  23. data/lib/delayed/backend/active_record.rb +340 -0
  24. data/lib/delayed/backend/base.rb +335 -0
  25. data/lib/delayed/backend/redis/bulk_update.lua +50 -0
  26. data/lib/delayed/backend/redis/destroy_job.lua +2 -0
  27. data/lib/delayed/backend/redis/enqueue.lua +29 -0
  28. data/lib/delayed/backend/redis/fail_job.lua +5 -0
  29. data/lib/delayed/backend/redis/find_available.lua +3 -0
  30. data/lib/delayed/backend/redis/functions.rb +57 -0
  31. data/lib/delayed/backend/redis/get_and_lock_next_available.lua +17 -0
  32. data/lib/delayed/backend/redis/includes/jobs_common.lua +203 -0
  33. data/lib/delayed/backend/redis/job.rb +497 -0
  34. data/lib/delayed/backend/redis/set_running.lua +5 -0
  35. data/lib/delayed/backend/redis/tickle_strand.lua +2 -0
  36. data/lib/delayed/batch.rb +56 -0
  37. data/lib/delayed/cli.rb +101 -0
  38. data/lib/delayed/daemon.rb +103 -0
  39. data/lib/delayed/engine.rb +4 -0
  40. data/lib/delayed/job_tracking.rb +31 -0
  41. data/lib/delayed/lifecycle.rb +90 -0
  42. data/lib/delayed/log_tailer.rb +22 -0
  43. data/lib/delayed/message_sending.rb +134 -0
  44. data/lib/delayed/performable_method.rb +52 -0
  45. data/lib/delayed/periodic.rb +85 -0
  46. data/lib/delayed/plugin.rb +22 -0
  47. data/lib/delayed/pool.rb +161 -0
  48. data/lib/delayed/server/helpers.rb +28 -0
  49. data/lib/delayed/server/public/css/app.css +12 -0
  50. data/lib/delayed/server/public/js/app.js +132 -0
  51. data/lib/delayed/server/views/index.erb +90 -0
  52. data/lib/delayed/server/views/layout.erb +47 -0
  53. data/lib/delayed/server.rb +120 -0
  54. data/lib/delayed/settings.rb +90 -0
  55. data/lib/delayed/testing.rb +32 -0
  56. data/lib/delayed/version.rb +3 -0
  57. data/lib/delayed/work_queue/in_process.rb +13 -0
  58. data/lib/delayed/work_queue/parent_process.rb +180 -0
  59. data/lib/delayed/worker.rb +234 -0
  60. data/lib/delayed/yaml_extensions.rb +109 -0
  61. data/lib/delayed_job.rb +46 -0
  62. data/lib/inst-jobs.rb +1 -0
  63. data/spec/active_record_job_spec.rb +246 -0
  64. data/spec/delayed/cli_spec.rb +23 -0
  65. data/spec/delayed/daemon_spec.rb +35 -0
  66. data/spec/delayed/server_spec.rb +63 -0
  67. data/spec/delayed/settings_spec.rb +32 -0
  68. data/spec/delayed/work_queue/in_process_spec.rb +31 -0
  69. data/spec/delayed/work_queue/parent_process_spec.rb +159 -0
  70. data/spec/delayed/worker_spec.rb +16 -0
  71. data/spec/gemfiles/32.gemfile +6 -0
  72. data/spec/gemfiles/40.gemfile +5 -0
  73. data/spec/gemfiles/41.gemfile +5 -0
  74. data/spec/gemfiles/42.gemfile +5 -0
  75. data/spec/migrate/20140924140513_add_story_table.rb +7 -0
  76. data/spec/redis_job_spec.rb +140 -0
  77. data/spec/sample_jobs.rb +28 -0
  78. data/spec/shared/delayed_batch.rb +85 -0
  79. data/spec/shared/delayed_method.rb +419 -0
  80. data/spec/shared/performable_method.rb +66 -0
  81. data/spec/shared/shared_backend.rb +819 -0
  82. data/spec/shared/testing.rb +48 -0
  83. data/spec/shared/worker.rb +378 -0
  84. data/spec/shared_jobs_specs.rb +15 -0
  85. data/spec/spec_helper.rb +97 -0
  86. metadata +390 -0
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: bc63ba5e864d96838548c1219044e8f8a1cc9435
4
+ data.tar.gz: 94ba64f0cac98fafb26b46552267e0ad84197561
5
+ SHA512:
6
+ metadata.gz: 277a619901441ba700552e5c6bff8c93117929d8615ce0a87aa7582f16c045572b33468e4b0e8a967421bc1ad7d3a5bdf5c2c201f95b9e1b9bedd3bf17fbf4f3
7
+ data.tar.gz: 3b2b21740daba532b04504f720601430e8603203e4815376d7522ec8bbd9ba9d7c836a9d041a15e84fdcf688709cfbabbe94fa4653b58ef008343d323e1bc41c
data/bin/inst_job ADDED
@@ -0,0 +1,4 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require File.expand_path('config/environment')
4
+ Delayed::CLI.new.run()
@@ -0,0 +1,40 @@
1
+ class CreateDelayedJobs < ActiveRecord::Migration
2
+ def connection
3
+ Delayed::Backend::ActiveRecord::Job.connection
4
+ end
5
+
6
+ def up
7
+ raise("#{connection.adapter_name} is not supported for delayed jobs queue") unless connection.adapter_name == 'PostgreSQL'
8
+
9
+ create_table :delayed_jobs do |table|
10
+ # Allows some jobs to jump to the front of the queue
11
+ table.integer :priority, :default => 0
12
+ # Provides for retries, but still fail eventually.
13
+ table.integer :attempts, :default => 0
14
+ # YAML-encoded string of the object that will do work
15
+ table.text :handler, :limit => (500 * 1024)
16
+ # reason for last failure (See Note below)
17
+ table.text :last_error
18
+ # The queue that this job is in
19
+ table.string :queue, :default => nil
20
+ # When to run.
21
+ # Could be Time.zone.now for immediately, or sometime in the future.
22
+ table.datetime :run_at
23
+ # Set when a client is working on this object
24
+ table.datetime :locked_at
25
+ # Set when all retries have failed
26
+ table.datetime :failed_at
27
+ # Who is working on this object (if locked)
28
+ table.string :locked_by
29
+
30
+ table.timestamps
31
+ end
32
+
33
+ add_index :delayed_jobs, [:priority, :run_at], :name => 'delayed_jobs_priority'
34
+ add_index :delayed_jobs, [:queue], :name => 'delayed_jobs_queue'
35
+ end
36
+
37
+ def down
38
+ drop_table :delayed_jobs
39
+ end
40
+ end
@@ -0,0 +1,14 @@
1
+ class AddDelayedJobsTag < ActiveRecord::Migration
2
+ def connection
3
+ Delayed::Backend::ActiveRecord::Job.connection
4
+ end
5
+
6
+ def up
7
+ add_column :delayed_jobs, :tag, :string
8
+ add_index :delayed_jobs, [:tag]
9
+ end
10
+
11
+ def down
12
+ remove_column :delayed_jobs, :tag
13
+ end
14
+ end
@@ -0,0 +1,13 @@
1
+ class AddDelayedJobsMaxAttempts < ActiveRecord::Migration
2
+ def connection
3
+ Delayed::Backend::ActiveRecord::Job.connection
4
+ end
5
+
6
+ def up
7
+ add_column :delayed_jobs, :max_attempts, :integer
8
+ end
9
+
10
+ def down
11
+ remove_column :delayed_jobs, :max_attempts
12
+ end
13
+ end
@@ -0,0 +1,14 @@
1
+ class AddDelayedJobsStrand < ActiveRecord::Migration
2
+ def connection
3
+ Delayed::Backend::ActiveRecord::Job.connection
4
+ end
5
+
6
+ def up
7
+ add_column :delayed_jobs, :strand, :string
8
+ add_index :delayed_jobs, :strand
9
+ end
10
+
11
+ def down
12
+ remove_column :delayed_jobs, :strand
13
+ end
14
+ end
@@ -0,0 +1,26 @@
1
+ class CleanupDelayedJobsIndexes < ActiveRecord::Migration
2
+ def connection
3
+ Delayed::Backend::ActiveRecord::Job.connection
4
+ end
5
+
6
+ def up
7
+ case connection.adapter_name
8
+ when 'PostgreSQL'
9
+ # "nulls first" syntax is postgresql specific, and allows for more
10
+ # efficient querying for the next job
11
+ connection.execute("CREATE INDEX get_delayed_jobs_index ON delayed_jobs (priority, run_at, failed_at nulls first, locked_at nulls first, queue)")
12
+ else
13
+ add_index :delayed_jobs, %w(priority run_at locked_at failed_at queue), :name => 'get_delayed_jobs_index'
14
+ end
15
+
16
+ # unused indexes
17
+ remove_index :delayed_jobs, :name => 'delayed_jobs_queue'
18
+ remove_index :delayed_jobs, :name => 'delayed_jobs_priority'
19
+ end
20
+
21
+ def down
22
+ remove_index :delayed_jobs, :name => 'get_delayed_jobs_index'
23
+ add_index :delayed_jobs, [:priority, :run_at], :name => 'delayed_jobs_priority'
24
+ add_index :delayed_jobs, [:queue], :name => 'delayed_jobs_queue'
25
+ end
26
+ end
@@ -0,0 +1,40 @@
1
+ class OptimizeDelayedJobs < ActiveRecord::Migration
2
+ def connection
3
+ Delayed::Backend::ActiveRecord::Job.connection
4
+ end
5
+
6
+ def up
7
+ create_table :failed_jobs do |t|
8
+ t.integer "priority", :default => 0
9
+ t.integer "attempts", :default => 0
10
+ t.string "handler", :limit => 512000
11
+ t.integer "original_id", :limit => 8
12
+ t.text "last_error"
13
+ t.string "queue"
14
+ t.datetime "run_at"
15
+ t.datetime "locked_at"
16
+ t.datetime "failed_at"
17
+ t.string "locked_by"
18
+ t.datetime "created_at"
19
+ t.datetime "updated_at"
20
+ t.string "tag"
21
+ t.integer "max_attempts"
22
+ t.string "strand"
23
+ end
24
+
25
+ remove_index :delayed_jobs, :name => 'get_delayed_jobs_index'
26
+ remove_index :delayed_jobs, [:strand]
27
+
28
+ add_index :delayed_jobs, %w(run_at queue locked_at strand priority), :name => 'index_delayed_jobs_for_get_next'
29
+ add_index :delayed_jobs, %w(strand id), :name => 'index_delayed_jobs_on_strand'
30
+
31
+ # move all failed jobs to the new failed table
32
+ Delayed::Backend::ActiveRecord::Job.where("failed_at IS NOT NULL").find_each do |job|
33
+ job.fail! unless job.on_hold?
34
+ end
35
+ end
36
+
37
+ def down
38
+ raise ActiveRecord::IrreversibleMigration
39
+ end
40
+ end
@@ -0,0 +1,52 @@
1
+ class AddDelayedJobsNextInStrand < ActiveRecord::Migration
2
+ def connection
3
+ Delayed::Backend::ActiveRecord::Job.connection
4
+ end
5
+
6
+ def up
7
+ remove_index :delayed_jobs, :name => 'index_delayed_jobs_for_get_next'
8
+
9
+ add_column :delayed_jobs, :next_in_strand, :boolean, :default => true, :null => false
10
+
11
+ # create the new index
12
+ connection.execute("CREATE INDEX get_delayed_jobs_index ON delayed_jobs (priority, run_at, queue) WHERE locked_at IS NULL AND next_in_strand = 't'")
13
+
14
+ # create the insert trigger
15
+ execute(<<-CODE)
16
+ CREATE FUNCTION delayed_jobs_before_insert_row_tr_fn () RETURNS trigger AS $$
17
+ BEGIN
18
+ LOCK delayed_jobs IN SHARE ROW EXCLUSIVE MODE;
19
+ IF (SELECT 1 FROM delayed_jobs WHERE strand = NEW.strand LIMIT 1) = 1 THEN
20
+ NEW.next_in_strand := 'f';
21
+ END IF;
22
+ RETURN NEW;
23
+ END;
24
+ $$ LANGUAGE plpgsql;
25
+ CODE
26
+ execute("CREATE TRIGGER delayed_jobs_before_insert_row_tr BEFORE INSERT ON delayed_jobs FOR EACH ROW WHEN (NEW.strand IS NOT NULL) EXECUTE PROCEDURE delayed_jobs_before_insert_row_tr_fn()")
27
+
28
+ # create the delete trigger
29
+ execute(<<-CODE)
30
+ CREATE FUNCTION delayed_jobs_after_delete_row_tr_fn () RETURNS trigger AS $$
31
+ BEGIN
32
+ UPDATE delayed_jobs SET next_in_strand = 't' WHERE id = (SELECT id FROM delayed_jobs j2 WHERE j2.strand = OLD.strand ORDER BY j2.strand, j2.id ASC LIMIT 1);
33
+ RETURN OLD;
34
+ END;
35
+ $$ LANGUAGE plpgsql;
36
+ CODE
37
+ execute("CREATE TRIGGER delayed_jobs_after_delete_row_tr AFTER DELETE ON delayed_jobs FOR EACH ROW WHEN (OLD.strand IS NOT NULL AND OLD.next_in_strand = 't') EXECUTE PROCEDURE delayed_jobs_after_delete_row_tr_fn()")
38
+
39
+ execute(%{UPDATE delayed_jobs SET next_in_strand = 'f' WHERE strand IS NOT NULL AND id <> (SELECT id FROM delayed_jobs j2 WHERE j2.strand = delayed_jobs.strand ORDER BY j2.strand, j2.id ASC LIMIT 1)})
40
+ end
41
+
42
+ def down
43
+ execute %{DROP TRIGGER delayed_jobs_before_insert_row_tr ON delayed_jobs}
44
+ execute %{DROP FUNCTION delayed_jobs_before_insert_row_tr_fn()}
45
+ execute %{DROP TRIGGER delayed_jobs_after_delete_row_tr ON delayed_jobs}
46
+ execute %{DROP FUNCTION delayed_jobs_after_delete_row_tr_fn()}
47
+
48
+ remove_column :delayed_jobs, :next_in_strand
49
+ remove_index :delayed_jobs, :name => 'get_delayed_jobs_index'
50
+ add_index :delayed_jobs, %w(run_at queue locked_at strand priority), :name => 'index_delayed_jobs_for_get_next'
51
+ end
52
+ end
@@ -0,0 +1,31 @@
1
+ class DelayedJobsDeleteTriggerLockForUpdate < ActiveRecord::Migration
2
+ def connection
3
+ Delayed::Backend::ActiveRecord::Job.connection
4
+ end
5
+
6
+ def up
7
+ if connection.adapter_name == 'PostgreSQL'
8
+ execute(<<-CODE)
9
+ CREATE OR REPLACE FUNCTION delayed_jobs_after_delete_row_tr_fn () RETURNS trigger AS $$
10
+ BEGIN
11
+ UPDATE delayed_jobs SET next_in_strand = 't' WHERE id = (SELECT id FROM delayed_jobs j2 WHERE j2.strand = OLD.strand ORDER BY j2.strand, j2.id ASC LIMIT 1 FOR UPDATE);
12
+ RETURN OLD;
13
+ END;
14
+ $$ LANGUAGE plpgsql;
15
+ CODE
16
+ end
17
+ end
18
+
19
+ def down
20
+ if connection.adapter_name == 'PostgreSQL'
21
+ execute(<<-CODE)
22
+ CREATE OR REPLACE FUNCTION delayed_jobs_after_delete_row_tr_fn () RETURNS trigger AS $$
23
+ BEGIN
24
+ UPDATE delayed_jobs SET next_in_strand = 't' WHERE id = (SELECT id FROM delayed_jobs j2 WHERE j2.strand = OLD.strand ORDER BY j2.strand, j2.id ASC LIMIT 1);
25
+ RETURN OLD;
26
+ END;
27
+ $$ LANGUAGE plpgsql;
28
+ CODE
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,15 @@
1
+ class DropPsqlJobsPopFn < ActiveRecord::Migration
2
+ def connection
3
+ Delayed::Backend::ActiveRecord::Job.connection
4
+ end
5
+
6
+ def up
7
+ if connection.adapter_name == 'PostgreSQL'
8
+ connection.execute("DROP FUNCTION IF EXISTS pop_from_delayed_jobs(varchar, varchar, integer, integer, timestamp without time zone)")
9
+ end
10
+ end
11
+
12
+ def down
13
+ raise ActiveRecord::IrreversibleMigration
14
+ end
15
+ end
@@ -0,0 +1,80 @@
1
+ class DelayedJobsUseAdvisoryLocks < ActiveRecord::Migration
2
+ def connection
3
+ Delayed::Backend::ActiveRecord::Job.connection
4
+ end
5
+
6
+ def up
7
+ # use an advisory lock based on the name of the strand, instead of locking the whole table
8
+ # note that we're using half of the md5, so collisions are possible, but we don't really
9
+ # care because that would just be the old behavior, whereas for the most part locking will
10
+ # be much smaller
11
+ if connection.adapter_name == 'PostgreSQL'
12
+ execute(<<-CODE)
13
+ CREATE FUNCTION half_md5_as_bigint(strand varchar) RETURNS bigint AS $$
14
+ DECLARE
15
+ strand_md5 bytea;
16
+ BEGIN
17
+ strand_md5 := decode(md5(strand), 'hex');
18
+ RETURN (CAST(get_byte(strand_md5, 0) AS bigint) << 56) +
19
+ (CAST(get_byte(strand_md5, 1) AS bigint) << 48) +
20
+ (CAST(get_byte(strand_md5, 2) AS bigint) << 40) +
21
+ (CAST(get_byte(strand_md5, 3) AS bigint) << 32) +
22
+ (CAST(get_byte(strand_md5, 4) AS bigint) << 24) +
23
+ (get_byte(strand_md5, 5) << 16) +
24
+ (get_byte(strand_md5, 6) << 8) +
25
+ get_byte(strand_md5, 7);
26
+ END;
27
+ $$ LANGUAGE plpgsql;
28
+ CODE
29
+
30
+ execute(<<-CODE)
31
+ CREATE OR REPLACE FUNCTION delayed_jobs_before_insert_row_tr_fn () RETURNS trigger AS $$
32
+ BEGIN
33
+ PERFORM pg_advisory_xact_lock(half_md5_as_bigint(NEW.strand));
34
+ IF (SELECT 1 FROM delayed_jobs WHERE strand = NEW.strand LIMIT 1) = 1 THEN
35
+ NEW.next_in_strand := 'f';
36
+ END IF;
37
+ RETURN NEW;
38
+ END;
39
+ $$ LANGUAGE plpgsql;
40
+ CODE
41
+
42
+ execute(<<-CODE)
43
+ CREATE OR REPLACE FUNCTION delayed_jobs_after_delete_row_tr_fn () RETURNS trigger AS $$
44
+ BEGIN
45
+ PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
46
+ UPDATE delayed_jobs SET next_in_strand = 't' WHERE id = (SELECT id FROM delayed_jobs j2 WHERE j2.strand = OLD.strand ORDER BY j2.strand, j2.id ASC LIMIT 1 FOR UPDATE);
47
+ RETURN OLD;
48
+ END;
49
+ $$ LANGUAGE plpgsql;
50
+ CODE
51
+ end
52
+ end
53
+
54
+ def down
55
+ if connection.adapter_name == 'PostgreSQL'
56
+ execute(<<-CODE)
57
+ CREATE OR REPLACE FUNCTION delayed_jobs_before_insert_row_tr_fn () RETURNS trigger AS $$
58
+ BEGIN
59
+ LOCK delayed_jobs IN SHARE ROW EXCLUSIVE MODE;
60
+ IF (SELECT 1 FROM delayed_jobs WHERE strand = NEW.strand LIMIT 1) = 1 THEN
61
+ NEW.next_in_strand := 'f';
62
+ END IF;
63
+ RETURN NEW;
64
+ END;
65
+ $$ LANGUAGE plpgsql;
66
+ CODE
67
+
68
+ execute(<<-CODE)
69
+ CREATE OR REPLACE FUNCTION delayed_jobs_after_delete_row_tr_fn () RETURNS trigger AS $$
70
+ BEGIN
71
+ UPDATE delayed_jobs SET next_in_strand = 't' WHERE id = (SELECT id FROM delayed_jobs j2 WHERE j2.strand = OLD.strand ORDER BY j2.strand, j2.id ASC LIMIT 1 FOR UPDATE);
72
+ RETURN OLD;
73
+ END;
74
+ $$ LANGUAGE plpgsql;
75
+ CODE
76
+
77
+ execute('DROP FUNCTION half_md5_as_bigint(varchar)')
78
+ end
79
+ end
80
+ end
@@ -0,0 +1,15 @@
1
+ class IndexJobsOnLockedBy < ActiveRecord::Migration
2
+ disable_ddl_transaction! if respond_to?(:disable_ddl_transaction!)
3
+
4
+ def connection
5
+ Delayed::Backend::ActiveRecord::Job.connection
6
+ end
7
+
8
+ def up
9
+ add_index :delayed_jobs, :locked_by, :algorithm => :concurrently, :where => "locked_by IS NOT NULL"
10
+ end
11
+
12
+ def down
13
+ remove_index :delayed_jobs, :locked_by
14
+ end
15
+ end
@@ -0,0 +1,15 @@
1
+ class AddJobsRunAtIndex < ActiveRecord::Migration
2
+ disable_ddl_transaction! if respond_to?(:disable_ddl_transaction!)
3
+
4
+ def connection
5
+ Delayed::Backend::ActiveRecord::Job.connection
6
+ end
7
+
8
+ def up
9
+ add_index :delayed_jobs, %w[run_at tag], :algorithm => :concurrently
10
+ end
11
+
12
+ def down
13
+ remove_index :delayed_jobs, :name => "index_delayed_jobs_on_run_at_and_tag"
14
+ end
15
+ end
@@ -0,0 +1,13 @@
1
+ class ChangeDelayedJobsHandlerToText < ActiveRecord::Migration
2
+ def connection
3
+ Delayed::Job.connection
4
+ end
5
+
6
+ def up
7
+ change_column :delayed_jobs, :handler, :text
8
+ end
9
+
10
+ def down
11
+ change_column :delayed_jobs, :handler, :string, :limit => 500.kilobytes
12
+ end
13
+ end
@@ -0,0 +1,13 @@
1
+ class AddFailedJobsOriginalJobId < ActiveRecord::Migration
2
+ def connection
3
+ Delayed::Backend::ActiveRecord::Job.connection
4
+ end
5
+
6
+ def up
7
+ add_column :failed_jobs, :original_job_id, :integer, limit: 8
8
+ end
9
+
10
+ def down
11
+ remove_column :failed_jobs, :original_job_id
12
+ end
13
+ end
@@ -0,0 +1,13 @@
1
+ class CopyFailedJobsOriginalId < ActiveRecord::Migration
2
+ def connection
3
+ Delayed::Backend::ActiveRecord::Job.connection
4
+ end
5
+
6
+ def up
7
+ # this is a smaller, less frequently accessed table, so we just update all at once
8
+ Delayed::Backend::ActiveRecord::Job::Failed.where("original_job_id is null").update_all("original_job_id = original_id")
9
+ end
10
+
11
+ def down
12
+ end
13
+ end
@@ -0,0 +1,13 @@
1
+ class DropFailedJobsOriginalId < ActiveRecord::Migration
2
+ def connection
3
+ Delayed::Backend::ActiveRecord::Job.connection
4
+ end
5
+
6
+ def up
7
+ remove_column :failed_jobs, :original_id
8
+ end
9
+
10
+ def down
11
+ add_column :failed_jobs, :original_id, :integer, limit: 8
12
+ end
13
+ end
@@ -0,0 +1,15 @@
1
+ class AddSourceToJobs < ActiveRecord::Migration
2
+ def connection
3
+ Delayed::Job.connection
4
+ end
5
+
6
+ def up
7
+ add_column :delayed_jobs, :source, :string
8
+ add_column :failed_jobs, :source, :string
9
+ end
10
+
11
+ def down
12
+ remove_column :delayed_jobs, :source
13
+ remove_column :failed_jobs, :source
14
+ end
15
+ end
@@ -0,0 +1,70 @@
1
+ class AddMaxConcurrentToJobs < ActiveRecord::Migration
2
+ def connection
3
+ Delayed::Job.connection
4
+ end
5
+
6
+ def up
7
+ add_column :delayed_jobs, :max_concurrent, :integer, :default => 1, :null => false
8
+
9
+ if connection.adapter_name == 'PostgreSQL'
10
+ execute(<<-CODE)
11
+ CREATE OR REPLACE FUNCTION delayed_jobs_before_insert_row_tr_fn () RETURNS trigger AS $$
12
+ BEGIN
13
+ IF NEW.strand IS NOT NULL THEN
14
+ PERFORM pg_advisory_xact_lock(half_md5_as_bigint(NEW.strand));
15
+ IF (SELECT COUNT(*) FROM delayed_jobs WHERE strand = NEW.strand) >= NEW.max_concurrent THEN
16
+ NEW.next_in_strand := 'f';
17
+ END IF;
18
+ END IF;
19
+ RETURN NEW;
20
+ END;
21
+ $$ LANGUAGE plpgsql;
22
+ CODE
23
+
24
+ execute(<<-CODE)
25
+ CREATE OR REPLACE FUNCTION delayed_jobs_after_delete_row_tr_fn () RETURNS trigger AS $$
26
+ BEGIN
27
+ IF OLD.strand IS NOT NULL THEN
28
+ PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
29
+ IF (SELECT COUNT(*) FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't') < OLD.max_concurrent THEN
30
+ UPDATE delayed_jobs SET next_in_strand = 't' WHERE id = (
31
+ SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
32
+ j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE
33
+ );
34
+ END IF;
35
+ END IF;
36
+ RETURN OLD;
37
+ END;
38
+ $$ LANGUAGE plpgsql;
39
+ CODE
40
+ end
41
+ end
42
+
43
+ def down
44
+ remove_column :delayed_jobs, :max_concurrent
45
+
46
+ if connection.adapter_name == 'PostgreSQL'
47
+ execute(<<-CODE)
48
+ CREATE OR REPLACE FUNCTION delayed_jobs_before_insert_row_tr_fn () RETURNS trigger AS $$
49
+ BEGIN
50
+ PERFORM pg_advisory_xact_lock(half_md5_as_bigint(NEW.strand));
51
+ IF (SELECT 1 FROM delayed_jobs WHERE strand = NEW.strand LIMIT 1) = 1 THEN
52
+ NEW.next_in_strand := 'f';
53
+ END IF;
54
+ RETURN NEW;
55
+ END;
56
+ $$ LANGUAGE plpgsql;
57
+ CODE
58
+
59
+ execute(<<-CODE)
60
+ CREATE OR REPLACE FUNCTION delayed_jobs_after_delete_row_tr_fn () RETURNS trigger AS $$
61
+ BEGIN
62
+ PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
63
+ UPDATE delayed_jobs SET next_in_strand = 't' WHERE id = (SELECT id FROM delayed_jobs j2 WHERE j2.strand = OLD.strand ORDER BY j2.strand, j2.id ASC LIMIT 1 FOR UPDATE);
64
+ RETURN OLD;
65
+ END;
66
+ $$ LANGUAGE plpgsql;
67
+ CODE
68
+ end
69
+ end
70
+ end
@@ -0,0 +1,15 @@
1
+ class AddExpiresAtToJobs < ActiveRecord::Migration
2
+ def connection
3
+ Delayed::Job.connection
4
+ end
5
+
6
+ def up
7
+ add_column :delayed_jobs, :expires_at, :datetime
8
+ add_column :failed_jobs, :expires_at, :datetime
9
+ end
10
+
11
+ def down
12
+ remove_column :delayed_jobs, :expires_at
13
+ remove_column :failed_jobs, :expires_at
14
+ end
15
+ end
@@ -0,0 +1,50 @@
1
+ class ImproveMaxConcurrent < ActiveRecord::Migration
2
+ def connection
3
+ Delayed::Job.connection
4
+ end
5
+
6
+ def up
7
+ if connection.adapter_name == 'PostgreSQL'
8
+ execute(<<-CODE)
9
+ CREATE OR REPLACE FUNCTION delayed_jobs_after_delete_row_tr_fn () RETURNS trigger AS $$
10
+ DECLARE
11
+ running_count integer;
12
+ BEGIN
13
+ IF OLD.strand IS NOT NULL THEN
14
+ PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
15
+ running_count := (SELECT COUNT(*) FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't');
16
+ IF running_count < OLD.max_concurrent THEN
17
+ UPDATE delayed_jobs SET next_in_strand = 't' WHERE id IN (
18
+ SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
19
+ j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT (OLD.max_concurrent - running_count) FOR UPDATE
20
+ );
21
+ END IF;
22
+ END IF;
23
+ RETURN OLD;
24
+ END;
25
+ $$ LANGUAGE plpgsql;
26
+ CODE
27
+ end
28
+ end
29
+
30
+ def down
31
+ if connection.adapter_name == 'PostgreSQL'
32
+ execute(<<-CODE)
33
+ CREATE OR REPLACE FUNCTION delayed_jobs_after_delete_row_tr_fn () RETURNS trigger AS $$
34
+ BEGIN
35
+ IF OLD.strand IS NOT NULL THEN
36
+ PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
37
+ IF (SELECT COUNT(*) FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't') < OLD.max_concurrent THEN
38
+ UPDATE delayed_jobs SET next_in_strand = 't' WHERE id = (
39
+ SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
40
+ j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE
41
+ );
42
+ END IF;
43
+ END IF;
44
+ RETURN OLD;
45
+ END;
46
+ $$ LANGUAGE plpgsql;
47
+ CODE
48
+ end
49
+ end
50
+ end