inst-jobs 0.15.22 → 0.16.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 33b7f135ba67d0b2b056851cec2e95672ac03766fa48d614ef53fa0e25a04255
4
- data.tar.gz: 29878f6688d376811085f7615c701ae2f7cf31cc3bf6462440823f4e91ccb271
3
+ metadata.gz: d1547b62f3c5fd86703cc209ec4b0feb6954db3c6fb4b58f4bcdbcf58dc9a9eb
4
+ data.tar.gz: 95ad1cacd71ac4511ee29eb0d3fda780037768234c232302b6fc10084de9e351
5
5
  SHA512:
6
- metadata.gz: 8695ff4bd7e9c10a5b1a393adc56d0be6cb1b45c668a8ba413bf91a59a8b1eb76633f0f8e3429ff15c6a0f03a9404e3e4f0d3bacd6f2cbd0ef3984ac5b0e0ad7
7
- data.tar.gz: 63f911fce5ddf75e7565095c134ffa0a3808fc5bf2b8e86f62466092b32a127afa449cbb44007774c771838fd30222a957246cbb44e9b8b125b01c2a7e8f2fe0
6
+ metadata.gz: bd27fc7e6c0ca5a613ba98ea0928d7947f2344d05c57ab14f588ff615c69bab60a0b6f08e60d1b610b6746ad8a467d151237f1469129f6144d5e93ce20ea71ef
7
+ data.tar.gz: 25ce44b9f6eeedae9605c4bc2077a9e9b51a4b7df94ba4d68be1e78f70801af65273c332caba780c11bf523cff31daed9c44d8f068f697a8910781b09a2daf88
@@ -0,0 +1,95 @@
1
+ class SpeedUpMaxConcurrentDeleteTrigger < ActiveRecord::Migration[4.2]
2
+ def connection
3
+ Delayed::Job.connection
4
+ end
5
+
6
+ def up
7
+ if connection.adapter_name == 'PostgreSQL'
8
+ # tl;dr sacrifice some responsiveness to max_concurrent changes for faster performance
9
+ # don't get the count every single time - it's usually safe to just set the next one in line
10
+ # since the max_concurrent doesn't change all that often for a strand
11
+ execute(<<-SQL)
12
+ CREATE OR REPLACE FUNCTION delayed_jobs_after_delete_row_tr_fn () RETURNS trigger AS $$
13
+ DECLARE
14
+ running_count integer;
15
+ should_lock boolean;
16
+ should_be_precise boolean;
17
+ BEGIN
18
+ IF OLD.strand IS NOT NULL THEN
19
+ should_lock := true;
20
+ should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
21
+
22
+ IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
23
+ running_count := (SELECT COUNT(*) FROM (
24
+ SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
25
+ ) subquery_for_count);
26
+ should_lock := running_count < OLD.max_concurrent;
27
+ END IF;
28
+
29
+ IF should_lock THEN
30
+ PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
31
+ END IF;
32
+
33
+ IF should_be_precise THEN
34
+ running_count := (SELECT COUNT(*) FROM (
35
+ SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
36
+ ) subquery_for_count);
37
+ IF running_count < OLD.max_concurrent THEN
38
+ UPDATE delayed_jobs SET next_in_strand = 't' WHERE id IN (
39
+ SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
40
+ j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT (OLD.max_concurrent - running_count) FOR UPDATE
41
+ );
42
+ END IF;
43
+ ELSE
44
+ -- n-strands don't require precise ordering; we can make this query more performant
45
+ IF OLD.max_concurrent > 1 THEN
46
+ UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
47
+ (SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
48
+ j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE SKIP LOCKED);
49
+ ELSE
50
+ UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
51
+ (SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
52
+ j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE);
53
+ END IF;
54
+ END IF;
55
+ END IF;
56
+ RETURN OLD;
57
+ END;
58
+ $$ LANGUAGE plpgsql;
59
+ SQL
60
+ end
61
+ end
62
+
63
+ def down
64
+ if connection.adapter_name == 'PostgreSQL'
65
+ execute(<<-SQL)
66
+ CREATE OR REPLACE FUNCTION delayed_jobs_after_delete_row_tr_fn () RETURNS trigger AS $$
67
+ DECLARE
68
+ running_count integer;
69
+ BEGIN
70
+ IF OLD.strand IS NOT NULL THEN
71
+ PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
72
+ IF OLD.id % 20 = 0 THEN
73
+ running_count := (SELECT COUNT(*) FROM (
74
+ SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
75
+ ) subquery_for_count);
76
+ IF running_count < OLD.max_concurrent THEN
77
+ UPDATE delayed_jobs SET next_in_strand = 't' WHERE id IN (
78
+ SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
79
+ j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT (OLD.max_concurrent - running_count) FOR UPDATE
80
+ );
81
+ END IF;
82
+ ELSE
83
+ UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
84
+ (SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
85
+ j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE);
86
+ END IF;
87
+ END IF;
88
+ RETURN OLD;
89
+ END;
90
+ $$ LANGUAGE plpgsql;
91
+ SQL
92
+ end
93
+ end
94
+ end
95
+
@@ -0,0 +1,126 @@
1
+ class AddStrandOrderOverride < ActiveRecord::Migration[4.2]
2
+ disable_ddl_transaction! if respond_to?(:disable_ddl_transaction!)
3
+
4
+ def connection
5
+ Delayed::Job.connection
6
+ end
7
+
8
+ def up
9
+ add_column :delayed_jobs, :strand_order_override, :integer, default: 0, null: false
10
+ add_column :failed_jobs, :strand_order_override, :integer, default: 0, null: false
11
+ add_index :delayed_jobs, [:strand, :strand_order_override, :id],
12
+ algorithm: :concurrently,
13
+ where: "strand IS NOT NULL",
14
+ name: "next_in_strand_index"
15
+
16
+ if connection.adapter_name == 'PostgreSQL'
17
+ # Use the strand_order_override as the primary sorting mechanism (useful when moving between jobs queues without preserving ID ordering)
18
+ execute(<<-SQL)
19
+ CREATE OR REPLACE FUNCTION delayed_jobs_after_delete_row_tr_fn () RETURNS trigger AS $$
20
+ DECLARE
21
+ running_count integer;
22
+ should_lock boolean;
23
+ should_be_precise boolean;
24
+ BEGIN
25
+ IF OLD.strand IS NOT NULL THEN
26
+ should_lock := true;
27
+ should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
28
+
29
+ IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
30
+ running_count := (SELECT COUNT(*) FROM (
31
+ SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
32
+ ) subquery_for_count);
33
+ should_lock := running_count < OLD.max_concurrent;
34
+ END IF;
35
+
36
+ IF should_lock THEN
37
+ PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
38
+ END IF;
39
+
40
+ IF should_be_precise THEN
41
+ running_count := (SELECT COUNT(*) FROM (
42
+ SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
43
+ ) subquery_for_count);
44
+ IF running_count < OLD.max_concurrent THEN
45
+ UPDATE delayed_jobs SET next_in_strand = 't' WHERE id IN (
46
+ SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
47
+ j2.strand = OLD.strand ORDER BY j2.strand_order_override ASC, j2.id ASC LIMIT (OLD.max_concurrent - running_count) FOR UPDATE
48
+ );
49
+ END IF;
50
+ ELSE
51
+ -- n-strands don't require precise ordering; we can make this query more performant
52
+ IF OLD.max_concurrent > 1 THEN
53
+ UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
54
+ (SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
55
+ j2.strand = OLD.strand ORDER BY j2.strand_order_override ASC, j2.id ASC LIMIT 1 FOR UPDATE SKIP LOCKED);
56
+ ELSE
57
+ UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
58
+ (SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
59
+ j2.strand = OLD.strand ORDER BY j2.strand_order_override ASC, j2.id ASC LIMIT 1 FOR UPDATE);
60
+ END IF;
61
+ END IF;
62
+ END IF;
63
+ RETURN OLD;
64
+ END;
65
+ $$ LANGUAGE plpgsql;
66
+ SQL
67
+ end
68
+ end
69
+
70
+ def down
71
+ remove_column :delayed_jobs, :strand_order_override, :integer
72
+ remove_column :failed_jobs, :strand_order_override, :integer
73
+
74
+ if connection.adapter_name == 'PostgreSQL'
75
+ execute(<<-SQL)
76
+ CREATE OR REPLACE FUNCTION delayed_jobs_after_delete_row_tr_fn () RETURNS trigger AS $$
77
+ DECLARE
78
+ running_count integer;
79
+ should_lock boolean;
80
+ should_be_precise boolean;
81
+ BEGIN
82
+ IF OLD.strand IS NOT NULL THEN
83
+ should_lock := true;
84
+ should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
85
+
86
+ IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
87
+ running_count := (SELECT COUNT(*) FROM (
88
+ SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
89
+ ) subquery_for_count);
90
+ should_lock := running_count < OLD.max_concurrent;
91
+ END IF;
92
+
93
+ IF should_lock THEN
94
+ PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
95
+ END IF;
96
+
97
+ IF should_be_precise THEN
98
+ running_count := (SELECT COUNT(*) FROM (
99
+ SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
100
+ ) subquery_for_count);
101
+ IF running_count < OLD.max_concurrent THEN
102
+ UPDATE delayed_jobs SET next_in_strand = 't' WHERE id IN (
103
+ SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
104
+ j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT (OLD.max_concurrent - running_count) FOR UPDATE
105
+ );
106
+ END IF;
107
+ ELSE
108
+ -- n-strands don't require precise ordering; we can make this query more performant
109
+ IF OLD.max_concurrent > 1 THEN
110
+ UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
111
+ (SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
112
+ j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE SKIP LOCKED);
113
+ ELSE
114
+ UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
115
+ (SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
116
+ j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE);
117
+ END IF;
118
+ END IF;
119
+ END IF;
120
+ RETURN OLD;
121
+ END;
122
+ $$ LANGUAGE plpgsql;
123
+ SQL
124
+ end
125
+ end
126
+ end
@@ -17,6 +17,8 @@ module Delayed
17
17
  include Delayed::Backend::Base
18
18
  self.table_name = :delayed_jobs
19
19
 
20
+ scope :next_in_strand_order, -> { order(:strand_order_override, :id) }
21
+
20
22
  def self.reconnect!
21
23
  clear_all_connections!
22
24
  end
@@ -398,7 +400,7 @@ module Delayed
398
400
  strand = options[:strand]
399
401
  on_conflict = options.delete(:on_conflict) || :use_earliest
400
402
  transaction_for_singleton(strand, on_conflict) do
401
- job = self.where(:strand => strand, :locked_at => nil).order(:id).first
403
+ job = self.where(:strand => strand, :locked_at => nil).next_in_strand_order.first
402
404
  new_job = new(options)
403
405
  if job
404
406
  new_job.initialize_defaults
@@ -1,3 +1,3 @@
1
1
  module Delayed
2
- VERSION = "0.15.22"
2
+ VERSION = "0.16.0"
3
3
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: inst-jobs
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.15.22
4
+ version: 0.16.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Tobias Luetke
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: exe
11
11
  cert_chain: []
12
- date: 2020-07-09 00:00:00.000000000 Z
12
+ date: 2020-08-25 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: activerecord
@@ -343,6 +343,8 @@ files:
343
343
  - db/migrate/20181217155351_speed_up_max_concurrent_triggers.rb
344
344
  - db/migrate/20190726154743_make_critical_columns_not_null.rb
345
345
  - db/migrate/20200330230722_add_id_to_get_delayed_jobs_index.rb
346
+ - db/migrate/20200824222232_speed_up_max_concurrent_delete_trigger.rb
347
+ - db/migrate/20200825011002_add_strand_order_override.rb
346
348
  - exe/inst_jobs
347
349
  - lib/delayed/backend/active_record.rb
348
350
  - lib/delayed/backend/base.rb