postjob 0.5.5 → 0.5.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/lib/postjob/cli/cron.rb +16 -0
- data/lib/postjob/cli/enqueue.rb +14 -0
- data/lib/postjob/cli/job.rb +15 -19
- data/lib/postjob/cli/ps.rb +8 -2
- data/lib/postjob/host.rb +41 -0
- data/lib/postjob/job.rb +23 -23
- data/lib/postjob/migrations/006_enqueue.sql +48 -4
- data/lib/postjob/migrations/006a_processing.sql +5 -2
- data/lib/postjob/migrations/007_job_results.sql +6 -5
- data/lib/postjob/migrations/008a_childjobs.sql +33 -38
- data/lib/postjob/migrations/010_settings.sql +1 -1
- data/lib/postjob/migrations/{008_checkout_runnable.sql → 013a_checkout_runnable.sql} +11 -7
- data/lib/postjob/migrations/017_zombie_check.sql +11 -6
- data/lib/postjob/migrations/021_cron_jobs.sql +65 -0
- data/lib/postjob/migrations/023_sticky_jobs.sql +16 -0
- data/lib/postjob/migrations.rb +3 -1
- data/lib/postjob/queue.rb +23 -4
- data/lib/postjob/record.rb +4 -10
- data/lib/postjob/registry.rb +117 -20
- data/lib/postjob/runner.rb +169 -166
- data/lib/postjob/worker_session.rb +9 -25
- data/lib/postjob/workflow.rb +26 -50
- data/lib/postjob.rb +97 -23
- data/lib/tools/heartbeat.rb +2 -2
- data/lib/tools/history.rb +1 -1
- data/spec/postjob/enqueue_spec.rb +3 -0
- data/spec/postjob/events/heartbeat_event_spec.rb +5 -67
- data/spec/postjob/events/zombie_event_spec.rb +61 -0
- data/spec/postjob/worker_session_spec.rb +1 -24
- data/spec/spec_helper.rb +1 -1
- data/spec/support/configure_active_record.rb +7 -16
- data/spec/support/configure_database.rb +11 -0
- data/spec/support/configure_simple_sql.rb +0 -16
- metadata +10 -5
- data/lib/tools/atomic_store.rb +0 -17
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 574e3a114fa762835314b2e72a2d12b73840569661d9ff8ab06f1ba08d95fb92
|
4
|
+
data.tar.gz: e922e1f1116423b4895a4725107f0456260fa6382edcb308e220fdf62622b528
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 9d7a61a2298c1fc15efc9ce424111d1baf102326b4f73a8314b3e42085a3929443390df44f8e721ef9bac0d4d2273d57a21c8252a4e30a140c6c523ee3766939
|
7
|
+
data.tar.gz: 782b95ae3ab74ad021772b7eb4f3886b8c92835ef3484fa34f1c4ba84792db80e5448273a8318661004d4a36229f554674b87ee00e453882388a27a72d7403be
|
@@ -0,0 +1,16 @@
|
|
1
|
+
module Postjob::CLI
|
2
|
+
# Enqueues a cron workflow
|
3
|
+
def cron_enqueue(workflow, *args, interval:, queue: "ruby", tags: nil)
|
4
|
+
interval = Integer(interval)
|
5
|
+
|
6
|
+
connect_to_database!
|
7
|
+
|
8
|
+
Postjob.enqueue! workflow, *args, queue: queue, tags: parse_tags(tags), cron_interval: interval
|
9
|
+
end
|
10
|
+
|
11
|
+
# Disable an existing cron job
|
12
|
+
def cron_disable(workflow, *args)
|
13
|
+
connect_to_database!
|
14
|
+
Postjob::Queue.disable_cron_jobs(workflow, args)
|
15
|
+
end
|
16
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
module Postjob::CLI
|
2
|
+
# Enqueues a workflow
|
3
|
+
#
|
4
|
+
# Adds a workflow to the job table, with name <workflow> and the given
|
5
|
+
# arguments.
|
6
|
+
#
|
7
|
+
# Note that the workflow will receive the arguments as strings and must be
|
8
|
+
# prepared to handle these.
|
9
|
+
def enqueue(workflow, *args, queue: "ruby", tags: nil)
|
10
|
+
connect_to_database!
|
11
|
+
|
12
|
+
Postjob.enqueue! workflow, *args, queue: queue, tags: parse_tags(tags)
|
13
|
+
end
|
14
|
+
end
|
data/lib/postjob/cli/job.rb
CHANGED
@@ -5,23 +5,9 @@ module Postjob::CLI
|
|
5
5
|
Postjob.resolve(token: token, result: result)
|
6
6
|
end
|
7
7
|
|
8
|
-
# # Fail a job by token
|
9
|
-
# def job_resolve(token, result)
|
10
|
-
# connect_to_database!
|
11
|
-
# Postjob.resolve(token: token, result: result)
|
12
|
-
# end
|
13
|
-
|
14
|
-
# Enqueues a workflow
|
15
|
-
#
|
16
|
-
# Adds a workflow to the job table, with name <workflow> and the given
|
17
|
-
# arguments.
|
18
|
-
#
|
19
|
-
# Note that the workflow will receive the arguments as strings and must be
|
20
|
-
# prepared to handle these.
|
21
8
|
def job_enqueue(workflow, *args, queue: "ruby", tags: nil)
|
22
|
-
|
23
|
-
|
24
|
-
Postjob.enqueue! workflow, *args, queue: queue, tags: parse_tags(tags)
|
9
|
+
logger "The job:enqueue command is deprecated, pls use enqueue instead."
|
10
|
+
enqueue(workflow, *args, queue: queue, tags: tags)
|
25
11
|
end
|
26
12
|
|
27
13
|
# Reset failed jobs
|
@@ -101,8 +87,18 @@ module Postjob::CLI
|
|
101
87
|
public
|
102
88
|
|
103
89
|
def registry
|
104
|
-
|
105
|
-
|
106
|
-
|
90
|
+
workflows_with_versions = Postjob::Registry.workflows.keys.reject { |k| k[1] == "" }
|
91
|
+
workflows_with_versions = workflows_with_versions.sort_by { |name, _version| name }
|
92
|
+
|
93
|
+
data = workflows_with_versions.map do |name, version|
|
94
|
+
spec = Postjob::Registry.lookup! name: name, version: version
|
95
|
+
{
|
96
|
+
name: name,
|
97
|
+
version: version.inspect,
|
98
|
+
options: spec.options.map { |k, v| "#{k}: #{v.inspect}" }.join(", ")
|
99
|
+
}
|
100
|
+
end
|
101
|
+
|
102
|
+
tp data
|
107
103
|
end
|
108
104
|
end
|
data/lib/postjob/cli/ps.rb
CHANGED
@@ -24,9 +24,15 @@ module Postjob::CLI
|
|
24
24
|
END AS status,
|
25
25
|
error,
|
26
26
|
COALESCE((results->0)::varchar, error_message) AS result,
|
27
|
+
max_attempts,
|
28
|
+
cron_interval AS cron,
|
29
|
+
CASE
|
30
|
+
WHEN is_sticky THEN COALESCE(sticky_host_id::varchar, 'yes')
|
31
|
+
ELSE 'no'
|
32
|
+
END AS sticky,
|
27
33
|
next_run_at,
|
28
|
-
next_run_at -
|
29
|
-
to_char(EXTRACT(EPOCH FROM (now() at time zone 'utc'
|
34
|
+
to_char(EXTRACT(EPOCH FROM (next_run_at - now() at time zone 'utc')), '999999999.99') AS next_run_in,
|
35
|
+
to_char(EXTRACT(EPOCH FROM (now() at time zone 'utc' - postjobs.created_at)), '999999999.99') AS age,
|
30
36
|
|
31
37
|
tags
|
32
38
|
FROM postjob.postjobs AS postjobs
|
data/lib/postjob/host.rb
ADDED
@@ -0,0 +1,41 @@
|
|
1
|
+
require_relative "./record"
|
2
|
+
|
3
|
+
class Postjob::Host < Postjob::Record
|
4
|
+
class << self
|
5
|
+
def clear_storage
|
6
|
+
File.unlink(storage_path) if File.exist?(storage_path)
|
7
|
+
end
|
8
|
+
|
9
|
+
def host_id
|
10
|
+
@host_id ||= atomic_set_and_get(storage_path) { register_host }
|
11
|
+
end
|
12
|
+
|
13
|
+
private
|
14
|
+
|
15
|
+
def storage_path
|
16
|
+
env = ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
|
17
|
+
".postjob.#{env}.host_id"
|
18
|
+
end
|
19
|
+
|
20
|
+
def atomic_set_and_get(path)
|
21
|
+
value = nil
|
22
|
+
File.open(path, File::RDWR | File::CREAT, 0644) do |f|
|
23
|
+
f.flock(File::LOCK_EX)
|
24
|
+
|
25
|
+
value = f.read
|
26
|
+
value = yield if value == "" || value.nil?
|
27
|
+
f.rewind
|
28
|
+
f.write(value)
|
29
|
+
f.flush
|
30
|
+
f.truncate(f.pos)
|
31
|
+
end
|
32
|
+
value
|
33
|
+
end
|
34
|
+
|
35
|
+
def register_host
|
36
|
+
attributes = {}
|
37
|
+
Postjob.logger.debug "registering host w/#{attributes.inspect}"
|
38
|
+
::Postjob::Queue.host_register(attributes)
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
data/lib/postjob/job.rb
CHANGED
@@ -9,29 +9,29 @@ class Postjob::Job < Postjob::Record
|
|
9
9
|
Simple::SQL.ask(scope, into: Postjob::Job)
|
10
10
|
end
|
11
11
|
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
12
|
+
attr_reader :id
|
13
|
+
attr_reader :parent_id
|
14
|
+
attr_reader :full_id
|
15
|
+
attr_reader :root_id
|
16
|
+
attr_reader :created_at
|
17
|
+
attr_reader :queue
|
18
|
+
attr_reader :workflow
|
19
|
+
attr_reader :workflow_method
|
20
|
+
attr_reader :workflow_version
|
21
|
+
attr_reader :args
|
22
|
+
attr_reader :next_run_at
|
23
|
+
attr_reader :timing_out_at
|
24
|
+
attr_reader :failed_attempts
|
25
|
+
attr_reader :max_attempts
|
26
|
+
attr_reader :status
|
27
|
+
attr_reader :results
|
28
|
+
attr_reader :error
|
29
|
+
attr_reader :error_message
|
30
|
+
attr_reader :error_backtrace
|
31
|
+
attr_reader :recipients
|
32
|
+
attr_reader :timed_out
|
33
|
+
attr_reader :tags
|
34
|
+
attr_reader :last_worker_session_id
|
35
35
|
|
36
36
|
STATUSES = %w(ok ready processing sleep err failed timeout)
|
37
37
|
|
@@ -1,3 +1,28 @@
|
|
1
|
+
DROP FUNCTION IF EXISTS {SCHEMA_NAME}.enqueue(
|
2
|
+
p_worker_session_id UUID,
|
3
|
+
queue VARCHAR,
|
4
|
+
workflow VARCHAR,
|
5
|
+
workflow_method VARCHAR,
|
6
|
+
workflow_version VARCHAR,
|
7
|
+
args JSONB,
|
8
|
+
parent_id BIGINT,
|
9
|
+
tags JSONB,
|
10
|
+
max_attempts INTEGER,
|
11
|
+
timeout DOUBLE PRECISION);
|
12
|
+
|
13
|
+
DROP FUNCTION IF EXISTS {SCHEMA_NAME}.enqueue(
|
14
|
+
p_worker_session_id UUID,
|
15
|
+
queue VARCHAR,
|
16
|
+
workflow VARCHAR,
|
17
|
+
workflow_method VARCHAR,
|
18
|
+
workflow_version VARCHAR,
|
19
|
+
args JSONB,
|
20
|
+
parent_id BIGINT,
|
21
|
+
tags JSONB,
|
22
|
+
max_attempts INTEGER,
|
23
|
+
timeout DOUBLE PRECISION,
|
24
|
+
p_cron_interval integer);
|
25
|
+
|
1
26
|
CREATE OR REPLACE FUNCTION {SCHEMA_NAME}.enqueue(
|
2
27
|
p_worker_session_id UUID,
|
3
28
|
queue VARCHAR,
|
@@ -8,34 +33,53 @@ CREATE OR REPLACE FUNCTION {SCHEMA_NAME}.enqueue(
|
|
8
33
|
parent_id BIGINT,
|
9
34
|
tags JSONB,
|
10
35
|
max_attempts INTEGER,
|
11
|
-
timeout DOUBLE PRECISION
|
36
|
+
timeout DOUBLE PRECISION,
|
37
|
+
p_cron_interval INTEGER,
|
38
|
+
p_is_sticky BOOLEAN)
|
12
39
|
RETURNS SETOF {SCHEMA_NAME}.postjobs
|
13
40
|
AS $$
|
14
41
|
DECLARE
|
15
42
|
job_id BIGINT;
|
16
43
|
BEGIN
|
17
|
-
--
|
44
|
+
-- check arguments --------------------------------------------------------
|
45
|
+
|
18
46
|
IF workflow = '__manual__' AND COALESCE(max_attempts, 1) != 1 THEN
|
19
47
|
RAISE NOTICE 'Adjusting max_attempts of __manual__ job to 1';
|
20
48
|
max_attempts := 1;
|
21
49
|
END IF;
|
22
50
|
|
51
|
+
IF p_cron_interval IS NOT NULL THEN
|
52
|
+
IF parent_id IS NOT NULL THEN
|
53
|
+
RAISE 'A cron job must be a root job';
|
54
|
+
END IF;
|
55
|
+
IF p_cron_interval < 1 THEN
|
56
|
+
RAISE 'The cron_interval must be positive';
|
57
|
+
END IF;
|
58
|
+
END IF;
|
59
|
+
|
60
|
+
-- check arguments --------------------------------------------------------
|
61
|
+
|
23
62
|
workflow_version := COALESCE(workflow_version, '');
|
24
63
|
queue := COALESCE(queue, 'q');
|
25
64
|
max_attempts := COALESCE(max_attempts, 5);
|
65
|
+
p_is_sticky := COALESCE(p_is_sticky, FALSE);
|
26
66
|
|
27
67
|
-- create postjobs entry ------------------------------------------
|
28
68
|
INSERT INTO {SCHEMA_NAME}.postjobs (
|
29
69
|
last_worker_session_id,
|
30
70
|
queue, workflow, workflow_method, workflow_version, args,
|
31
71
|
parent_id, tags, max_attempts,
|
32
|
-
timing_out_at
|
72
|
+
timing_out_at,
|
73
|
+
cron_interval,
|
74
|
+
is_sticky
|
33
75
|
)
|
34
76
|
VALUES(
|
35
77
|
p_worker_session_id,
|
36
78
|
queue, workflow, workflow_method, workflow_version, args,
|
37
79
|
parent_id, tags, max_attempts,
|
38
|
-
(now() at time zone 'utc') + timeout * interval '1 second'
|
80
|
+
(now() at time zone 'utc') + timeout * interval '1 second',
|
81
|
+
p_cron_interval,
|
82
|
+
p_is_sticky
|
39
83
|
) RETURNING {SCHEMA_NAME}.postjobs.id INTO job_id;
|
40
84
|
|
41
85
|
-- fill in root_id and full_id ------------------------------------
|
@@ -2,8 +2,10 @@ DROP FUNCTION IF EXISTS {SCHEMA_NAME}._set_job_processing(job_id BIGINT); -- re
|
|
2
2
|
CREATE OR REPLACE FUNCTION {SCHEMA_NAME}._set_job_processing(worker_session_id UUID, job_id BIGINT) RETURNS VOID AS $$
|
3
3
|
DECLARE
|
4
4
|
v_pid int;
|
5
|
+
v_host_id uuid;
|
5
6
|
BEGIN
|
6
7
|
v_pid := pg_backend_pid();
|
8
|
+
SELECT host_id INTO v_host_id FROM {SCHEMA_NAME}.worker_sessions WHERE id=worker_session_id;
|
7
9
|
|
8
10
|
UPDATE {SCHEMA_NAME}.postjobs
|
9
11
|
SET
|
@@ -12,8 +14,9 @@ BEGIN
|
|
12
14
|
error=NULL,
|
13
15
|
error_message=NULL,
|
14
16
|
error_backtrace=NULL,
|
15
|
-
next_run_at=NULL
|
16
|
-
|
17
|
+
next_run_at=NULL,
|
18
|
+
sticky_host_id=(CASE WHEN is_sticky THEN v_host_id ELSE NULL END)
|
19
|
+
WHERE id=job_id;
|
17
20
|
END;
|
18
21
|
$$ LANGUAGE plpgsql;
|
19
22
|
|
@@ -45,10 +45,12 @@ BEGIN
|
|
45
45
|
END;
|
46
46
|
$$ LANGUAGE plpgsql;
|
47
47
|
|
48
|
+
DROP FUNCTION IF EXISTS {SCHEMA_NAME}._prepare_next_run(BIGINT, {SCHEMA_NAME}.statuses, BOOLEAN);
|
49
|
+
|
48
50
|
-- If this is a recoverable error and if we have another run possible
|
49
51
|
-- we'll set next_run_at, and the status to "err", otherwise
|
50
52
|
-- next_run_at will be NULL and the status would be "failed" or "timeout"
|
51
|
-
CREATE OR REPLACE FUNCTION {SCHEMA_NAME}.
|
53
|
+
CREATE OR REPLACE FUNCTION {SCHEMA_NAME}._prepare_rerun(
|
52
54
|
job_id BIGINT,
|
53
55
|
p_status {SCHEMA_NAME}.statuses,
|
54
56
|
p_fast_mode BOOLEAN) RETURNS VOID AS $$
|
@@ -119,8 +121,7 @@ BEGIN
|
|
119
121
|
next_run_at=NULL
|
120
122
|
WHERE id=job_id;
|
121
123
|
|
122
|
-
|
123
|
-
PERFORM {SCHEMA_NAME}._prepare_next_run(job_id, p_status, p_fast_mode);
|
124
|
+
PERFORM {SCHEMA_NAME}._prepare_rerun(job_id, p_status, p_fast_mode);
|
124
125
|
PERFORM {SCHEMA_NAME}._wakeup_parent_job(p_worker_session_id, job_id);
|
125
126
|
END;
|
126
127
|
$$ LANGUAGE plpgsql;
|
@@ -144,7 +145,7 @@ BEGIN
|
|
144
145
|
WHERE id=job_id;
|
145
146
|
|
146
147
|
-- prepare next run, if any
|
147
|
-
PERFORM {SCHEMA_NAME}.
|
148
|
+
PERFORM {SCHEMA_NAME}._prepare_rerun(job_id, 'timeout', p_fast_mode);
|
148
149
|
PERFORM {SCHEMA_NAME}._wakeup_parent_job(p_worker_session_id, job_id);
|
149
150
|
END;
|
150
151
|
$$ LANGUAGE plpgsql;
|
@@ -170,7 +171,7 @@ BEGIN
|
|
170
171
|
WHERE id=job_id;
|
171
172
|
|
172
173
|
-- prepare next run, if any
|
173
|
-
PERFORM {SCHEMA_NAME}.
|
174
|
+
PERFORM {SCHEMA_NAME}._prepare_rerun(job_id, 'err', p_fast_mode);
|
174
175
|
PERFORM {SCHEMA_NAME}._wakeup_parent_job(p_worker_session_id, job_id);
|
175
176
|
END;
|
176
177
|
$$ LANGUAGE plpgsql;
|
@@ -49,47 +49,42 @@ BEGIN
|
|
49
49
|
RAISE 'Invalid parent job id NULL';
|
50
50
|
END IF;
|
51
51
|
|
52
|
-
|
53
|
-
SELECT INTO parent * FROM {SCHEMA_NAME}.postjobs WHERE id=v_parent_id;
|
54
|
-
IF parent.id IS NULL THEN
|
55
|
-
RAISE 'No such job: %', v_parent_id;
|
56
|
-
END IF;
|
57
|
-
END IF;
|
52
|
+
-- create a child job if it doesn't exist yet.
|
58
53
|
|
59
|
-
|
54
|
+
SELECT id INTO child_id FROM {SCHEMA_NAME}.postjobs
|
55
|
+
WHERE parent_id=v_parent_id
|
56
|
+
AND workflow=v_workflow
|
57
|
+
AND workflow_method=v_workflow_method
|
58
|
+
AND args=v_args
|
59
|
+
;
|
60
60
|
|
61
|
-
|
62
|
-
SELECT
|
63
|
-
|
64
|
-
|
65
|
-
AND workflow_method=v_workflow_method
|
66
|
-
AND args=v_args
|
67
|
-
;
|
61
|
+
IF child_id IS NULL THEN
|
62
|
+
SELECT * INTO parent
|
63
|
+
FROM {SCHEMA_NAME}.postjobs
|
64
|
+
WHERE postjobs.id=v_parent_id;
|
68
65
|
|
69
|
-
|
70
|
-
|
71
|
-
--
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
66
|
+
SELECT id INTO child_id FROM {SCHEMA_NAME}.enqueue(
|
67
|
+
p_worker_session_id,
|
68
|
+
COALESCE(v_queue, parent.queue), -- queue VARCHAR,
|
69
|
+
v_workflow, -- workflow VARCHAR,
|
70
|
+
v_workflow_method, -- workflow_method VARCHAR,
|
71
|
+
NULL, -- workflow_version VARCHAR,
|
72
|
+
v_args, -- args JSONB,
|
73
|
+
v_parent_id, -- parent_id BIGINT,
|
74
|
+
parent.tags, -- tags JSONB,
|
75
|
+
v_max_attempts, -- max_attempts INTEGER,
|
76
|
+
v_timeout, -- timeout
|
77
|
+
NULL, -- cron_interval,
|
78
|
+
parent.is_sticky -- is_sticky?
|
79
|
+
);
|
79
80
|
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
v_parent_id, -- parent_id BIGINT,
|
89
|
-
parent.tags, -- tags JSONB,
|
90
|
-
v_max_attempts, -- max_attempts INTEGER,
|
91
|
-
v_timeout);
|
92
|
-
END IF;
|
93
|
-
-- END IF;
|
81
|
+
UPDATE {SCHEMA_NAME}.postjobs
|
82
|
+
SET sticky_host_id=parent.sticky_host_id
|
83
|
+
WHERE id=child_id;
|
84
|
+
END IF;
|
85
|
+
|
86
|
+
RETURN QUERY
|
87
|
+
SELECT * FROM {SCHEMA_NAME}.postjobs WHERE id=child_id
|
88
|
+
;
|
94
89
|
END;
|
95
90
|
$$ LANGUAGE plpgsql;
|
@@ -27,5 +27,5 @@ $$ LANGUAGE plpgsql;
|
|
27
27
|
|
28
28
|
-- define version settings ----------------------------------------------------
|
29
29
|
|
30
|
-
SELECT {SCHEMA_NAME}.settings_set('version', '0.5.
|
30
|
+
SELECT {SCHEMA_NAME}.settings_set('version', '0.5.6');
|
31
31
|
SELECT {SCHEMA_NAME}.settings_set('client_version', '{CLIENT_VERSION}');
|
@@ -4,9 +4,10 @@ CREATE OR REPLACE FUNCTION {SCHEMA_NAME}.time_to_next_job(p_worker_session_id UU
|
|
4
4
|
AS $$
|
5
5
|
DECLARE
|
6
6
|
p_processable_at timestamp;
|
7
|
-
|
7
|
+
session {SCHEMA_NAME}.worker_sessions;
|
8
8
|
BEGIN
|
9
|
-
SELECT
|
9
|
+
SELECT * INTO session
|
10
|
+
FROM {SCHEMA_NAME}.worker_sessions WHERE id=p_worker_session_id;
|
10
11
|
|
11
12
|
SELECT MIN(processable_at) INTO p_processable_at FROM (
|
12
13
|
SELECT MIN(timing_out_at) AS processable_at
|
@@ -16,10 +17,11 @@ BEGIN
|
|
16
17
|
SELECT MIN(next_run_at) AS processable_at
|
17
18
|
FROM {SCHEMA_NAME}.postjobs
|
18
19
|
WHERE status IN ('ready', 'err')
|
19
|
-
AND workflow || workflow_version = ANY (
|
20
|
+
AND (workflow || workflow_version) = ANY (session.workflows)
|
21
|
+
AND COALESCE(sticky_host_id, {SCHEMA_NAME}._null_uuid()) IN (session.host_id, {SCHEMA_NAME}._null_uuid())
|
20
22
|
) sq;
|
21
23
|
|
22
|
-
RETURN EXTRACT(EPOCH FROM
|
24
|
+
RETURN EXTRACT(EPOCH FROM p_processable_at - (now() at time zone 'utc'));
|
23
25
|
END;
|
24
26
|
$$ LANGUAGE plpgsql;
|
25
27
|
|
@@ -29,9 +31,10 @@ CREATE OR REPLACE FUNCTION {SCHEMA_NAME}.checkout(p_worker_session_id UUID, p_fa
|
|
29
31
|
AS $$
|
30
32
|
DECLARE
|
31
33
|
job {SCHEMA_NAME}.postjobs;
|
32
|
-
|
34
|
+
session {SCHEMA_NAME}.worker_sessions;
|
33
35
|
BEGIN
|
34
|
-
SELECT
|
36
|
+
SELECT * INTO session
|
37
|
+
FROM {SCHEMA_NAME}.worker_sessions WHERE id=p_worker_session_id;
|
35
38
|
|
36
39
|
LOOP
|
37
40
|
-- try to checkout a job. Each of the conditions here is matching
|
@@ -47,7 +50,8 @@ BEGIN
|
|
47
50
|
(
|
48
51
|
s.status IN ('ready', 'err')
|
49
52
|
AND s.next_run_at <= (now() at time zone 'utc')
|
50
|
-
AND (s.workflow || s.workflow_version) = ANY (
|
53
|
+
AND (s.workflow || s.workflow_version) = ANY (session.workflows)
|
54
|
+
AND COALESCE(s.sticky_host_id, {SCHEMA_NAME}._null_uuid()) IN (session.host_id, {SCHEMA_NAME}._null_uuid())
|
51
55
|
)
|
52
56
|
ORDER BY (LEAST(s.next_run_at, s.timing_out_at))
|
53
57
|
FOR UPDATE SKIP LOCKED
|
@@ -15,6 +15,12 @@ DECLARE
|
|
15
15
|
BEGIN
|
16
16
|
zombie_count := 0;
|
17
17
|
FOR zombie_id, _one IN
|
18
|
+
-- select jobs that have a last_worker_session_id, which points to a
|
19
|
+
-- host whose latest heartbeat is older than +zombie_threshold+.
|
20
|
+
--
|
21
|
+
-- note that we ignore hosts (and, for that matter, jobs) that don't
|
22
|
+
-- have any heartbeats, since this scenario should only appear during
|
23
|
+
-- tests.
|
18
24
|
SELECT jobs.id, 1
|
19
25
|
FROM {SCHEMA_NAME}.postjobs jobs
|
20
26
|
LEFT JOIN {SCHEMA_NAME}.worker_sessions sessions ON jobs.last_worker_session_id=sessions.id
|
@@ -27,10 +33,7 @@ BEGIN
|
|
27
33
|
) heartbeat ON sessions.host_id = heartbeat.host_id
|
28
34
|
WHERE
|
29
35
|
jobs.status IN ('processing')
|
30
|
-
AND (
|
31
|
-
heartbeat.created_at IS NULL OR
|
32
|
-
heartbeat.created_at < ((now() at time zone 'utc') - zombie_threshold)
|
33
|
-
)
|
36
|
+
AND heartbeat.created_at < ((now() at time zone 'utc') - zombie_threshold)
|
34
37
|
LOOP
|
35
38
|
PERFORM {SCHEMA_NAME}._set_job_zombie(zombie_id, p_fast_mode);
|
36
39
|
zombie_count := zombie_count + 1;
|
@@ -51,8 +54,10 @@ BEGIN
|
|
51
54
|
-- event, which has a zombie count value in its attributes.
|
52
55
|
IF NOT EXISTS (SELECT 1 FROM {SCHEMA_NAME}.events WHERE name='zombie' AND created_at > (now() at time zone 'utc') - zombie_check_interval) THEN
|
53
56
|
p_zombie_count := {SCHEMA_NAME}._zombie_check(p_fast_mode);
|
54
|
-
|
55
|
-
|
57
|
+
IF p_zombie_count > 0 THEN
|
58
|
+
INSERT INTO {SCHEMA_NAME}.events(name, host_id, attributes)
|
59
|
+
VALUES('zombie', {SCHEMA_NAME}._null_uuid(), jsonb_build_object('zombie_count', p_zombie_count));
|
60
|
+
END IF;
|
56
61
|
END IF;
|
57
62
|
END;
|
58
63
|
$$ LANGUAGE plpgsql;
|
@@ -0,0 +1,65 @@
|
|
1
|
+
DO $$
|
2
|
+
BEGIN
|
3
|
+
ALTER TABLE {SCHEMA_NAME}.postjobs ADD COLUMN cron_interval integer;
|
4
|
+
EXCEPTION
|
5
|
+
WHEN duplicate_column THEN RAISE DEBUG 'column {SCHEMA_NAME}.postjobs.cron_interval already exists';
|
6
|
+
END;
|
7
|
+
$$;
|
8
|
+
|
9
|
+
--- define triggers to automatically restart cron jobs ------------------------
|
10
|
+
|
11
|
+
-- This method is called whenever a job's status changes. It enqueues a fresh
|
12
|
+
|
13
|
+
--
|
14
|
+
-- An event is created whenever a job's status changes.
|
15
|
+
--
|
16
|
+
|
17
|
+
CREATE OR REPLACE FUNCTION {SCHEMA_NAME}._restart_cron_job()
|
18
|
+
RETURNS TRIGGER
|
19
|
+
AS $$
|
20
|
+
DECLARE
|
21
|
+
p_new_jonb_id BIGINT;
|
22
|
+
BEGIN
|
23
|
+
IF NEW.status = OLD.status THEN
|
24
|
+
RETURN NEW;
|
25
|
+
END IF;
|
26
|
+
|
27
|
+
IF TG_OP != 'UPDATE' OR NEW.status NOT IN ('failed', 'ok') THEN
|
28
|
+
RETURN NEW;
|
29
|
+
END IF;
|
30
|
+
|
31
|
+
IF NEW.cron_interval IS NULL THEN
|
32
|
+
RETURN NEW;
|
33
|
+
END IF;
|
34
|
+
|
35
|
+
SELECT id INTO p_new_jonb_id FROM {SCHEMA_NAME}.enqueue(
|
36
|
+
NEW.last_worker_session_id, -- p_worker_session_id
|
37
|
+
NEW.queue, -- queue
|
38
|
+
NEW.workflow, -- workflow
|
39
|
+
NEW.workflow_method, -- workflow_method
|
40
|
+
NULL, -- workflow_version
|
41
|
+
NEW.args, -- args
|
42
|
+
NULL, -- parent_id
|
43
|
+
NEW.tags, -- tags
|
44
|
+
NEW.max_attempts, -- max_attempts
|
45
|
+
(EXTRACT(EPOCH FROM NEW.timing_out_at) - EXTRACT(EPOCH FROM NEW.created_at)), -- timeout
|
46
|
+
NEW.cron_interval,
|
47
|
+
NEW.is_sticky
|
48
|
+
);
|
49
|
+
|
50
|
+
UPDATE {SCHEMA_NAME}.postjobs
|
51
|
+
SET next_run_at = now() at time zone 'utc' + NEW.cron_interval * interval '1 second'
|
52
|
+
WHERE id=p_new_jonb_id;
|
53
|
+
|
54
|
+
RETURN NEW;
|
55
|
+
END;
|
56
|
+
$$ LANGUAGE plpgsql;
|
57
|
+
|
58
|
+
BEGIN;
|
59
|
+
DROP TRIGGER IF EXISTS _restart_cron_job ON {SCHEMA_NAME}.postjobs;
|
60
|
+
|
61
|
+
CREATE TRIGGER _restart_cron_job AFTER UPDATE
|
62
|
+
ON {SCHEMA_NAME}.postjobs
|
63
|
+
FOR EACH ROW
|
64
|
+
EXECUTE PROCEDURE {SCHEMA_NAME}._restart_cron_job();
|
65
|
+
COMMIT;
|
@@ -0,0 +1,16 @@
|
|
1
|
+
DO $$
|
2
|
+
BEGIN
|
3
|
+
ALTER TABLE {SCHEMA_NAME}.postjobs ADD COLUMN is_sticky BOOLEAN NOT NULL DEFAULT FALSE;
|
4
|
+
EXCEPTION
|
5
|
+
WHEN duplicate_column THEN RAISE DEBUG 'column {SCHEMA_NAME}.postjobs.is_sticky already exists';
|
6
|
+
END;
|
7
|
+
$$;
|
8
|
+
|
9
|
+
DO $$
|
10
|
+
BEGIN
|
11
|
+
ALTER TABLE {SCHEMA_NAME}.postjobs ADD COLUMN sticky_host_id UUID
|
12
|
+
REFERENCES {SCHEMA_NAME}.hosts;
|
13
|
+
EXCEPTION
|
14
|
+
WHEN duplicate_column THEN RAISE DEBUG 'column {SCHEMA_NAME}.postjobs.sticky_host_id already exists';
|
15
|
+
END;
|
16
|
+
$$;
|
data/lib/postjob/migrations.rb
CHANGED
@@ -16,6 +16,8 @@ module Postjob
|
|
16
16
|
expect! SCHEMA_NAME != "public"
|
17
17
|
|
18
18
|
def unmigrate!
|
19
|
+
::Postjob::Host.clear_storage
|
20
|
+
|
19
21
|
SQL.exec "SET client_min_messages TO WARNING"
|
20
22
|
SQL.exec "DROP SCHEMA IF EXISTS #{SCHEMA_NAME} CASCADE"
|
21
23
|
end
|
@@ -45,7 +47,7 @@ module Postjob
|
|
45
47
|
|
46
48
|
def run_migration_sql(file)
|
47
49
|
sql = File.read(file)
|
48
|
-
sql.gsub!(/\{([^\}]+)\}/) { |_| const_get(
|
50
|
+
sql.gsub!(/\{([^\}]+)\}/) { |_| const_get(Regexp.last_match(1)) }
|
49
51
|
SQL.exec sql
|
50
52
|
end
|
51
53
|
end
|