postjob 0.5.5 → 0.5.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/lib/postjob/cli/cron.rb +16 -0
- data/lib/postjob/cli/enqueue.rb +14 -0
- data/lib/postjob/cli/job.rb +15 -19
- data/lib/postjob/cli/ps.rb +8 -2
- data/lib/postjob/host.rb +41 -0
- data/lib/postjob/job.rb +23 -23
- data/lib/postjob/migrations/006_enqueue.sql +48 -4
- data/lib/postjob/migrations/006a_processing.sql +5 -2
- data/lib/postjob/migrations/007_job_results.sql +6 -5
- data/lib/postjob/migrations/008a_childjobs.sql +33 -38
- data/lib/postjob/migrations/010_settings.sql +1 -1
- data/lib/postjob/migrations/{008_checkout_runnable.sql → 013a_checkout_runnable.sql} +11 -7
- data/lib/postjob/migrations/017_zombie_check.sql +11 -6
- data/lib/postjob/migrations/021_cron_jobs.sql +65 -0
- data/lib/postjob/migrations/023_sticky_jobs.sql +16 -0
- data/lib/postjob/migrations.rb +3 -1
- data/lib/postjob/queue.rb +23 -4
- data/lib/postjob/record.rb +4 -10
- data/lib/postjob/registry.rb +117 -20
- data/lib/postjob/runner.rb +169 -166
- data/lib/postjob/worker_session.rb +9 -25
- data/lib/postjob/workflow.rb +26 -50
- data/lib/postjob.rb +97 -23
- data/lib/tools/heartbeat.rb +2 -2
- data/lib/tools/history.rb +1 -1
- data/spec/postjob/enqueue_spec.rb +3 -0
- data/spec/postjob/events/heartbeat_event_spec.rb +5 -67
- data/spec/postjob/events/zombie_event_spec.rb +61 -0
- data/spec/postjob/worker_session_spec.rb +1 -24
- data/spec/spec_helper.rb +1 -1
- data/spec/support/configure_active_record.rb +7 -16
- data/spec/support/configure_database.rb +11 -0
- data/spec/support/configure_simple_sql.rb +0 -16
- metadata +10 -5
- data/lib/tools/atomic_store.rb +0 -17
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 574e3a114fa762835314b2e72a2d12b73840569661d9ff8ab06f1ba08d95fb92
|
4
|
+
data.tar.gz: e922e1f1116423b4895a4725107f0456260fa6382edcb308e220fdf62622b528
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 9d7a61a2298c1fc15efc9ce424111d1baf102326b4f73a8314b3e42085a3929443390df44f8e721ef9bac0d4d2273d57a21c8252a4e30a140c6c523ee3766939
|
7
|
+
data.tar.gz: 782b95ae3ab74ad021772b7eb4f3886b8c92835ef3484fa34f1c4ba84792db80e5448273a8318661004d4a36229f554674b87ee00e453882388a27a72d7403be
|
@@ -0,0 +1,16 @@
|
|
1
|
+
module Postjob::CLI
|
2
|
+
# Enqueues a cron workflow
|
3
|
+
def cron_enqueue(workflow, *args, interval:, queue: "ruby", tags: nil)
|
4
|
+
interval = Integer(interval)
|
5
|
+
|
6
|
+
connect_to_database!
|
7
|
+
|
8
|
+
Postjob.enqueue! workflow, *args, queue: queue, tags: parse_tags(tags), cron_interval: interval
|
9
|
+
end
|
10
|
+
|
11
|
+
# Disable an existing cron job
|
12
|
+
def cron_disable(workflow, *args)
|
13
|
+
connect_to_database!
|
14
|
+
Postjob::Queue.disable_cron_jobs(workflow, args)
|
15
|
+
end
|
16
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
module Postjob::CLI
|
2
|
+
# Enqueues a workflow
|
3
|
+
#
|
4
|
+
# Adds a workflow to the job table, with name <workflow> and the given
|
5
|
+
# arguments.
|
6
|
+
#
|
7
|
+
# Note that the workflow will receive the arguments as strings and must be
|
8
|
+
# prepared to handle these.
|
9
|
+
def enqueue(workflow, *args, queue: "ruby", tags: nil)
|
10
|
+
connect_to_database!
|
11
|
+
|
12
|
+
Postjob.enqueue! workflow, *args, queue: queue, tags: parse_tags(tags)
|
13
|
+
end
|
14
|
+
end
|
data/lib/postjob/cli/job.rb
CHANGED
@@ -5,23 +5,9 @@ module Postjob::CLI
|
|
5
5
|
Postjob.resolve(token: token, result: result)
|
6
6
|
end
|
7
7
|
|
8
|
-
# # Fail a job by token
|
9
|
-
# def job_resolve(token, result)
|
10
|
-
# connect_to_database!
|
11
|
-
# Postjob.resolve(token: token, result: result)
|
12
|
-
# end
|
13
|
-
|
14
|
-
# Enqueues a workflow
|
15
|
-
#
|
16
|
-
# Adds a workflow to the job table, with name <workflow> and the given
|
17
|
-
# arguments.
|
18
|
-
#
|
19
|
-
# Note that the workflow will receive the arguments as strings and must be
|
20
|
-
# prepared to handle these.
|
21
8
|
def job_enqueue(workflow, *args, queue: "ruby", tags: nil)
|
22
|
-
|
23
|
-
|
24
|
-
Postjob.enqueue! workflow, *args, queue: queue, tags: parse_tags(tags)
|
9
|
+
logger "The job:enqueue command is deprecated, pls use enqueue instead."
|
10
|
+
enqueue(workflow, *args, queue: queue, tags: tags)
|
25
11
|
end
|
26
12
|
|
27
13
|
# Reset failed jobs
|
@@ -101,8 +87,18 @@ module Postjob::CLI
|
|
101
87
|
public
|
102
88
|
|
103
89
|
def registry
|
104
|
-
|
105
|
-
|
106
|
-
|
90
|
+
workflows_with_versions = Postjob::Registry.workflows.keys.reject { |k| k[1] == "" }
|
91
|
+
workflows_with_versions = workflows_with_versions.sort_by { |name, _version| name }
|
92
|
+
|
93
|
+
data = workflows_with_versions.map do |name, version|
|
94
|
+
spec = Postjob::Registry.lookup! name: name, version: version
|
95
|
+
{
|
96
|
+
name: name,
|
97
|
+
version: version.inspect,
|
98
|
+
options: spec.options.map { |k, v| "#{k}: #{v.inspect}" }.join(", ")
|
99
|
+
}
|
100
|
+
end
|
101
|
+
|
102
|
+
tp data
|
107
103
|
end
|
108
104
|
end
|
data/lib/postjob/cli/ps.rb
CHANGED
@@ -24,9 +24,15 @@ module Postjob::CLI
|
|
24
24
|
END AS status,
|
25
25
|
error,
|
26
26
|
COALESCE((results->0)::varchar, error_message) AS result,
|
27
|
+
max_attempts,
|
28
|
+
cron_interval AS cron,
|
29
|
+
CASE
|
30
|
+
WHEN is_sticky THEN COALESCE(sticky_host_id::varchar, 'yes')
|
31
|
+
ELSE 'no'
|
32
|
+
END AS sticky,
|
27
33
|
next_run_at,
|
28
|
-
next_run_at -
|
29
|
-
to_char(EXTRACT(EPOCH FROM (now() at time zone 'utc'
|
34
|
+
to_char(EXTRACT(EPOCH FROM (next_run_at - now() at time zone 'utc')), '999999999.99') AS next_run_in,
|
35
|
+
to_char(EXTRACT(EPOCH FROM (now() at time zone 'utc' - postjobs.created_at)), '999999999.99') AS age,
|
30
36
|
|
31
37
|
tags
|
32
38
|
FROM postjob.postjobs AS postjobs
|
data/lib/postjob/host.rb
ADDED
@@ -0,0 +1,41 @@
|
|
1
|
+
require_relative "./record"
|
2
|
+
|
3
|
+
class Postjob::Host < Postjob::Record
|
4
|
+
class << self
|
5
|
+
def clear_storage
|
6
|
+
File.unlink(storage_path) if File.exist?(storage_path)
|
7
|
+
end
|
8
|
+
|
9
|
+
def host_id
|
10
|
+
@host_id ||= atomic_set_and_get(storage_path) { register_host }
|
11
|
+
end
|
12
|
+
|
13
|
+
private
|
14
|
+
|
15
|
+
def storage_path
|
16
|
+
env = ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
|
17
|
+
".postjob.#{env}.host_id"
|
18
|
+
end
|
19
|
+
|
20
|
+
def atomic_set_and_get(path)
|
21
|
+
value = nil
|
22
|
+
File.open(path, File::RDWR | File::CREAT, 0644) do |f|
|
23
|
+
f.flock(File::LOCK_EX)
|
24
|
+
|
25
|
+
value = f.read
|
26
|
+
value = yield if value == "" || value.nil?
|
27
|
+
f.rewind
|
28
|
+
f.write(value)
|
29
|
+
f.flush
|
30
|
+
f.truncate(f.pos)
|
31
|
+
end
|
32
|
+
value
|
33
|
+
end
|
34
|
+
|
35
|
+
def register_host
|
36
|
+
attributes = {}
|
37
|
+
Postjob.logger.debug "registering host w/#{attributes.inspect}"
|
38
|
+
::Postjob::Queue.host_register(attributes)
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
data/lib/postjob/job.rb
CHANGED
@@ -9,29 +9,29 @@ class Postjob::Job < Postjob::Record
|
|
9
9
|
Simple::SQL.ask(scope, into: Postjob::Job)
|
10
10
|
end
|
11
11
|
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
12
|
+
attr_reader :id
|
13
|
+
attr_reader :parent_id
|
14
|
+
attr_reader :full_id
|
15
|
+
attr_reader :root_id
|
16
|
+
attr_reader :created_at
|
17
|
+
attr_reader :queue
|
18
|
+
attr_reader :workflow
|
19
|
+
attr_reader :workflow_method
|
20
|
+
attr_reader :workflow_version
|
21
|
+
attr_reader :args
|
22
|
+
attr_reader :next_run_at
|
23
|
+
attr_reader :timing_out_at
|
24
|
+
attr_reader :failed_attempts
|
25
|
+
attr_reader :max_attempts
|
26
|
+
attr_reader :status
|
27
|
+
attr_reader :results
|
28
|
+
attr_reader :error
|
29
|
+
attr_reader :error_message
|
30
|
+
attr_reader :error_backtrace
|
31
|
+
attr_reader :recipients
|
32
|
+
attr_reader :timed_out
|
33
|
+
attr_reader :tags
|
34
|
+
attr_reader :last_worker_session_id
|
35
35
|
|
36
36
|
STATUSES = %w(ok ready processing sleep err failed timeout)
|
37
37
|
|
@@ -1,3 +1,28 @@
|
|
1
|
+
DROP FUNCTION IF EXISTS {SCHEMA_NAME}.enqueue(
|
2
|
+
p_worker_session_id UUID,
|
3
|
+
queue VARCHAR,
|
4
|
+
workflow VARCHAR,
|
5
|
+
workflow_method VARCHAR,
|
6
|
+
workflow_version VARCHAR,
|
7
|
+
args JSONB,
|
8
|
+
parent_id BIGINT,
|
9
|
+
tags JSONB,
|
10
|
+
max_attempts INTEGER,
|
11
|
+
timeout DOUBLE PRECISION);
|
12
|
+
|
13
|
+
DROP FUNCTION IF EXISTS {SCHEMA_NAME}.enqueue(
|
14
|
+
p_worker_session_id UUID,
|
15
|
+
queue VARCHAR,
|
16
|
+
workflow VARCHAR,
|
17
|
+
workflow_method VARCHAR,
|
18
|
+
workflow_version VARCHAR,
|
19
|
+
args JSONB,
|
20
|
+
parent_id BIGINT,
|
21
|
+
tags JSONB,
|
22
|
+
max_attempts INTEGER,
|
23
|
+
timeout DOUBLE PRECISION,
|
24
|
+
p_cron_interval integer);
|
25
|
+
|
1
26
|
CREATE OR REPLACE FUNCTION {SCHEMA_NAME}.enqueue(
|
2
27
|
p_worker_session_id UUID,
|
3
28
|
queue VARCHAR,
|
@@ -8,34 +33,53 @@ CREATE OR REPLACE FUNCTION {SCHEMA_NAME}.enqueue(
|
|
8
33
|
parent_id BIGINT,
|
9
34
|
tags JSONB,
|
10
35
|
max_attempts INTEGER,
|
11
|
-
timeout DOUBLE PRECISION
|
36
|
+
timeout DOUBLE PRECISION,
|
37
|
+
p_cron_interval INTEGER,
|
38
|
+
p_is_sticky BOOLEAN)
|
12
39
|
RETURNS SETOF {SCHEMA_NAME}.postjobs
|
13
40
|
AS $$
|
14
41
|
DECLARE
|
15
42
|
job_id BIGINT;
|
16
43
|
BEGIN
|
17
|
-
--
|
44
|
+
-- check arguments --------------------------------------------------------
|
45
|
+
|
18
46
|
IF workflow = '__manual__' AND COALESCE(max_attempts, 1) != 1 THEN
|
19
47
|
RAISE NOTICE 'Adjusting max_attempts of __manual__ job to 1';
|
20
48
|
max_attempts := 1;
|
21
49
|
END IF;
|
22
50
|
|
51
|
+
IF p_cron_interval IS NOT NULL THEN
|
52
|
+
IF parent_id IS NOT NULL THEN
|
53
|
+
RAISE 'A cron job must be a root job';
|
54
|
+
END IF;
|
55
|
+
IF p_cron_interval < 1 THEN
|
56
|
+
RAISE 'The cron_interval must be positive';
|
57
|
+
END IF;
|
58
|
+
END IF;
|
59
|
+
|
60
|
+
-- check arguments --------------------------------------------------------
|
61
|
+
|
23
62
|
workflow_version := COALESCE(workflow_version, '');
|
24
63
|
queue := COALESCE(queue, 'q');
|
25
64
|
max_attempts := COALESCE(max_attempts, 5);
|
65
|
+
p_is_sticky := COALESCE(p_is_sticky, FALSE);
|
26
66
|
|
27
67
|
-- create postjobs entry ------------------------------------------
|
28
68
|
INSERT INTO {SCHEMA_NAME}.postjobs (
|
29
69
|
last_worker_session_id,
|
30
70
|
queue, workflow, workflow_method, workflow_version, args,
|
31
71
|
parent_id, tags, max_attempts,
|
32
|
-
timing_out_at
|
72
|
+
timing_out_at,
|
73
|
+
cron_interval,
|
74
|
+
is_sticky
|
33
75
|
)
|
34
76
|
VALUES(
|
35
77
|
p_worker_session_id,
|
36
78
|
queue, workflow, workflow_method, workflow_version, args,
|
37
79
|
parent_id, tags, max_attempts,
|
38
|
-
(now() at time zone 'utc') + timeout * interval '1 second'
|
80
|
+
(now() at time zone 'utc') + timeout * interval '1 second',
|
81
|
+
p_cron_interval,
|
82
|
+
p_is_sticky
|
39
83
|
) RETURNING {SCHEMA_NAME}.postjobs.id INTO job_id;
|
40
84
|
|
41
85
|
-- fill in root_id and full_id ------------------------------------
|
@@ -2,8 +2,10 @@ DROP FUNCTION IF EXISTS {SCHEMA_NAME}._set_job_processing(job_id BIGINT); -- re
|
|
2
2
|
CREATE OR REPLACE FUNCTION {SCHEMA_NAME}._set_job_processing(worker_session_id UUID, job_id BIGINT) RETURNS VOID AS $$
|
3
3
|
DECLARE
|
4
4
|
v_pid int;
|
5
|
+
v_host_id uuid;
|
5
6
|
BEGIN
|
6
7
|
v_pid := pg_backend_pid();
|
8
|
+
SELECT host_id INTO v_host_id FROM {SCHEMA_NAME}.worker_sessions WHERE id=worker_session_id;
|
7
9
|
|
8
10
|
UPDATE {SCHEMA_NAME}.postjobs
|
9
11
|
SET
|
@@ -12,8 +14,9 @@ BEGIN
|
|
12
14
|
error=NULL,
|
13
15
|
error_message=NULL,
|
14
16
|
error_backtrace=NULL,
|
15
|
-
next_run_at=NULL
|
16
|
-
|
17
|
+
next_run_at=NULL,
|
18
|
+
sticky_host_id=(CASE WHEN is_sticky THEN v_host_id ELSE NULL END)
|
19
|
+
WHERE id=job_id;
|
17
20
|
END;
|
18
21
|
$$ LANGUAGE plpgsql;
|
19
22
|
|
@@ -45,10 +45,12 @@ BEGIN
|
|
45
45
|
END;
|
46
46
|
$$ LANGUAGE plpgsql;
|
47
47
|
|
48
|
+
DROP FUNCTION IF EXISTS {SCHEMA_NAME}._prepare_next_run(BIGINT, {SCHEMA_NAME}.statuses, BOOLEAN);
|
49
|
+
|
48
50
|
-- If this is a recoverable error and if we have another run possible
|
49
51
|
-- we'll set next_run_at, and the status to "err", otherwise
|
50
52
|
-- next_run_at will be NULL and the status would be "failed" or "timeout"
|
51
|
-
CREATE OR REPLACE FUNCTION {SCHEMA_NAME}.
|
53
|
+
CREATE OR REPLACE FUNCTION {SCHEMA_NAME}._prepare_rerun(
|
52
54
|
job_id BIGINT,
|
53
55
|
p_status {SCHEMA_NAME}.statuses,
|
54
56
|
p_fast_mode BOOLEAN) RETURNS VOID AS $$
|
@@ -119,8 +121,7 @@ BEGIN
|
|
119
121
|
next_run_at=NULL
|
120
122
|
WHERE id=job_id;
|
121
123
|
|
122
|
-
|
123
|
-
PERFORM {SCHEMA_NAME}._prepare_next_run(job_id, p_status, p_fast_mode);
|
124
|
+
PERFORM {SCHEMA_NAME}._prepare_rerun(job_id, p_status, p_fast_mode);
|
124
125
|
PERFORM {SCHEMA_NAME}._wakeup_parent_job(p_worker_session_id, job_id);
|
125
126
|
END;
|
126
127
|
$$ LANGUAGE plpgsql;
|
@@ -144,7 +145,7 @@ BEGIN
|
|
144
145
|
WHERE id=job_id;
|
145
146
|
|
146
147
|
-- prepare next run, if any
|
147
|
-
PERFORM {SCHEMA_NAME}.
|
148
|
+
PERFORM {SCHEMA_NAME}._prepare_rerun(job_id, 'timeout', p_fast_mode);
|
148
149
|
PERFORM {SCHEMA_NAME}._wakeup_parent_job(p_worker_session_id, job_id);
|
149
150
|
END;
|
150
151
|
$$ LANGUAGE plpgsql;
|
@@ -170,7 +171,7 @@ BEGIN
|
|
170
171
|
WHERE id=job_id;
|
171
172
|
|
172
173
|
-- prepare next run, if any
|
173
|
-
PERFORM {SCHEMA_NAME}.
|
174
|
+
PERFORM {SCHEMA_NAME}._prepare_rerun(job_id, 'err', p_fast_mode);
|
174
175
|
PERFORM {SCHEMA_NAME}._wakeup_parent_job(p_worker_session_id, job_id);
|
175
176
|
END;
|
176
177
|
$$ LANGUAGE plpgsql;
|
@@ -49,47 +49,42 @@ BEGIN
|
|
49
49
|
RAISE 'Invalid parent job id NULL';
|
50
50
|
END IF;
|
51
51
|
|
52
|
-
|
53
|
-
SELECT INTO parent * FROM {SCHEMA_NAME}.postjobs WHERE id=v_parent_id;
|
54
|
-
IF parent.id IS NULL THEN
|
55
|
-
RAISE 'No such job: %', v_parent_id;
|
56
|
-
END IF;
|
57
|
-
END IF;
|
52
|
+
-- create a child job if it doesn't exist yet.
|
58
53
|
|
59
|
-
|
54
|
+
SELECT id INTO child_id FROM {SCHEMA_NAME}.postjobs
|
55
|
+
WHERE parent_id=v_parent_id
|
56
|
+
AND workflow=v_workflow
|
57
|
+
AND workflow_method=v_workflow_method
|
58
|
+
AND args=v_args
|
59
|
+
;
|
60
60
|
|
61
|
-
|
62
|
-
SELECT
|
63
|
-
|
64
|
-
|
65
|
-
AND workflow_method=v_workflow_method
|
66
|
-
AND args=v_args
|
67
|
-
;
|
61
|
+
IF child_id IS NULL THEN
|
62
|
+
SELECT * INTO parent
|
63
|
+
FROM {SCHEMA_NAME}.postjobs
|
64
|
+
WHERE postjobs.id=v_parent_id;
|
68
65
|
|
69
|
-
|
70
|
-
|
71
|
-
--
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
66
|
+
SELECT id INTO child_id FROM {SCHEMA_NAME}.enqueue(
|
67
|
+
p_worker_session_id,
|
68
|
+
COALESCE(v_queue, parent.queue), -- queue VARCHAR,
|
69
|
+
v_workflow, -- workflow VARCHAR,
|
70
|
+
v_workflow_method, -- workflow_method VARCHAR,
|
71
|
+
NULL, -- workflow_version VARCHAR,
|
72
|
+
v_args, -- args JSONB,
|
73
|
+
v_parent_id, -- parent_id BIGINT,
|
74
|
+
parent.tags, -- tags JSONB,
|
75
|
+
v_max_attempts, -- max_attempts INTEGER,
|
76
|
+
v_timeout, -- timeout
|
77
|
+
NULL, -- cron_interval,
|
78
|
+
parent.is_sticky -- is_sticky?
|
79
|
+
);
|
79
80
|
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
v_parent_id, -- parent_id BIGINT,
|
89
|
-
parent.tags, -- tags JSONB,
|
90
|
-
v_max_attempts, -- max_attempts INTEGER,
|
91
|
-
v_timeout);
|
92
|
-
END IF;
|
93
|
-
-- END IF;
|
81
|
+
UPDATE {SCHEMA_NAME}.postjobs
|
82
|
+
SET sticky_host_id=parent.sticky_host_id
|
83
|
+
WHERE id=child_id;
|
84
|
+
END IF;
|
85
|
+
|
86
|
+
RETURN QUERY
|
87
|
+
SELECT * FROM {SCHEMA_NAME}.postjobs WHERE id=child_id
|
88
|
+
;
|
94
89
|
END;
|
95
90
|
$$ LANGUAGE plpgsql;
|
@@ -27,5 +27,5 @@ $$ LANGUAGE plpgsql;
|
|
27
27
|
|
28
28
|
-- define version settings ----------------------------------------------------
|
29
29
|
|
30
|
-
SELECT {SCHEMA_NAME}.settings_set('version', '0.5.
|
30
|
+
SELECT {SCHEMA_NAME}.settings_set('version', '0.5.6');
|
31
31
|
SELECT {SCHEMA_NAME}.settings_set('client_version', '{CLIENT_VERSION}');
|
@@ -4,9 +4,10 @@ CREATE OR REPLACE FUNCTION {SCHEMA_NAME}.time_to_next_job(p_worker_session_id UU
|
|
4
4
|
AS $$
|
5
5
|
DECLARE
|
6
6
|
p_processable_at timestamp;
|
7
|
-
|
7
|
+
session {SCHEMA_NAME}.worker_sessions;
|
8
8
|
BEGIN
|
9
|
-
SELECT
|
9
|
+
SELECT * INTO session
|
10
|
+
FROM {SCHEMA_NAME}.worker_sessions WHERE id=p_worker_session_id;
|
10
11
|
|
11
12
|
SELECT MIN(processable_at) INTO p_processable_at FROM (
|
12
13
|
SELECT MIN(timing_out_at) AS processable_at
|
@@ -16,10 +17,11 @@ BEGIN
|
|
16
17
|
SELECT MIN(next_run_at) AS processable_at
|
17
18
|
FROM {SCHEMA_NAME}.postjobs
|
18
19
|
WHERE status IN ('ready', 'err')
|
19
|
-
AND workflow || workflow_version = ANY (
|
20
|
+
AND (workflow || workflow_version) = ANY (session.workflows)
|
21
|
+
AND COALESCE(sticky_host_id, {SCHEMA_NAME}._null_uuid()) IN (session.host_id, {SCHEMA_NAME}._null_uuid())
|
20
22
|
) sq;
|
21
23
|
|
22
|
-
RETURN EXTRACT(EPOCH FROM
|
24
|
+
RETURN EXTRACT(EPOCH FROM p_processable_at - (now() at time zone 'utc'));
|
23
25
|
END;
|
24
26
|
$$ LANGUAGE plpgsql;
|
25
27
|
|
@@ -29,9 +31,10 @@ CREATE OR REPLACE FUNCTION {SCHEMA_NAME}.checkout(p_worker_session_id UUID, p_fa
|
|
29
31
|
AS $$
|
30
32
|
DECLARE
|
31
33
|
job {SCHEMA_NAME}.postjobs;
|
32
|
-
|
34
|
+
session {SCHEMA_NAME}.worker_sessions;
|
33
35
|
BEGIN
|
34
|
-
SELECT
|
36
|
+
SELECT * INTO session
|
37
|
+
FROM {SCHEMA_NAME}.worker_sessions WHERE id=p_worker_session_id;
|
35
38
|
|
36
39
|
LOOP
|
37
40
|
-- try to checkout a job. Each of the conditions here is matching
|
@@ -47,7 +50,8 @@ BEGIN
|
|
47
50
|
(
|
48
51
|
s.status IN ('ready', 'err')
|
49
52
|
AND s.next_run_at <= (now() at time zone 'utc')
|
50
|
-
AND (s.workflow || s.workflow_version) = ANY (
|
53
|
+
AND (s.workflow || s.workflow_version) = ANY (session.workflows)
|
54
|
+
AND COALESCE(s.sticky_host_id, {SCHEMA_NAME}._null_uuid()) IN (session.host_id, {SCHEMA_NAME}._null_uuid())
|
51
55
|
)
|
52
56
|
ORDER BY (LEAST(s.next_run_at, s.timing_out_at))
|
53
57
|
FOR UPDATE SKIP LOCKED
|
@@ -15,6 +15,12 @@ DECLARE
|
|
15
15
|
BEGIN
|
16
16
|
zombie_count := 0;
|
17
17
|
FOR zombie_id, _one IN
|
18
|
+
-- select jobs that have a last_worker_session_id, which points to a
|
19
|
+
-- host whose latest heartbeat is older than +zombie_threshold+.
|
20
|
+
--
|
21
|
+
-- note that we ignore hosts (and, for that matter, jobs) that don't
|
22
|
+
-- have any heartbeats, since this scenario should only appear during
|
23
|
+
-- tests.
|
18
24
|
SELECT jobs.id, 1
|
19
25
|
FROM {SCHEMA_NAME}.postjobs jobs
|
20
26
|
LEFT JOIN {SCHEMA_NAME}.worker_sessions sessions ON jobs.last_worker_session_id=sessions.id
|
@@ -27,10 +33,7 @@ BEGIN
|
|
27
33
|
) heartbeat ON sessions.host_id = heartbeat.host_id
|
28
34
|
WHERE
|
29
35
|
jobs.status IN ('processing')
|
30
|
-
AND (
|
31
|
-
heartbeat.created_at IS NULL OR
|
32
|
-
heartbeat.created_at < ((now() at time zone 'utc') - zombie_threshold)
|
33
|
-
)
|
36
|
+
AND heartbeat.created_at < ((now() at time zone 'utc') - zombie_threshold)
|
34
37
|
LOOP
|
35
38
|
PERFORM {SCHEMA_NAME}._set_job_zombie(zombie_id, p_fast_mode);
|
36
39
|
zombie_count := zombie_count + 1;
|
@@ -51,8 +54,10 @@ BEGIN
|
|
51
54
|
-- event, which has a zombie count value in its attributes.
|
52
55
|
IF NOT EXISTS (SELECT 1 FROM {SCHEMA_NAME}.events WHERE name='zombie' AND created_at > (now() at time zone 'utc') - zombie_check_interval) THEN
|
53
56
|
p_zombie_count := {SCHEMA_NAME}._zombie_check(p_fast_mode);
|
54
|
-
|
55
|
-
|
57
|
+
IF p_zombie_count > 0 THEN
|
58
|
+
INSERT INTO {SCHEMA_NAME}.events(name, host_id, attributes)
|
59
|
+
VALUES('zombie', {SCHEMA_NAME}._null_uuid(), jsonb_build_object('zombie_count', p_zombie_count));
|
60
|
+
END IF;
|
56
61
|
END IF;
|
57
62
|
END;
|
58
63
|
$$ LANGUAGE plpgsql;
|
@@ -0,0 +1,65 @@
|
|
1
|
+
DO $$
|
2
|
+
BEGIN
|
3
|
+
ALTER TABLE {SCHEMA_NAME}.postjobs ADD COLUMN cron_interval integer;
|
4
|
+
EXCEPTION
|
5
|
+
WHEN duplicate_column THEN RAISE DEBUG 'column {SCHEMA_NAME}.postjobs.cron_interval already exists';
|
6
|
+
END;
|
7
|
+
$$;
|
8
|
+
|
9
|
+
--- define triggers to automatically restart cron jobs ------------------------
|
10
|
+
|
11
|
+
-- This method is called whenever a job's status changes. It enqueues a fresh
|
12
|
+
|
13
|
+
--
|
14
|
+
-- An event is created whenever a job's status changes.
|
15
|
+
--
|
16
|
+
|
17
|
+
CREATE OR REPLACE FUNCTION {SCHEMA_NAME}._restart_cron_job()
|
18
|
+
RETURNS TRIGGER
|
19
|
+
AS $$
|
20
|
+
DECLARE
|
21
|
+
p_new_jonb_id BIGINT;
|
22
|
+
BEGIN
|
23
|
+
IF NEW.status = OLD.status THEN
|
24
|
+
RETURN NEW;
|
25
|
+
END IF;
|
26
|
+
|
27
|
+
IF TG_OP != 'UPDATE' OR NEW.status NOT IN ('failed', 'ok') THEN
|
28
|
+
RETURN NEW;
|
29
|
+
END IF;
|
30
|
+
|
31
|
+
IF NEW.cron_interval IS NULL THEN
|
32
|
+
RETURN NEW;
|
33
|
+
END IF;
|
34
|
+
|
35
|
+
SELECT id INTO p_new_jonb_id FROM {SCHEMA_NAME}.enqueue(
|
36
|
+
NEW.last_worker_session_id, -- p_worker_session_id
|
37
|
+
NEW.queue, -- queue
|
38
|
+
NEW.workflow, -- workflow
|
39
|
+
NEW.workflow_method, -- workflow_method
|
40
|
+
NULL, -- workflow_version
|
41
|
+
NEW.args, -- args
|
42
|
+
NULL, -- parent_id
|
43
|
+
NEW.tags, -- tags
|
44
|
+
NEW.max_attempts, -- max_attempts
|
45
|
+
(EXTRACT(EPOCH FROM NEW.timing_out_at) - EXTRACT(EPOCH FROM NEW.created_at)), -- timeout
|
46
|
+
NEW.cron_interval,
|
47
|
+
NEW.is_sticky
|
48
|
+
);
|
49
|
+
|
50
|
+
UPDATE {SCHEMA_NAME}.postjobs
|
51
|
+
SET next_run_at = now() at time zone 'utc' + NEW.cron_interval * interval '1 second'
|
52
|
+
WHERE id=p_new_jonb_id;
|
53
|
+
|
54
|
+
RETURN NEW;
|
55
|
+
END;
|
56
|
+
$$ LANGUAGE plpgsql;
|
57
|
+
|
58
|
+
BEGIN;
|
59
|
+
DROP TRIGGER IF EXISTS _restart_cron_job ON {SCHEMA_NAME}.postjobs;
|
60
|
+
|
61
|
+
CREATE TRIGGER _restart_cron_job AFTER UPDATE
|
62
|
+
ON {SCHEMA_NAME}.postjobs
|
63
|
+
FOR EACH ROW
|
64
|
+
EXECUTE PROCEDURE {SCHEMA_NAME}._restart_cron_job();
|
65
|
+
COMMIT;
|
@@ -0,0 +1,16 @@
|
|
1
|
+
DO $$
|
2
|
+
BEGIN
|
3
|
+
ALTER TABLE {SCHEMA_NAME}.postjobs ADD COLUMN is_sticky BOOLEAN NOT NULL DEFAULT FALSE;
|
4
|
+
EXCEPTION
|
5
|
+
WHEN duplicate_column THEN RAISE DEBUG 'column {SCHEMA_NAME}.postjobs.is_sticky already exists';
|
6
|
+
END;
|
7
|
+
$$;
|
8
|
+
|
9
|
+
DO $$
|
10
|
+
BEGIN
|
11
|
+
ALTER TABLE {SCHEMA_NAME}.postjobs ADD COLUMN sticky_host_id UUID
|
12
|
+
REFERENCES {SCHEMA_NAME}.hosts;
|
13
|
+
EXCEPTION
|
14
|
+
WHEN duplicate_column THEN RAISE DEBUG 'column {SCHEMA_NAME}.postjobs.sticky_host_id already exists';
|
15
|
+
END;
|
16
|
+
$$;
|
data/lib/postjob/migrations.rb
CHANGED
@@ -16,6 +16,8 @@ module Postjob
|
|
16
16
|
expect! SCHEMA_NAME != "public"
|
17
17
|
|
18
18
|
def unmigrate!
|
19
|
+
::Postjob::Host.clear_storage
|
20
|
+
|
19
21
|
SQL.exec "SET client_min_messages TO WARNING"
|
20
22
|
SQL.exec "DROP SCHEMA IF EXISTS #{SCHEMA_NAME} CASCADE"
|
21
23
|
end
|
@@ -45,7 +47,7 @@ module Postjob
|
|
45
47
|
|
46
48
|
def run_migration_sql(file)
|
47
49
|
sql = File.read(file)
|
48
|
-
sql.gsub!(/\{([^\}]+)\}/) { |_| const_get(
|
50
|
+
sql.gsub!(/\{([^\}]+)\}/) { |_| const_get(Regexp.last_match(1)) }
|
49
51
|
SQL.exec sql
|
50
52
|
end
|
51
53
|
end
|