riverqueue 0.11.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +132 -0
- data/LICENSE +374 -0
- data/docs/README.md +765 -0
- data/docs/migrations.md +121 -0
- data/docs/testing.md +173 -0
- data/docs/workers.md +126 -0
- data/exe/river +7 -0
- data/lib/cli.rb +55 -0
- data/lib/client.rb +314 -54
- data/lib/client_runtime.rb +612 -0
- data/lib/config.rb +154 -0
- data/lib/driver/job_row_decoder.rb +85 -0
- data/lib/driver/postgres_capabilities.rb +41 -0
- data/lib/driver/runtime.rb +661 -0
- data/lib/driver.rb +13 -6
- data/lib/errors.rb +83 -0
- data/lib/event.rb +78 -0
- data/lib/insert_opts.rb +40 -11
- data/lib/job.rb +28 -13
- data/lib/job_persisted_logging/plugin.rb +108 -0
- data/lib/migration_cli.rb +90 -0
- data/lib/migrator.rb +189 -0
- data/lib/params.rb +96 -0
- data/lib/periodic_cron.rb +24 -0
- data/lib/periodic_job.rb +135 -0
- data/lib/resumable.rb +148 -0
- data/lib/riverqueue/testing/minitest.rb +18 -0
- data/lib/riverqueue/testing/rspec.rb +109 -0
- data/lib/riverqueue/testing.rb +134 -0
- data/lib/riverqueue.rb +16 -0
- data/lib/unique_args.rb +97 -0
- data/lib/unique_bitmask.rb +14 -12
- data/lib/worker.rb +152 -0
- data/lib/worker_runner.rb +106 -0
- data/migration/LICENSE +374 -0
- data/migration/README.md +11 -0
- data/migration/manifest.json +38 -0
- data/migration/postgresql/main/001_create_river_migration.down.sql +1 -0
- data/migration/postgresql/main/001_create_river_migration.up.sql +8 -0
- data/migration/postgresql/main/002_initial_schema.down.sql +5 -0
- data/migration/postgresql/main/002_initial_schema.up.sql +96 -0
- data/migration/postgresql/main/003_river_job_tags_non_null.down.sql +3 -0
- data/migration/postgresql/main/003_river_job_tags_non_null.up.sql +3 -0
- data/migration/postgresql/main/004_pending_and_more.down.sql +42 -0
- data/migration/postgresql/main/004_pending_and_more.up.sql +45 -0
- data/migration/postgresql/main/005_migration_unique_client.down.sql +57 -0
- data/migration/postgresql/main/005_migration_unique_client.up.sql +79 -0
- data/migration/postgresql/main/006_bulk_unique.down.sql +16 -0
- data/migration/postgresql/main/006_bulk_unique.up.sql +40 -0
- data/migration/postgresql/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.down.sql +56 -0
- data/migration/postgresql/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.up.sql +44 -0
- data/migration/postgresql/main/008_job_id_autoincrement.down.sql +3 -0
- data/migration/postgresql/main/008_job_id_autoincrement.up.sql +3 -0
- data/migration/sqlite/main/001_create_river_migration.down.sql +1 -0
- data/migration/sqlite/main/001_create_river_migration.up.sql +8 -0
- data/migration/sqlite/main/002_initial_schema.down.sql +8 -0
- data/migration/sqlite/main/002_initial_schema.up.sql +19 -0
- data/migration/sqlite/main/003_river_job_tags_non_null.down.sql +6 -0
- data/migration/sqlite/main/003_river_job_tags_non_null.up.sql +6 -0
- data/migration/sqlite/main/004_pending_and_more.down.sql +26 -0
- data/migration/sqlite/main/004_pending_and_more.up.sql +33 -0
- data/migration/sqlite/main/005_migration_unique_client.down.sql +37 -0
- data/migration/sqlite/main/005_migration_unique_client.up.sql +64 -0
- data/migration/sqlite/main/006_bulk_unique.down.sql +7 -0
- data/migration/sqlite/main/006_bulk_unique.up.sql +63 -0
- data/migration/sqlite/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.down.sql +255 -0
- data/migration/sqlite/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.up.sql +261 -0
- data/migration/sqlite/main/008_job_id_autoincrement.down.sql +121 -0
- data/migration/sqlite/main/008_job_id_autoincrement.up.sql +123 -0
- data/sig/client.rbs +95 -0
- data/sig/driver.rbs +170 -0
- data/sig/insert_opts.rbs +25 -0
- data/sig/job.rbs +72 -0
- data/sig/job_persisted_logging/plugin.rbs +29 -0
- data/sig/migrator.rbs +44 -0
- data/sig/riverqueue.rbs +2 -0
- data/sig/runtime.rbs +382 -0
- data/sig/testing.rbs +95 -0
- data/sig/unique_args.rbs +8 -0
- data/sig/unique_bitmask.rbs +8 -0
- data/sig/worker_runner.rbs +22 -0
- metadata +165 -10
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
--
|
|
2
|
+
-- Revert to migration table based only on `(version)`.
|
|
3
|
+
--
|
|
4
|
+
-- If any non-main migrations are present, 005 is considered irreversible.
|
|
5
|
+
--
|
|
6
|
+
|
|
7
|
+
DO
|
|
8
|
+
$body$
|
|
9
|
+
BEGIN
|
|
10
|
+
-- Tolerate users who may be using their own migration system rather than
|
|
11
|
+
-- River's. If they are, they will have skipped version 001 containing
|
|
12
|
+
-- `CREATE TABLE river_migration`, so this table won't exist.
|
|
13
|
+
IF (SELECT to_regclass('/* TEMPLATE: schema */river_migration') IS NOT NULL) THEN
|
|
14
|
+
IF EXISTS (
|
|
15
|
+
SELECT *
|
|
16
|
+
FROM /* TEMPLATE: schema */river_migration
|
|
17
|
+
WHERE line <> 'main'
|
|
18
|
+
) THEN
|
|
19
|
+
RAISE EXCEPTION 'Found non-main migration lines in the database; version 005 migration is irreversible because it would result in loss of migration information.';
|
|
20
|
+
END IF;
|
|
21
|
+
|
|
22
|
+
ALTER TABLE /* TEMPLATE: schema */river_migration
|
|
23
|
+
RENAME TO river_migration_old;
|
|
24
|
+
|
|
25
|
+
CREATE TABLE /* TEMPLATE: schema */river_migration(
|
|
26
|
+
id bigserial PRIMARY KEY,
|
|
27
|
+
created_at timestamptz NOT NULL DEFAULT NOW(),
|
|
28
|
+
version bigint NOT NULL,
|
|
29
|
+
CONSTRAINT version CHECK (version >= 1)
|
|
30
|
+
);
|
|
31
|
+
|
|
32
|
+
CREATE UNIQUE INDEX ON /* TEMPLATE: schema */river_migration USING btree(version);
|
|
33
|
+
|
|
34
|
+
INSERT INTO /* TEMPLATE: schema */river_migration
|
|
35
|
+
(created_at, version)
|
|
36
|
+
SELECT created_at, version
|
|
37
|
+
FROM /* TEMPLATE: schema */river_migration_old;
|
|
38
|
+
|
|
39
|
+
DROP TABLE /* TEMPLATE: schema */river_migration_old;
|
|
40
|
+
END IF;
|
|
41
|
+
END;
|
|
42
|
+
$body$
|
|
43
|
+
LANGUAGE 'plpgsql';
|
|
44
|
+
|
|
45
|
+
--
|
|
46
|
+
-- Drop `river_job.unique_key`.
|
|
47
|
+
--
|
|
48
|
+
|
|
49
|
+
ALTER TABLE /* TEMPLATE: schema */river_job
|
|
50
|
+
DROP COLUMN unique_key;
|
|
51
|
+
|
|
52
|
+
--
|
|
53
|
+
-- Drop `river_client` and derivative.
|
|
54
|
+
--
|
|
55
|
+
|
|
56
|
+
DROP TABLE /* TEMPLATE: schema */river_client_queue;
|
|
57
|
+
DROP TABLE /* TEMPLATE: schema */river_client;
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
--
|
|
2
|
+
-- Rebuild the migration table so it's based on `(line, version)`.
|
|
3
|
+
--
|
|
4
|
+
|
|
5
|
+
DO
|
|
6
|
+
$body$
|
|
7
|
+
BEGIN
|
|
8
|
+
-- Tolerate users who may be using their own migration system rather than
|
|
9
|
+
-- River's. If they are, they will have skipped version 001 containing
|
|
10
|
+
-- `CREATE TABLE river_migration`, so this table won't exist.
|
|
11
|
+
IF (SELECT to_regclass('/* TEMPLATE: schema */river_migration') IS NOT NULL) THEN
|
|
12
|
+
ALTER TABLE /* TEMPLATE: schema */river_migration
|
|
13
|
+
RENAME TO river_migration_old;
|
|
14
|
+
|
|
15
|
+
CREATE TABLE /* TEMPLATE: schema */river_migration(
|
|
16
|
+
line TEXT NOT NULL,
|
|
17
|
+
version bigint NOT NULL,
|
|
18
|
+
created_at timestamptz NOT NULL DEFAULT NOW(),
|
|
19
|
+
CONSTRAINT line_length CHECK (char_length(line) > 0 AND char_length(line) < 128),
|
|
20
|
+
CONSTRAINT version_gte_1 CHECK (version >= 1),
|
|
21
|
+
PRIMARY KEY (line, version)
|
|
22
|
+
);
|
|
23
|
+
|
|
24
|
+
INSERT INTO /* TEMPLATE: schema */river_migration
|
|
25
|
+
(created_at, line, version)
|
|
26
|
+
SELECT created_at, 'main', version
|
|
27
|
+
FROM /* TEMPLATE: schema */river_migration_old;
|
|
28
|
+
|
|
29
|
+
DROP TABLE /* TEMPLATE: schema */river_migration_old;
|
|
30
|
+
END IF;
|
|
31
|
+
END;
|
|
32
|
+
$body$
|
|
33
|
+
LANGUAGE 'plpgsql';
|
|
34
|
+
|
|
35
|
+
--
|
|
36
|
+
-- Add `river_job.unique_key` and bring up an index on it.
|
|
37
|
+
--
|
|
38
|
+
|
|
39
|
+
-- These statements use `IF NOT EXISTS` to allow users with a `river_job` table
|
|
40
|
+
-- of non-trivial size to build the index `CONCURRENTLY` out of band of this
|
|
41
|
+
-- migration, then follow by completing the migration.
|
|
42
|
+
ALTER TABLE /* TEMPLATE: schema */river_job
|
|
43
|
+
ADD COLUMN IF NOT EXISTS unique_key bytea;
|
|
44
|
+
|
|
45
|
+
CREATE UNIQUE INDEX IF NOT EXISTS river_job_kind_unique_key_idx ON /* TEMPLATE: schema */river_job (kind, unique_key) WHERE unique_key IS NOT NULL;
|
|
46
|
+
|
|
47
|
+
--
|
|
48
|
+
-- Create `river_client` and derivative.
|
|
49
|
+
--
|
|
50
|
+
-- This feature hasn't quite yet been implemented, but we're taking advantage of
|
|
51
|
+
-- the migration to add the schema early so that we can add it later without an
|
|
52
|
+
-- additional migration.
|
|
53
|
+
--
|
|
54
|
+
|
|
55
|
+
CREATE UNLOGGED TABLE /* TEMPLATE: schema */river_client (
|
|
56
|
+
id text PRIMARY KEY NOT NULL,
|
|
57
|
+
created_at timestamptz NOT NULL DEFAULT now(),
|
|
58
|
+
metadata jsonb NOT NULL DEFAULT '{}',
|
|
59
|
+
paused_at timestamptz,
|
|
60
|
+
updated_at timestamptz NOT NULL,
|
|
61
|
+
CONSTRAINT name_length CHECK (char_length(id) > 0 AND char_length(id) < 128)
|
|
62
|
+
);
|
|
63
|
+
|
|
64
|
+
-- Differs from `river_queue` in that it tracks the queue state for a particular
|
|
65
|
+
-- active client.
|
|
66
|
+
CREATE UNLOGGED TABLE /* TEMPLATE: schema */river_client_queue (
|
|
67
|
+
river_client_id text NOT NULL REFERENCES /* TEMPLATE: schema */river_client (id) ON DELETE CASCADE,
|
|
68
|
+
name text NOT NULL,
|
|
69
|
+
created_at timestamptz NOT NULL DEFAULT now(),
|
|
70
|
+
max_workers bigint NOT NULL DEFAULT 0,
|
|
71
|
+
metadata jsonb NOT NULL DEFAULT '{}',
|
|
72
|
+
num_jobs_completed bigint NOT NULL DEFAULT 0,
|
|
73
|
+
num_jobs_running bigint NOT NULL DEFAULT 0,
|
|
74
|
+
updated_at timestamptz NOT NULL,
|
|
75
|
+
PRIMARY KEY (river_client_id, name),
|
|
76
|
+
CONSTRAINT name_length CHECK (char_length(name) > 0 AND char_length(name) < 128),
|
|
77
|
+
CONSTRAINT num_jobs_completed_zero_or_positive CHECK (num_jobs_completed >= 0),
|
|
78
|
+
CONSTRAINT num_jobs_running_zero_or_positive CHECK (num_jobs_running >= 0)
|
|
79
|
+
);
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
|
|
2
|
+
--
|
|
3
|
+
-- Drop `river_job.unique_states` and its index.
|
|
4
|
+
--
|
|
5
|
+
|
|
6
|
+
DROP INDEX /* TEMPLATE: schema */river_job_unique_idx;
|
|
7
|
+
|
|
8
|
+
ALTER TABLE /* TEMPLATE: schema */river_job
|
|
9
|
+
DROP COLUMN unique_states;
|
|
10
|
+
|
|
11
|
+
CREATE UNIQUE INDEX IF NOT EXISTS river_job_kind_unique_key_idx ON /* TEMPLATE: schema */river_job (kind, unique_key) WHERE unique_key IS NOT NULL;
|
|
12
|
+
|
|
13
|
+
--
|
|
14
|
+
-- Drop `river_job_state_in_bitmask` function.
|
|
15
|
+
--
|
|
16
|
+
DROP FUNCTION /* TEMPLATE: schema */river_job_state_in_bitmask;
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
CREATE OR REPLACE FUNCTION /* TEMPLATE: schema */river_job_state_in_bitmask(bitmask BIT(8), state /* TEMPLATE: schema */river_job_state)
|
|
2
|
+
RETURNS boolean
|
|
3
|
+
LANGUAGE SQL
|
|
4
|
+
IMMUTABLE
|
|
5
|
+
AS $$
|
|
6
|
+
SELECT CASE state
|
|
7
|
+
WHEN 'available' THEN get_bit(bitmask, 7)
|
|
8
|
+
WHEN 'cancelled' THEN get_bit(bitmask, 6)
|
|
9
|
+
WHEN 'completed' THEN get_bit(bitmask, 5)
|
|
10
|
+
WHEN 'discarded' THEN get_bit(bitmask, 4)
|
|
11
|
+
WHEN 'pending' THEN get_bit(bitmask, 3)
|
|
12
|
+
WHEN 'retryable' THEN get_bit(bitmask, 2)
|
|
13
|
+
WHEN 'running' THEN get_bit(bitmask, 1)
|
|
14
|
+
WHEN 'scheduled' THEN get_bit(bitmask, 0)
|
|
15
|
+
ELSE 0
|
|
16
|
+
END = 1;
|
|
17
|
+
$$;
|
|
18
|
+
|
|
19
|
+
--
|
|
20
|
+
-- Add `river_job.unique_states` and bring up an index on it.
|
|
21
|
+
--
|
|
22
|
+
-- This column may exist already if users manually created the column and index
|
|
23
|
+
-- as instructed in the changelog so the index could be created `CONCURRENTLY`.
|
|
24
|
+
--
|
|
25
|
+
ALTER TABLE /* TEMPLATE: schema */river_job ADD COLUMN IF NOT EXISTS unique_states BIT(8);
|
|
26
|
+
|
|
27
|
+
-- This statement uses `IF NOT EXISTS` to allow users with a `river_job` table
|
|
28
|
+
-- of non-trivial size to build the index `CONCURRENTLY` out of band of this
|
|
29
|
+
-- migration, then follow by completing the migration.
|
|
30
|
+
CREATE UNIQUE INDEX IF NOT EXISTS river_job_unique_idx ON /* TEMPLATE: schema */river_job (unique_key)
|
|
31
|
+
WHERE unique_key IS NOT NULL
|
|
32
|
+
AND unique_states IS NOT NULL
|
|
33
|
+
AND /* TEMPLATE: schema */river_job_state_in_bitmask(unique_states, state);
|
|
34
|
+
|
|
35
|
+
-- Remove the old unique index. Users who are actively using the unique jobs
|
|
36
|
+
-- feature and who wish to avoid deploy downtime may want od drop this in a
|
|
37
|
+
-- subsequent migration once all jobs using the old unique system have been
|
|
38
|
+
-- completed (i.e. no more rows with non-null unique_key and null
|
|
39
|
+
-- unique_states).
|
|
40
|
+
DROP INDEX /* TEMPLATE: schema */river_job_kind_unique_key_idx;
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
--
|
|
2
|
+
-- SQL cleanup rollback.
|
|
3
|
+
--
|
|
4
|
+
|
|
5
|
+
--
|
|
6
|
+
-- Add back unused tables `river_client` and `river_client_queue`.
|
|
7
|
+
--
|
|
8
|
+
|
|
9
|
+
CREATE UNLOGGED TABLE /* TEMPLATE: schema */river_client (
|
|
10
|
+
id text PRIMARY KEY NOT NULL,
|
|
11
|
+
created_at timestamptz NOT NULL DEFAULT now(),
|
|
12
|
+
metadata jsonb NOT NULL DEFAULT '{}',
|
|
13
|
+
paused_at timestamptz,
|
|
14
|
+
updated_at timestamptz NOT NULL,
|
|
15
|
+
CONSTRAINT name_length CHECK (char_length(id) > 0 AND char_length(id) < 128)
|
|
16
|
+
);
|
|
17
|
+
|
|
18
|
+
CREATE UNLOGGED TABLE /* TEMPLATE: schema */river_client_queue (
|
|
19
|
+
river_client_id text NOT NULL REFERENCES /* TEMPLATE: schema */river_client (id) ON DELETE CASCADE,
|
|
20
|
+
name text NOT NULL,
|
|
21
|
+
created_at timestamptz NOT NULL DEFAULT now(),
|
|
22
|
+
max_workers bigint NOT NULL DEFAULT 0,
|
|
23
|
+
metadata jsonb NOT NULL DEFAULT '{}',
|
|
24
|
+
num_jobs_completed bigint NOT NULL DEFAULT 0,
|
|
25
|
+
num_jobs_running bigint NOT NULL DEFAULT 0,
|
|
26
|
+
updated_at timestamptz NOT NULL,
|
|
27
|
+
PRIMARY KEY (river_client_id, name),
|
|
28
|
+
CONSTRAINT name_length CHECK (char_length(name) > 0 AND char_length(name) < 128),
|
|
29
|
+
CONSTRAINT num_jobs_completed_zero_or_positive CHECK (num_jobs_completed >= 0),
|
|
30
|
+
CONSTRAINT num_jobs_running_zero_or_positive CHECK (num_jobs_running >= 0)
|
|
31
|
+
);
|
|
32
|
+
|
|
33
|
+
--
|
|
34
|
+
-- Revert addition of `DEFAULT 25` to `river_job.max_attempts`.
|
|
35
|
+
--
|
|
36
|
+
|
|
37
|
+
ALTER TABLE /* TEMPLATE: schema */river_job
|
|
38
|
+
ALTER COLUMN max_attempts DROP DEFAULT;
|
|
39
|
+
|
|
40
|
+
--
|
|
41
|
+
-- Changes `river_queue.updated_at` to revert the default of `CURRENT_TIMESTAMP`.
|
|
42
|
+
--
|
|
43
|
+
|
|
44
|
+
ALTER TABLE /* TEMPLATE: schema */river_queue
|
|
45
|
+
ALTER COLUMN updated_at DROP DEFAULT;
|
|
46
|
+
|
|
47
|
+
--
|
|
48
|
+
-- SQLite JSONB conversion rollback.
|
|
49
|
+
--
|
|
50
|
+
-- No-op. PostgreSQL already stores River JSON columns as jsonb.
|
|
51
|
+
|
|
52
|
+
--
|
|
53
|
+
-- Notification outbox rollback.
|
|
54
|
+
--
|
|
55
|
+
|
|
56
|
+
DROP TABLE /* TEMPLATE: schema */river_notification;
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
--
|
|
2
|
+
-- Notification outbox.
|
|
3
|
+
--
|
|
4
|
+
|
|
5
|
+
CREATE TABLE /* TEMPLATE: schema */river_notification (
|
|
6
|
+
id bigserial PRIMARY KEY,
|
|
7
|
+
created_at timestamptz NOT NULL DEFAULT now(),
|
|
8
|
+
payload text NOT NULL,
|
|
9
|
+
topic text NOT NULL,
|
|
10
|
+
CONSTRAINT topic_length CHECK (length(topic) > 0 AND length(topic) < 128)
|
|
11
|
+
);
|
|
12
|
+
|
|
13
|
+
CREATE INDEX river_notification_created_at_idx ON /* TEMPLATE: schema */river_notification (created_at);
|
|
14
|
+
CREATE INDEX river_notification_topic_id_idx ON /* TEMPLATE: schema */river_notification (topic, id);
|
|
15
|
+
|
|
16
|
+
--
|
|
17
|
+
-- SQLite JSONB conversion.
|
|
18
|
+
--
|
|
19
|
+
-- No-op. PostgreSQL already stores River JSON columns as jsonb.
|
|
20
|
+
|
|
21
|
+
--
|
|
22
|
+
-- SQL cleanup.
|
|
23
|
+
--
|
|
24
|
+
|
|
25
|
+
--
|
|
26
|
+
-- Drop unused tables `river_client` and `river_client_queue`.
|
|
27
|
+
--
|
|
28
|
+
|
|
29
|
+
DROP TABLE /* TEMPLATE: schema */river_client_queue;
|
|
30
|
+
DROP TABLE /* TEMPLATE: schema */river_client;
|
|
31
|
+
|
|
32
|
+
--
|
|
33
|
+
-- Adds `DEFAULT 25` to `river_job.max_attempts`.
|
|
34
|
+
--
|
|
35
|
+
|
|
36
|
+
ALTER TABLE /* TEMPLATE: schema */river_job
|
|
37
|
+
ALTER COLUMN max_attempts SET DEFAULT 25;
|
|
38
|
+
|
|
39
|
+
--
|
|
40
|
+
-- Changes `river_queue.updated_at` to have a default of `CURRENT_TIMESTAMP`.
|
|
41
|
+
--
|
|
42
|
+
|
|
43
|
+
ALTER TABLE /* TEMPLATE: schema */river_queue
|
|
44
|
+
ALTER COLUMN updated_at SET DEFAULT CURRENT_TIMESTAMP;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
DROP TABLE /* TEMPLATE: schema */river_migration;
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
CREATE TABLE /* TEMPLATE: schema */river_migration (
|
|
2
|
+
id integer PRIMARY KEY,
|
|
3
|
+
created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
4
|
+
version integer NOT NULL,
|
|
5
|
+
CONSTRAINT version CHECK (version >= 1)
|
|
6
|
+
);
|
|
7
|
+
|
|
8
|
+
CREATE UNIQUE INDEX /* TEMPLATE: schema */river_migration_version_idx ON river_migration (version);
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
--
|
|
2
|
+
-- Normally `river_job` and `river_job_notify()` are dropped here, but since
|
|
3
|
+
-- SQLite was added well after 002 came about, we push that to version 006 index.
|
|
4
|
+
--
|
|
5
|
+
|
|
6
|
+
DROP TABLE /* TEMPLATE: schema */river_job;
|
|
7
|
+
|
|
8
|
+
DROP TABLE /* TEMPLATE: schema */river_leader;
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
--
|
|
2
|
+
-- Normally `river_job` and `river_job_notify()` are raised here, but since
|
|
3
|
+
-- SQLite was added well after 002 came about, we push that to version 006 index.
|
|
4
|
+
--
|
|
5
|
+
|
|
6
|
+
-- Dummy `river_job` table so that there's something to truncate in tests when
|
|
7
|
+
-- migrated to this version specifically.
|
|
8
|
+
CREATE TABLE /* TEMPLATE: schema */river_job (
|
|
9
|
+
id integer PRIMARY KEY
|
|
10
|
+
);
|
|
11
|
+
|
|
12
|
+
CREATE TABLE /* TEMPLATE: schema */river_leader (
|
|
13
|
+
elected_at timestamp NOT NULL,
|
|
14
|
+
expires_at timestamp NOT NULL,
|
|
15
|
+
leader_id text NOT NULL,
|
|
16
|
+
name text PRIMARY KEY NOT NULL,
|
|
17
|
+
CONSTRAINT name_length CHECK (length(name) > 0 AND length(name) < 128),
|
|
18
|
+
CONSTRAINT leader_id_length CHECK (length(leader_id) > 0 AND length(leader_id) < 128)
|
|
19
|
+
);
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
--
|
|
2
|
+
-- Normally, args and metadata both become `NOT NULL`, `pending` is added, and
|
|
3
|
+
-- the constraint `finalized_at` is changed, but because SQLite was added later,
|
|
4
|
+
-- we've just pushed all of this to an initial `river_job` creation in 006.
|
|
5
|
+
--
|
|
6
|
+
|
|
7
|
+
--
|
|
8
|
+
-- Drop `river_queue`.
|
|
9
|
+
--
|
|
10
|
+
|
|
11
|
+
DROP TABLE /* TEMPLATE: schema */river_queue;
|
|
12
|
+
|
|
13
|
+
--
|
|
14
|
+
-- Reverse changes to `river_leader`.
|
|
15
|
+
--
|
|
16
|
+
|
|
17
|
+
DROP TABLE /* TEMPLATE: schema */river_leader;
|
|
18
|
+
|
|
19
|
+
CREATE TABLE /* TEMPLATE: schema */river_leader (
|
|
20
|
+
elected_at timestamp NOT NULL,
|
|
21
|
+
expires_at timestamp NOT NULL,
|
|
22
|
+
leader_id text NOT NULL,
|
|
23
|
+
name text PRIMARY KEY NOT NULL,
|
|
24
|
+
CONSTRAINT name_length CHECK (length(name) > 0 AND length(name) < 128),
|
|
25
|
+
CONSTRAINT leader_id_length CHECK (length(leader_id) > 0 AND length(leader_id) < 128)
|
|
26
|
+
);
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
--
|
|
2
|
+
-- Normally, args and metadata both become `NOT NULL`, `pending` is added, and
|
|
3
|
+
-- the constraint `finalized_at` is changed, but because SQLite was added later,
|
|
4
|
+
-- we've just pushed all of this to an initial `river_job` creation in 006.
|
|
5
|
+
--
|
|
6
|
+
|
|
7
|
+
--
|
|
8
|
+
-- Create table `river_queue`.
|
|
9
|
+
--
|
|
10
|
+
|
|
11
|
+
CREATE TABLE /* TEMPLATE: schema */river_queue (
|
|
12
|
+
name text PRIMARY KEY NOT NULL,
|
|
13
|
+
created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
14
|
+
metadata blob NOT NULL DEFAULT (json('{}')),
|
|
15
|
+
paused_at timestamp,
|
|
16
|
+
updated_at timestamp NOT NULL
|
|
17
|
+
);
|
|
18
|
+
|
|
19
|
+
--
|
|
20
|
+
-- Alter `river_leader` to add a default value of 'default` to `name`. SQLite
|
|
21
|
+
-- doesn't allow schema modifications, so this redefines the table entirely.
|
|
22
|
+
--
|
|
23
|
+
|
|
24
|
+
DROP TABLE /* TEMPLATE: schema */river_leader;
|
|
25
|
+
|
|
26
|
+
CREATE TABLE /* TEMPLATE: schema */river_leader (
|
|
27
|
+
elected_at timestamp NOT NULL,
|
|
28
|
+
expires_at timestamp NOT NULL,
|
|
29
|
+
leader_id text NOT NULL,
|
|
30
|
+
name text PRIMARY KEY NOT NULL DEFAULT 'default' CHECK (name = 'default'),
|
|
31
|
+
CONSTRAINT name_length CHECK (length(name) > 0 AND length(name) < 128),
|
|
32
|
+
CONSTRAINT leader_id_length CHECK (length(leader_id) > 0 AND length(leader_id) < 128)
|
|
33
|
+
);
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
--
|
|
2
|
+
-- Revert to migration table based only on `(version)`.
|
|
3
|
+
--
|
|
4
|
+
-- If any non-main migrations are present, 005 is considered irreversible.
|
|
5
|
+
--
|
|
6
|
+
|
|
7
|
+
ALTER TABLE /* TEMPLATE: schema */river_migration
|
|
8
|
+
RENAME TO river_migration_old;
|
|
9
|
+
|
|
10
|
+
CREATE TABLE /* TEMPLATE: schema */river_migration (
|
|
11
|
+
id integer PRIMARY KEY,
|
|
12
|
+
created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
13
|
+
version integer NOT NULL,
|
|
14
|
+
CONSTRAINT version CHECK (version >= 1)
|
|
15
|
+
);
|
|
16
|
+
|
|
17
|
+
CREATE UNIQUE INDEX /* TEMPLATE: schema */river_migration_version_idx ON river_migration (version);
|
|
18
|
+
|
|
19
|
+
INSERT INTO /* TEMPLATE: schema */river_migration
|
|
20
|
+
(created_at, version)
|
|
21
|
+
SELECT created_at, version
|
|
22
|
+
FROM /* TEMPLATE: schema */river_migration_old;
|
|
23
|
+
|
|
24
|
+
DROP TABLE /* TEMPLATE: schema */river_migration_old;
|
|
25
|
+
|
|
26
|
+
--
|
|
27
|
+
-- Normally, `unique_key` and an index are added here, but because SQLite was
|
|
28
|
+
-- added later, we've just pushed all of this to an initial `river_job` creation
|
|
29
|
+
-- in 006.
|
|
30
|
+
--
|
|
31
|
+
|
|
32
|
+
--
|
|
33
|
+
-- Drop `river_client` and derivative.
|
|
34
|
+
--
|
|
35
|
+
|
|
36
|
+
DROP TABLE /* TEMPLATE: schema */river_client_queue;
|
|
37
|
+
DROP TABLE /* TEMPLATE: schema */river_client;
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
--
|
|
2
|
+
-- Rebuild the migration table so it's based on `(line, version)`.
|
|
3
|
+
--
|
|
4
|
+
|
|
5
|
+
DROP INDEX /* TEMPLATE: schema */river_migration_version_idx;
|
|
6
|
+
|
|
7
|
+
ALTER TABLE /* TEMPLATE: schema */river_migration
|
|
8
|
+
RENAME TO river_migration_old;
|
|
9
|
+
|
|
10
|
+
CREATE TABLE /* TEMPLATE: schema */river_migration (
|
|
11
|
+
line text NOT NULL,
|
|
12
|
+
version integer NOT NULL,
|
|
13
|
+
created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
14
|
+
CONSTRAINT line_length CHECK (length(line) > 0 AND length(line) < 128),
|
|
15
|
+
CONSTRAINT version_gte_1 CHECK (version >= 1),
|
|
16
|
+
PRIMARY KEY (line, version)
|
|
17
|
+
);
|
|
18
|
+
|
|
19
|
+
INSERT INTO /* TEMPLATE: schema */river_migration
|
|
20
|
+
(created_at, line, version)
|
|
21
|
+
SELECT created_at, 'main', version
|
|
22
|
+
FROM /* TEMPLATE: schema */river_migration_old;
|
|
23
|
+
|
|
24
|
+
DROP TABLE /* TEMPLATE: schema */river_migration_old;
|
|
25
|
+
|
|
26
|
+
--
|
|
27
|
+
-- Normally, `unique_key` and an index are added here, but because SQLite was
|
|
28
|
+
-- added later, we've just pushed all of this to an initial `river_job` creation
|
|
29
|
+
-- in 006.
|
|
30
|
+
--
|
|
31
|
+
|
|
32
|
+
--
|
|
33
|
+
-- Create `river_client` and derivative.
|
|
34
|
+
--
|
|
35
|
+
-- This feature hasn't quite yet been implemented, but we're taking advantage of
|
|
36
|
+
-- the migration to add the schema early so that we can add it later without an
|
|
37
|
+
-- additional migration.
|
|
38
|
+
--
|
|
39
|
+
|
|
40
|
+
CREATE TABLE /* TEMPLATE: schema */river_client (
|
|
41
|
+
id text PRIMARY KEY NOT NULL,
|
|
42
|
+
created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
43
|
+
metadata blob NOT NULL DEFAULT (json('{}')),
|
|
44
|
+
paused_at timestamp,
|
|
45
|
+
updated_at timestamp NOT NULL,
|
|
46
|
+
CONSTRAINT name_length CHECK (length(id) > 0 AND length(id) < 128)
|
|
47
|
+
);
|
|
48
|
+
|
|
49
|
+
-- Differs from `river_queue` in that it tracks the queue state for a particular
|
|
50
|
+
-- active client.
|
|
51
|
+
CREATE TABLE /* TEMPLATE: schema */river_client_queue (
|
|
52
|
+
river_client_id text NOT NULL REFERENCES river_client (id) ON DELETE CASCADE,
|
|
53
|
+
name text NOT NULL,
|
|
54
|
+
created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
55
|
+
max_workers integer NOT NULL DEFAULT 0,
|
|
56
|
+
metadata blob NOT NULL DEFAULT (json('{}')),
|
|
57
|
+
num_jobs_completed integer NOT NULL DEFAULT 0,
|
|
58
|
+
num_jobs_running integer NOT NULL DEFAULT 0,
|
|
59
|
+
updated_at timestamp NOT NULL,
|
|
60
|
+
PRIMARY KEY (river_client_id, name),
|
|
61
|
+
CONSTRAINT name_length CHECK (length(name) > 0 AND length(name) < 128),
|
|
62
|
+
CONSTRAINT num_jobs_completed_zero_or_positive CHECK (num_jobs_completed >= 0),
|
|
63
|
+
CONSTRAINT num_jobs_running_zero_or_positive CHECK (num_jobs_running >= 0)
|
|
64
|
+
);
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
-- Only drops the trivial `river_job` we created in 002 which puts a placeholder
|
|
2
|
+
-- in place so that the right tables exist in the right versions. We don't
|
|
3
|
+
-- bother migrating any job data because it's not possible to have had any real
|
|
4
|
+
-- jobs by that point because this version (006) preexists the addition of SQLite.
|
|
5
|
+
DROP TABLE /* TEMPLATE: schema */river_job;
|
|
6
|
+
|
|
7
|
+
CREATE TABLE /* TEMPLATE: schema */river_job (
|
|
8
|
+
id integer PRIMARY KEY, -- SQLite aliases this to ROWID, which may reuse deleted IDs.
|
|
9
|
+
args blob NOT NULL DEFAULT '{}',
|
|
10
|
+
attempt integer NOT NULL DEFAULT 0,
|
|
11
|
+
attempted_at timestamp,
|
|
12
|
+
attempted_by blob, -- json
|
|
13
|
+
created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
14
|
+
errors blob, -- json
|
|
15
|
+
finalized_at timestamp,
|
|
16
|
+
kind text NOT NULL,
|
|
17
|
+
max_attempts integer NOT NULL,
|
|
18
|
+
metadata blob NOT NULL DEFAULT (json('{}')),
|
|
19
|
+
priority integer NOT NULL DEFAULT 1,
|
|
20
|
+
queue text NOT NULL DEFAULT 'default',
|
|
21
|
+
state text NOT NULL DEFAULT 'available',
|
|
22
|
+
scheduled_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
23
|
+
tags blob NOT NULL DEFAULT (json('[]')),
|
|
24
|
+
unique_key blob,
|
|
25
|
+
unique_states integer,
|
|
26
|
+
CONSTRAINT finalized_or_finalized_at_null CHECK (
|
|
27
|
+
(finalized_at IS NULL AND state NOT IN ('cancelled', 'completed', 'discarded')) OR
|
|
28
|
+
(finalized_at IS NOT NULL AND state IN ('cancelled', 'completed', 'discarded'))
|
|
29
|
+
),
|
|
30
|
+
CONSTRAINT priority_in_range CHECK (priority >= 1 AND priority <= 4),
|
|
31
|
+
CONSTRAINT queue_length CHECK (length(queue) > 0 AND length(queue) < 128),
|
|
32
|
+
CONSTRAINT kind_length CHECK (length(kind) > 0 AND length(kind) < 128),
|
|
33
|
+
CONSTRAINT state_valid CHECK (state IN ('available', 'cancelled', 'completed', 'discarded', 'pending', 'retryable', 'running', 'scheduled'))
|
|
34
|
+
);
|
|
35
|
+
|
|
36
|
+
-- All these indexes are normally brought up in version 002.
|
|
37
|
+
CREATE INDEX /* TEMPLATE: schema */river_job_kind ON river_job (kind);
|
|
38
|
+
CREATE INDEX /* TEMPLATE: schema */river_job_state_and_finalized_at_index ON river_job (state, finalized_at) WHERE finalized_at IS NOT NULL;
|
|
39
|
+
CREATE INDEX /* TEMPLATE: schema */river_job_prioritized_fetching_index ON river_job (state, queue, priority, scheduled_at, id);
|
|
40
|
+
|
|
41
|
+
-- Not raised because SQLite doesn't support Gin indexes. These aren't used in
|
|
42
|
+
-- River anyway.
|
|
43
|
+
-- CREATE INDEX river_job_args_index ON /* TEMPLATE: schema */river_job USING GIN(args);
|
|
44
|
+
-- CREATE INDEX river_job_metadata_index ON /* TEMPLATE: schema */river_job USING GIN(metadata);
|
|
45
|
+
|
|
46
|
+
-- SQLite doesn't support SQL functions, so where the bit extraction logic below
|
|
47
|
+
-- goes in the `river_job_state_in_bitmask` function in Postgres, here it's
|
|
48
|
+
-- baked right into the index. Use of helpers that don't exist in SQLite like
|
|
49
|
+
-- `get_bit` are also dropped by necessity.
|
|
50
|
+
CREATE UNIQUE INDEX /* TEMPLATE: schema */river_job_unique_idx ON river_job (unique_key)
|
|
51
|
+
WHERE unique_key IS NOT NULL
|
|
52
|
+
AND unique_states IS NOT NULL
|
|
53
|
+
AND CASE state
|
|
54
|
+
WHEN 'available' THEN unique_states & (1 << 0)
|
|
55
|
+
WHEN 'cancelled' THEN unique_states & (1 << 1)
|
|
56
|
+
WHEN 'completed' THEN unique_states & (1 << 2)
|
|
57
|
+
WHEN 'discarded' THEN unique_states & (1 << 3)
|
|
58
|
+
WHEN 'pending' THEN unique_states & (1 << 4)
|
|
59
|
+
WHEN 'retryable' THEN unique_states & (1 << 5)
|
|
60
|
+
WHEN 'running' THEN unique_states & (1 << 6)
|
|
61
|
+
WHEN 'scheduled' THEN unique_states & (1 << 7)
|
|
62
|
+
ELSE 0
|
|
63
|
+
END >= 1;
|