riverqueue 0.11.0 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +132 -0
  3. data/LICENSE +374 -0
  4. data/docs/README.md +765 -0
  5. data/docs/migrations.md +121 -0
  6. data/docs/testing.md +173 -0
  7. data/docs/workers.md +126 -0
  8. data/exe/river +7 -0
  9. data/lib/cli.rb +55 -0
  10. data/lib/client.rb +314 -54
  11. data/lib/client_runtime.rb +612 -0
  12. data/lib/config.rb +154 -0
  13. data/lib/driver/job_row_decoder.rb +85 -0
  14. data/lib/driver/postgres_capabilities.rb +41 -0
  15. data/lib/driver/runtime.rb +661 -0
  16. data/lib/driver.rb +13 -6
  17. data/lib/errors.rb +83 -0
  18. data/lib/event.rb +78 -0
  19. data/lib/insert_opts.rb +40 -11
  20. data/lib/job.rb +28 -13
  21. data/lib/job_persisted_logging/plugin.rb +108 -0
  22. data/lib/migration_cli.rb +90 -0
  23. data/lib/migrator.rb +189 -0
  24. data/lib/params.rb +96 -0
  25. data/lib/periodic_cron.rb +24 -0
  26. data/lib/periodic_job.rb +135 -0
  27. data/lib/resumable.rb +148 -0
  28. data/lib/riverqueue/testing/minitest.rb +18 -0
  29. data/lib/riverqueue/testing/rspec.rb +109 -0
  30. data/lib/riverqueue/testing.rb +134 -0
  31. data/lib/riverqueue.rb +16 -0
  32. data/lib/unique_args.rb +97 -0
  33. data/lib/unique_bitmask.rb +14 -12
  34. data/lib/worker.rb +152 -0
  35. data/lib/worker_runner.rb +106 -0
  36. data/migration/LICENSE +374 -0
  37. data/migration/README.md +11 -0
  38. data/migration/manifest.json +38 -0
  39. data/migration/postgresql/main/001_create_river_migration.down.sql +1 -0
  40. data/migration/postgresql/main/001_create_river_migration.up.sql +8 -0
  41. data/migration/postgresql/main/002_initial_schema.down.sql +5 -0
  42. data/migration/postgresql/main/002_initial_schema.up.sql +96 -0
  43. data/migration/postgresql/main/003_river_job_tags_non_null.down.sql +3 -0
  44. data/migration/postgresql/main/003_river_job_tags_non_null.up.sql +3 -0
  45. data/migration/postgresql/main/004_pending_and_more.down.sql +42 -0
  46. data/migration/postgresql/main/004_pending_and_more.up.sql +45 -0
  47. data/migration/postgresql/main/005_migration_unique_client.down.sql +57 -0
  48. data/migration/postgresql/main/005_migration_unique_client.up.sql +79 -0
  49. data/migration/postgresql/main/006_bulk_unique.down.sql +16 -0
  50. data/migration/postgresql/main/006_bulk_unique.up.sql +40 -0
  51. data/migration/postgresql/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.down.sql +56 -0
  52. data/migration/postgresql/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.up.sql +44 -0
  53. data/migration/postgresql/main/008_job_id_autoincrement.down.sql +3 -0
  54. data/migration/postgresql/main/008_job_id_autoincrement.up.sql +3 -0
  55. data/migration/sqlite/main/001_create_river_migration.down.sql +1 -0
  56. data/migration/sqlite/main/001_create_river_migration.up.sql +8 -0
  57. data/migration/sqlite/main/002_initial_schema.down.sql +8 -0
  58. data/migration/sqlite/main/002_initial_schema.up.sql +19 -0
  59. data/migration/sqlite/main/003_river_job_tags_non_null.down.sql +6 -0
  60. data/migration/sqlite/main/003_river_job_tags_non_null.up.sql +6 -0
  61. data/migration/sqlite/main/004_pending_and_more.down.sql +26 -0
  62. data/migration/sqlite/main/004_pending_and_more.up.sql +33 -0
  63. data/migration/sqlite/main/005_migration_unique_client.down.sql +37 -0
  64. data/migration/sqlite/main/005_migration_unique_client.up.sql +64 -0
  65. data/migration/sqlite/main/006_bulk_unique.down.sql +7 -0
  66. data/migration/sqlite/main/006_bulk_unique.up.sql +63 -0
  67. data/migration/sqlite/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.down.sql +255 -0
  68. data/migration/sqlite/main/007_notification_outbox_sqlite_jsonb_and_sql_cleanup.up.sql +261 -0
  69. data/migration/sqlite/main/008_job_id_autoincrement.down.sql +121 -0
  70. data/migration/sqlite/main/008_job_id_autoincrement.up.sql +123 -0
  71. data/sig/client.rbs +95 -0
  72. data/sig/driver.rbs +170 -0
  73. data/sig/insert_opts.rbs +25 -0
  74. data/sig/job.rbs +72 -0
  75. data/sig/job_persisted_logging/plugin.rbs +29 -0
  76. data/sig/migrator.rbs +44 -0
  77. data/sig/riverqueue.rbs +2 -0
  78. data/sig/runtime.rbs +382 -0
  79. data/sig/testing.rbs +95 -0
  80. data/sig/unique_args.rbs +8 -0
  81. data/sig/unique_bitmask.rbs +8 -0
  82. data/sig/worker_runner.rbs +22 -0
  83. metadata +165 -10
@@ -0,0 +1,57 @@
1
+ --
2
+ -- Revert to migration table based only on `(version)`.
3
+ --
4
+ -- If any non-main migrations are present, 005 is considered irreversible.
5
+ --
6
+
7
+ DO
8
+ $body$
9
+ BEGIN
10
+ -- Tolerate users who may be using their own migration system rather than
11
+ -- River's. If they are, they will have skipped version 001 containing
12
+ -- `CREATE TABLE river_migration`, so this table won't exist.
13
+ IF (SELECT to_regclass('/* TEMPLATE: schema */river_migration') IS NOT NULL) THEN
14
+ IF EXISTS (
15
+ SELECT *
16
+ FROM /* TEMPLATE: schema */river_migration
17
+ WHERE line <> 'main'
18
+ ) THEN
19
+ RAISE EXCEPTION 'Found non-main migration lines in the database; version 005 migration is irreversible because it would result in loss of migration information.';
20
+ END IF;
21
+
22
+ ALTER TABLE /* TEMPLATE: schema */river_migration
23
+ RENAME TO river_migration_old;
24
+
25
+ CREATE TABLE /* TEMPLATE: schema */river_migration(
26
+ id bigserial PRIMARY KEY,
27
+ created_at timestamptz NOT NULL DEFAULT NOW(),
28
+ version bigint NOT NULL,
29
+ CONSTRAINT version CHECK (version >= 1)
30
+ );
31
+
32
+ CREATE UNIQUE INDEX ON /* TEMPLATE: schema */river_migration USING btree(version);
33
+
34
+ INSERT INTO /* TEMPLATE: schema */river_migration
35
+ (created_at, version)
36
+ SELECT created_at, version
37
+ FROM /* TEMPLATE: schema */river_migration_old;
38
+
39
+ DROP TABLE /* TEMPLATE: schema */river_migration_old;
40
+ END IF;
41
+ END;
42
+ $body$
43
+ LANGUAGE 'plpgsql';
44
+
45
+ --
46
+ -- Drop `river_job.unique_key`.
47
+ --
48
+
49
+ ALTER TABLE /* TEMPLATE: schema */river_job
50
+ DROP COLUMN unique_key;
51
+
52
+ --
53
+ -- Drop `river_client` and derivative.
54
+ --
55
+
56
+ DROP TABLE /* TEMPLATE: schema */river_client_queue;
57
+ DROP TABLE /* TEMPLATE: schema */river_client;
@@ -0,0 +1,79 @@
1
+ --
2
+ -- Rebuild the migration table so it's based on `(line, version)`.
3
+ --
4
+
5
+ DO
6
+ $body$
7
+ BEGIN
8
+ -- Tolerate users who may be using their own migration system rather than
9
+ -- River's. If they are, they will have skipped version 001 containing
10
+ -- `CREATE TABLE river_migration`, so this table won't exist.
11
+ IF (SELECT to_regclass('/* TEMPLATE: schema */river_migration') IS NOT NULL) THEN
12
+ ALTER TABLE /* TEMPLATE: schema */river_migration
13
+ RENAME TO river_migration_old;
14
+
15
+ CREATE TABLE /* TEMPLATE: schema */river_migration(
16
+ line TEXT NOT NULL,
17
+ version bigint NOT NULL,
18
+ created_at timestamptz NOT NULL DEFAULT NOW(),
19
+ CONSTRAINT line_length CHECK (char_length(line) > 0 AND char_length(line) < 128),
20
+ CONSTRAINT version_gte_1 CHECK (version >= 1),
21
+ PRIMARY KEY (line, version)
22
+ );
23
+
24
+ INSERT INTO /* TEMPLATE: schema */river_migration
25
+ (created_at, line, version)
26
+ SELECT created_at, 'main', version
27
+ FROM /* TEMPLATE: schema */river_migration_old;
28
+
29
+ DROP TABLE /* TEMPLATE: schema */river_migration_old;
30
+ END IF;
31
+ END;
32
+ $body$
33
+ LANGUAGE 'plpgsql';
34
+
35
+ --
36
+ -- Add `river_job.unique_key` and bring up an index on it.
37
+ --
38
+
39
+ -- These statements use `IF NOT EXISTS` to allow users with a `river_job` table
40
+ -- of non-trivial size to build the index `CONCURRENTLY` out of band of this
41
+ -- migration, then follow by completing the migration.
42
+ ALTER TABLE /* TEMPLATE: schema */river_job
43
+ ADD COLUMN IF NOT EXISTS unique_key bytea;
44
+
45
+ CREATE UNIQUE INDEX IF NOT EXISTS river_job_kind_unique_key_idx ON /* TEMPLATE: schema */river_job (kind, unique_key) WHERE unique_key IS NOT NULL;
46
+
47
+ --
48
+ -- Create `river_client` and derivative.
49
+ --
50
+ -- This feature hasn't quite yet been implemented, but we're taking advantage of
51
+ -- the migration to add the schema early so that we can add it later without an
52
+ -- additional migration.
53
+ --
54
+
55
+ CREATE UNLOGGED TABLE /* TEMPLATE: schema */river_client (
56
+ id text PRIMARY KEY NOT NULL,
57
+ created_at timestamptz NOT NULL DEFAULT now(),
58
+ metadata jsonb NOT NULL DEFAULT '{}',
59
+ paused_at timestamptz,
60
+ updated_at timestamptz NOT NULL,
61
+ CONSTRAINT name_length CHECK (char_length(id) > 0 AND char_length(id) < 128)
62
+ );
63
+
64
+ -- Differs from `river_queue` in that it tracks the queue state for a particular
65
+ -- active client.
66
+ CREATE UNLOGGED TABLE /* TEMPLATE: schema */river_client_queue (
67
+ river_client_id text NOT NULL REFERENCES /* TEMPLATE: schema */river_client (id) ON DELETE CASCADE,
68
+ name text NOT NULL,
69
+ created_at timestamptz NOT NULL DEFAULT now(),
70
+ max_workers bigint NOT NULL DEFAULT 0,
71
+ metadata jsonb NOT NULL DEFAULT '{}',
72
+ num_jobs_completed bigint NOT NULL DEFAULT 0,
73
+ num_jobs_running bigint NOT NULL DEFAULT 0,
74
+ updated_at timestamptz NOT NULL,
75
+ PRIMARY KEY (river_client_id, name),
76
+ CONSTRAINT name_length CHECK (char_length(name) > 0 AND char_length(name) < 128),
77
+ CONSTRAINT num_jobs_completed_zero_or_positive CHECK (num_jobs_completed >= 0),
78
+ CONSTRAINT num_jobs_running_zero_or_positive CHECK (num_jobs_running >= 0)
79
+ );
@@ -0,0 +1,16 @@
1
+
2
+ --
3
+ -- Drop `river_job.unique_states` and its index.
4
+ --
5
+
6
+ DROP INDEX /* TEMPLATE: schema */river_job_unique_idx;
7
+
8
+ ALTER TABLE /* TEMPLATE: schema */river_job
9
+ DROP COLUMN unique_states;
10
+
11
+ CREATE UNIQUE INDEX IF NOT EXISTS river_job_kind_unique_key_idx ON /* TEMPLATE: schema */river_job (kind, unique_key) WHERE unique_key IS NOT NULL;
12
+
13
+ --
14
+ -- Drop `river_job_state_in_bitmask` function.
15
+ --
16
+ DROP FUNCTION /* TEMPLATE: schema */river_job_state_in_bitmask;
@@ -0,0 +1,40 @@
1
+ CREATE OR REPLACE FUNCTION /* TEMPLATE: schema */river_job_state_in_bitmask(bitmask BIT(8), state /* TEMPLATE: schema */river_job_state)
2
+ RETURNS boolean
3
+ LANGUAGE SQL
4
+ IMMUTABLE
5
+ AS $$
6
+ SELECT CASE state
7
+ WHEN 'available' THEN get_bit(bitmask, 7)
8
+ WHEN 'cancelled' THEN get_bit(bitmask, 6)
9
+ WHEN 'completed' THEN get_bit(bitmask, 5)
10
+ WHEN 'discarded' THEN get_bit(bitmask, 4)
11
+ WHEN 'pending' THEN get_bit(bitmask, 3)
12
+ WHEN 'retryable' THEN get_bit(bitmask, 2)
13
+ WHEN 'running' THEN get_bit(bitmask, 1)
14
+ WHEN 'scheduled' THEN get_bit(bitmask, 0)
15
+ ELSE 0
16
+ END = 1;
17
+ $$;
18
+
19
+ --
20
+ -- Add `river_job.unique_states` and bring up an index on it.
21
+ --
22
+ -- This column may exist already if users manually created the column and index
23
+ -- as instructed in the changelog so the index could be created `CONCURRENTLY`.
24
+ --
25
+ ALTER TABLE /* TEMPLATE: schema */river_job ADD COLUMN IF NOT EXISTS unique_states BIT(8);
26
+
27
+ -- This statement uses `IF NOT EXISTS` to allow users with a `river_job` table
28
+ -- of non-trivial size to build the index `CONCURRENTLY` out of band of this
29
+ -- migration, then follow by completing the migration.
30
+ CREATE UNIQUE INDEX IF NOT EXISTS river_job_unique_idx ON /* TEMPLATE: schema */river_job (unique_key)
31
+ WHERE unique_key IS NOT NULL
32
+ AND unique_states IS NOT NULL
33
+ AND /* TEMPLATE: schema */river_job_state_in_bitmask(unique_states, state);
34
+
35
+ -- Remove the old unique index. Users who are actively using the unique jobs
36
+ -- feature and who wish to avoid deploy downtime may want od drop this in a
37
+ -- subsequent migration once all jobs using the old unique system have been
38
+ -- completed (i.e. no more rows with non-null unique_key and null
39
+ -- unique_states).
40
+ DROP INDEX /* TEMPLATE: schema */river_job_kind_unique_key_idx;
@@ -0,0 +1,56 @@
1
+ --
2
+ -- SQL cleanup rollback.
3
+ --
4
+
5
+ --
6
+ -- Add back unused tables `river_client` and `river_client_queue`.
7
+ --
8
+
9
+ CREATE UNLOGGED TABLE /* TEMPLATE: schema */river_client (
10
+ id text PRIMARY KEY NOT NULL,
11
+ created_at timestamptz NOT NULL DEFAULT now(),
12
+ metadata jsonb NOT NULL DEFAULT '{}',
13
+ paused_at timestamptz,
14
+ updated_at timestamptz NOT NULL,
15
+ CONSTRAINT name_length CHECK (char_length(id) > 0 AND char_length(id) < 128)
16
+ );
17
+
18
+ CREATE UNLOGGED TABLE /* TEMPLATE: schema */river_client_queue (
19
+ river_client_id text NOT NULL REFERENCES /* TEMPLATE: schema */river_client (id) ON DELETE CASCADE,
20
+ name text NOT NULL,
21
+ created_at timestamptz NOT NULL DEFAULT now(),
22
+ max_workers bigint NOT NULL DEFAULT 0,
23
+ metadata jsonb NOT NULL DEFAULT '{}',
24
+ num_jobs_completed bigint NOT NULL DEFAULT 0,
25
+ num_jobs_running bigint NOT NULL DEFAULT 0,
26
+ updated_at timestamptz NOT NULL,
27
+ PRIMARY KEY (river_client_id, name),
28
+ CONSTRAINT name_length CHECK (char_length(name) > 0 AND char_length(name) < 128),
29
+ CONSTRAINT num_jobs_completed_zero_or_positive CHECK (num_jobs_completed >= 0),
30
+ CONSTRAINT num_jobs_running_zero_or_positive CHECK (num_jobs_running >= 0)
31
+ );
32
+
33
+ --
34
+ -- Revert addition of `DEFAULT 25` to `river_job.max_attempts`.
35
+ --
36
+
37
+ ALTER TABLE /* TEMPLATE: schema */river_job
38
+ ALTER COLUMN max_attempts DROP DEFAULT;
39
+
40
+ --
41
+ -- Changes `river_queue.updated_at` to revert the default of `CURRENT_TIMESTAMP`.
42
+ --
43
+
44
+ ALTER TABLE /* TEMPLATE: schema */river_queue
45
+ ALTER COLUMN updated_at DROP DEFAULT;
46
+
47
+ --
48
+ -- SQLite JSONB conversion rollback.
49
+ --
50
+ -- No-op. PostgreSQL already stores River JSON columns as jsonb.
51
+
52
+ --
53
+ -- Notification outbox rollback.
54
+ --
55
+
56
+ DROP TABLE /* TEMPLATE: schema */river_notification;
@@ -0,0 +1,44 @@
1
+ --
2
+ -- Notification outbox.
3
+ --
4
+
5
+ CREATE TABLE /* TEMPLATE: schema */river_notification (
6
+ id bigserial PRIMARY KEY,
7
+ created_at timestamptz NOT NULL DEFAULT now(),
8
+ payload text NOT NULL,
9
+ topic text NOT NULL,
10
+ CONSTRAINT topic_length CHECK (length(topic) > 0 AND length(topic) < 128)
11
+ );
12
+
13
+ CREATE INDEX river_notification_created_at_idx ON /* TEMPLATE: schema */river_notification (created_at);
14
+ CREATE INDEX river_notification_topic_id_idx ON /* TEMPLATE: schema */river_notification (topic, id);
15
+
16
+ --
17
+ -- SQLite JSONB conversion.
18
+ --
19
+ -- No-op. PostgreSQL already stores River JSON columns as jsonb.
20
+
21
+ --
22
+ -- SQL cleanup.
23
+ --
24
+
25
+ --
26
+ -- Drop unused tables `river_client` and `river_client_queue`.
27
+ --
28
+
29
+ DROP TABLE /* TEMPLATE: schema */river_client_queue;
30
+ DROP TABLE /* TEMPLATE: schema */river_client;
31
+
32
+ --
33
+ -- Adds `DEFAULT 25` to `river_job.max_attempts`.
34
+ --
35
+
36
+ ALTER TABLE /* TEMPLATE: schema */river_job
37
+ ALTER COLUMN max_attempts SET DEFAULT 25;
38
+
39
+ --
40
+ -- Changes `river_queue.updated_at` to have a default of `CURRENT_TIMESTAMP`.
41
+ --
42
+
43
+ ALTER TABLE /* TEMPLATE: schema */river_queue
44
+ ALTER COLUMN updated_at SET DEFAULT CURRENT_TIMESTAMP;
@@ -0,0 +1,3 @@
1
+ -- No-op. PostgreSQL sequences already prevent automatically generated job IDs
2
+ -- from being reused.
3
+ SELECT 1;
@@ -0,0 +1,3 @@
1
+ -- No-op. PostgreSQL sequences already prevent automatically generated job IDs
2
+ -- from being reused.
3
+ SELECT 1;
@@ -0,0 +1 @@
1
+ DROP TABLE /* TEMPLATE: schema */river_migration;
@@ -0,0 +1,8 @@
1
+ CREATE TABLE /* TEMPLATE: schema */river_migration (
2
+ id integer PRIMARY KEY,
3
+ created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
4
+ version integer NOT NULL,
5
+ CONSTRAINT version CHECK (version >= 1)
6
+ );
7
+
8
+ CREATE UNIQUE INDEX /* TEMPLATE: schema */river_migration_version_idx ON river_migration (version);
@@ -0,0 +1,8 @@
1
+ --
2
+ -- Normally `river_job` and `river_job_notify()` are dropped here, but since
3
+ -- SQLite was added well after 002 came about, we push that to version 006 index.
4
+ --
5
+
6
+ DROP TABLE /* TEMPLATE: schema */river_job;
7
+
8
+ DROP TABLE /* TEMPLATE: schema */river_leader;
@@ -0,0 +1,19 @@
1
+ --
2
+ -- Normally `river_job` and `river_job_notify()` are raised here, but since
3
+ -- SQLite was added well after 002 came about, we push that to version 006 index.
4
+ --
5
+
6
+ -- Dummy `river_job` table so that there's something to truncate in tests when
7
+ -- migrated to this version specifically.
8
+ CREATE TABLE /* TEMPLATE: schema */river_job (
9
+ id integer PRIMARY KEY
10
+ );
11
+
12
+ CREATE TABLE /* TEMPLATE: schema */river_leader (
13
+ elected_at timestamp NOT NULL,
14
+ expires_at timestamp NOT NULL,
15
+ leader_id text NOT NULL,
16
+ name text PRIMARY KEY NOT NULL,
17
+ CONSTRAINT name_length CHECK (length(name) > 0 AND length(name) < 128),
18
+ CONSTRAINT leader_id_length CHECK (length(leader_id) > 0 AND length(leader_id) < 128)
19
+ );
@@ -0,0 +1,6 @@
1
+ --
2
+ -- Normally `river_job.tags` is set back to nullable here, but since SQLite was
3
+ -- added well after 003 came about, we push that to version 006 index.
4
+ --
5
+
6
+ SELECT 1;
@@ -0,0 +1,6 @@
1
+ --
2
+ -- Normally `river_job.tags` is set to `NOT NULL` with a `DEFAULT` here, but since
3
+ -- SQLite was added well after 003 came about, we push that to version 006 index.
4
+ --
5
+
6
+ SELECT 1;
@@ -0,0 +1,26 @@
1
+ --
2
+ -- Normally, args and metadata both become `NOT NULL`, `pending` is added, and
3
+ -- the constraint `finalized_at` is changed, but because SQLite was added later,
4
+ -- we've just pushed all of this to an initial `river_job` creation in 006.
5
+ --
6
+
7
+ --
8
+ -- Drop `river_queue`.
9
+ --
10
+
11
+ DROP TABLE /* TEMPLATE: schema */river_queue;
12
+
13
+ --
14
+ -- Reverse changes to `river_leader`.
15
+ --
16
+
17
+ DROP TABLE /* TEMPLATE: schema */river_leader;
18
+
19
+ CREATE TABLE /* TEMPLATE: schema */river_leader (
20
+ elected_at timestamp NOT NULL,
21
+ expires_at timestamp NOT NULL,
22
+ leader_id text NOT NULL,
23
+ name text PRIMARY KEY NOT NULL,
24
+ CONSTRAINT name_length CHECK (length(name) > 0 AND length(name) < 128),
25
+ CONSTRAINT leader_id_length CHECK (length(leader_id) > 0 AND length(leader_id) < 128)
26
+ );
@@ -0,0 +1,33 @@
1
+ --
2
+ -- Normally, args and metadata both become `NOT NULL`, `pending` is added, and
3
+ -- the constraint `finalized_at` is changed, but because SQLite was added later,
4
+ -- we've just pushed all of this to an initial `river_job` creation in 006.
5
+ --
6
+
7
+ --
8
+ -- Create table `river_queue`.
9
+ --
10
+
11
+ CREATE TABLE /* TEMPLATE: schema */river_queue (
12
+ name text PRIMARY KEY NOT NULL,
13
+ created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
14
+ metadata blob NOT NULL DEFAULT (json('{}')),
15
+ paused_at timestamp,
16
+ updated_at timestamp NOT NULL
17
+ );
18
+
19
+ --
20
+ -- Alter `river_leader` to add a default value of 'default` to `name`. SQLite
21
+ -- doesn't allow schema modifications, so this redefines the table entirely.
22
+ --
23
+
24
+ DROP TABLE /* TEMPLATE: schema */river_leader;
25
+
26
+ CREATE TABLE /* TEMPLATE: schema */river_leader (
27
+ elected_at timestamp NOT NULL,
28
+ expires_at timestamp NOT NULL,
29
+ leader_id text NOT NULL,
30
+ name text PRIMARY KEY NOT NULL DEFAULT 'default' CHECK (name = 'default'),
31
+ CONSTRAINT name_length CHECK (length(name) > 0 AND length(name) < 128),
32
+ CONSTRAINT leader_id_length CHECK (length(leader_id) > 0 AND length(leader_id) < 128)
33
+ );
@@ -0,0 +1,37 @@
1
+ --
2
+ -- Revert to migration table based only on `(version)`.
3
+ --
4
+ -- If any non-main migrations are present, 005 is considered irreversible.
5
+ --
6
+
7
+ ALTER TABLE /* TEMPLATE: schema */river_migration
8
+ RENAME TO river_migration_old;
9
+
10
+ CREATE TABLE /* TEMPLATE: schema */river_migration (
11
+ id integer PRIMARY KEY,
12
+ created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
13
+ version integer NOT NULL,
14
+ CONSTRAINT version CHECK (version >= 1)
15
+ );
16
+
17
+ CREATE UNIQUE INDEX /* TEMPLATE: schema */river_migration_version_idx ON river_migration (version);
18
+
19
+ INSERT INTO /* TEMPLATE: schema */river_migration
20
+ (created_at, version)
21
+ SELECT created_at, version
22
+ FROM /* TEMPLATE: schema */river_migration_old;
23
+
24
+ DROP TABLE /* TEMPLATE: schema */river_migration_old;
25
+
26
+ --
27
+ -- Normally, `unique_key` and an index are added here, but because SQLite was
28
+ -- added later, we've just pushed all of this to an initial `river_job` creation
29
+ -- in 006.
30
+ --
31
+
32
+ --
33
+ -- Drop `river_client` and derivative.
34
+ --
35
+
36
+ DROP TABLE /* TEMPLATE: schema */river_client_queue;
37
+ DROP TABLE /* TEMPLATE: schema */river_client;
@@ -0,0 +1,64 @@
1
+ --
2
+ -- Rebuild the migration table so it's based on `(line, version)`.
3
+ --
4
+
5
+ DROP INDEX /* TEMPLATE: schema */river_migration_version_idx;
6
+
7
+ ALTER TABLE /* TEMPLATE: schema */river_migration
8
+ RENAME TO river_migration_old;
9
+
10
+ CREATE TABLE /* TEMPLATE: schema */river_migration (
11
+ line text NOT NULL,
12
+ version integer NOT NULL,
13
+ created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
14
+ CONSTRAINT line_length CHECK (length(line) > 0 AND length(line) < 128),
15
+ CONSTRAINT version_gte_1 CHECK (version >= 1),
16
+ PRIMARY KEY (line, version)
17
+ );
18
+
19
+ INSERT INTO /* TEMPLATE: schema */river_migration
20
+ (created_at, line, version)
21
+ SELECT created_at, 'main', version
22
+ FROM /* TEMPLATE: schema */river_migration_old;
23
+
24
+ DROP TABLE /* TEMPLATE: schema */river_migration_old;
25
+
26
+ --
27
+ -- Normally, `unique_key` and an index are added here, but because SQLite was
28
+ -- added later, we've just pushed all of this to an initial `river_job` creation
29
+ -- in 006.
30
+ --
31
+
32
+ --
33
+ -- Create `river_client` and derivative.
34
+ --
35
+ -- This feature hasn't quite yet been implemented, but we're taking advantage of
36
+ -- the migration to add the schema early so that we can add it later without an
37
+ -- additional migration.
38
+ --
39
+
40
+ CREATE TABLE /* TEMPLATE: schema */river_client (
41
+ id text PRIMARY KEY NOT NULL,
42
+ created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
43
+ metadata blob NOT NULL DEFAULT (json('{}')),
44
+ paused_at timestamp,
45
+ updated_at timestamp NOT NULL,
46
+ CONSTRAINT name_length CHECK (length(id) > 0 AND length(id) < 128)
47
+ );
48
+
49
+ -- Differs from `river_queue` in that it tracks the queue state for a particular
50
+ -- active client.
51
+ CREATE TABLE /* TEMPLATE: schema */river_client_queue (
52
+ river_client_id text NOT NULL REFERENCES river_client (id) ON DELETE CASCADE,
53
+ name text NOT NULL,
54
+ created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
55
+ max_workers integer NOT NULL DEFAULT 0,
56
+ metadata blob NOT NULL DEFAULT (json('{}')),
57
+ num_jobs_completed integer NOT NULL DEFAULT 0,
58
+ num_jobs_running integer NOT NULL DEFAULT 0,
59
+ updated_at timestamp NOT NULL,
60
+ PRIMARY KEY (river_client_id, name),
61
+ CONSTRAINT name_length CHECK (length(name) > 0 AND length(name) < 128),
62
+ CONSTRAINT num_jobs_completed_zero_or_positive CHECK (num_jobs_completed >= 0),
63
+ CONSTRAINT num_jobs_running_zero_or_positive CHECK (num_jobs_running >= 0)
64
+ );
@@ -0,0 +1,7 @@
1
+ DROP TABLE /* TEMPLATE: schema */river_job;
2
+
3
+ -- Dummy `river_job` table so that there's something to truncate in tests when
4
+ -- migrated to this version specifically.
5
+ CREATE TABLE /* TEMPLATE: schema */river_job (
6
+ id integer PRIMARY KEY
7
+ );
@@ -0,0 +1,63 @@
1
+ -- Only drops the trivial `river_job` we created in 002 which puts a placeholder
2
+ -- in place so that the right tables exist in the right versions. We don't
3
+ -- bother migrating any job data because it's not possible to have had any real
4
+ -- jobs by that point because this version (006) preexists the addition of SQLite.
5
+ DROP TABLE /* TEMPLATE: schema */river_job;
6
+
7
+ CREATE TABLE /* TEMPLATE: schema */river_job (
8
+ id integer PRIMARY KEY, -- SQLite aliases this to ROWID, which may reuse deleted IDs.
9
+ args blob NOT NULL DEFAULT '{}',
10
+ attempt integer NOT NULL DEFAULT 0,
11
+ attempted_at timestamp,
12
+ attempted_by blob, -- json
13
+ created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
14
+ errors blob, -- json
15
+ finalized_at timestamp,
16
+ kind text NOT NULL,
17
+ max_attempts integer NOT NULL,
18
+ metadata blob NOT NULL DEFAULT (json('{}')),
19
+ priority integer NOT NULL DEFAULT 1,
20
+ queue text NOT NULL DEFAULT 'default',
21
+ state text NOT NULL DEFAULT 'available',
22
+ scheduled_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
23
+ tags blob NOT NULL DEFAULT (json('[]')),
24
+ unique_key blob,
25
+ unique_states integer,
26
+ CONSTRAINT finalized_or_finalized_at_null CHECK (
27
+ (finalized_at IS NULL AND state NOT IN ('cancelled', 'completed', 'discarded')) OR
28
+ (finalized_at IS NOT NULL AND state IN ('cancelled', 'completed', 'discarded'))
29
+ ),
30
+ CONSTRAINT priority_in_range CHECK (priority >= 1 AND priority <= 4),
31
+ CONSTRAINT queue_length CHECK (length(queue) > 0 AND length(queue) < 128),
32
+ CONSTRAINT kind_length CHECK (length(kind) > 0 AND length(kind) < 128),
33
+ CONSTRAINT state_valid CHECK (state IN ('available', 'cancelled', 'completed', 'discarded', 'pending', 'retryable', 'running', 'scheduled'))
34
+ );
35
+
36
+ -- All these indexes are normally brought up in version 002.
37
+ CREATE INDEX /* TEMPLATE: schema */river_job_kind ON river_job (kind);
38
+ CREATE INDEX /* TEMPLATE: schema */river_job_state_and_finalized_at_index ON river_job (state, finalized_at) WHERE finalized_at IS NOT NULL;
39
+ CREATE INDEX /* TEMPLATE: schema */river_job_prioritized_fetching_index ON river_job (state, queue, priority, scheduled_at, id);
40
+
41
+ -- Not raised because SQLite doesn't support Gin indexes. These aren't used in
42
+ -- River anyway.
43
+ -- CREATE INDEX river_job_args_index ON /* TEMPLATE: schema */river_job USING GIN(args);
44
+ -- CREATE INDEX river_job_metadata_index ON /* TEMPLATE: schema */river_job USING GIN(metadata);
45
+
46
+ -- SQLite doesn't support SQL functions, so where the bit extraction logic below
47
+ -- goes in the `river_job_state_in_bitmask` function in Postgres, here it's
48
+ -- baked right into the index. Use of helpers that don't exist in SQLite like
49
+ -- `get_bit` are also dropped by necessity.
50
+ CREATE UNIQUE INDEX /* TEMPLATE: schema */river_job_unique_idx ON river_job (unique_key)
51
+ WHERE unique_key IS NOT NULL
52
+ AND unique_states IS NOT NULL
53
+ AND CASE state
54
+ WHEN 'available' THEN unique_states & (1 << 0)
55
+ WHEN 'cancelled' THEN unique_states & (1 << 1)
56
+ WHEN 'completed' THEN unique_states & (1 << 2)
57
+ WHEN 'discarded' THEN unique_states & (1 << 3)
58
+ WHEN 'pending' THEN unique_states & (1 << 4)
59
+ WHEN 'retryable' THEN unique_states & (1 << 5)
60
+ WHEN 'running' THEN unique_states & (1 << 6)
61
+ WHEN 'scheduled' THEN unique_states & (1 << 7)
62
+ ELSE 0
63
+ END >= 1;