stampeded 0.4.0 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: aea73ceb5b2fa4853bf916936c6d8a2eb00ac163eeafc783c7e78c5cb484587d
-  data.tar.gz: 67aca9ad1c1f2a134ef2e6fb518845350de1db077fa5c2141b35f5f0a6def411
+  metadata.gz: d3ce5382689990aa3c0d7280ec0e86af16f18a7494a1fd96603bd6942c467c4b
+  data.tar.gz: 9e055b934cca7b96d9a7b464235d9d6b515937dc9c8d53618e2461a12d4134f4
 SHA512:
-  metadata.gz: 5e3d3f74a9dc65be10e925b600c643ab85aae2e2ee4916622742272fb11a581ab6a3dc0daa48f0e8f2e518d86ebed6d48bf1d73622d05fb560fd0302fb0f8bec
-  data.tar.gz: 1722d9b09bb76ac23e31abbb43ecb39b025c742931e2123caf528149d4b28216e70a3dfcda3c74a4d6ad4f5ceb203593dd11b727f9f50245b64257a505424603
+  metadata.gz: aaa523d7d9dd7d567bd1818da6a9c0064b8cc1582f76cb9279ce6f2b2b7c209f75e21aa36185f4e8a0faef2033f1f732d3c48846232f2576ffec25729ad271c4
+  data.tar.gz: dbba8d9afc205878ae795aad19953daaabb080c593769c30ab54f2c363f6d06abff8e0c41d7592a9b31e0ebe12c300df86fed9caaa4d6bc7f2de47b499f039e0

data/lib/stampeded/schema.rb

@@ -1,8 +1,11 @@
 # frozen_string_literal: true
 
+require "digest"
+
 module Stampeded
   module Schema
-    FILES = %w[schema.sql].freeze
+    # Where the daemon's sqlx migrator records what it applied.
+    TABLE = "stampeded_migrations"
 
     module_function
 
@@ -15,15 +18,40 @@ module Stampeded
         raise(Error, "cannot find schema/*.sql (looked in #{candidates.join(", ")})")
     end
 
-    def sql
-      FILES.map { |f| File.read(File.join(root, f)) }.join("\n")
+    # [version, description, sql] per migration, in order: sqlx's names.
+    def migrations
+      Dir[File.join(root, "*.sql")].sort.map do |path|
+        version, name = File.basename(path, ".sql").split("_", 2)
+        [Integer(version, 10), name.tr("_", " "), File.read(path)]
+      end
     end
 
+    def sql = migrations.map(&:last).join("\n")
+
+    # A fresh install, recorded as sqlx records it, so the daemon's
+    # --migrate finds nothing to apply.
     def install!(conn = nil)
       run = lambda do |c|
         c.exec("BEGIN")
         begin
-          c.exec(sql)
+          c.exec(<<~SQL)
+            CREATE TABLE #{TABLE} (
+                version BIGINT PRIMARY KEY,
+                description TEXT NOT NULL,
+                installed_on TIMESTAMPTZ NOT NULL DEFAULT now(),
+                success BOOLEAN NOT NULL,
+                checksum BYTEA NOT NULL,
+                execution_time BIGINT NOT NULL
+            )
+          SQL
+          migrations.each do |version, description, sql|
+            c.exec(sql)
+            c.exec_params(
+              "INSERT INTO #{TABLE} (version, description, success, checksum, execution_time) " \
+              "VALUES ($1, $2, true, decode($3, 'hex'), 0)",
+              [version, description, Digest::SHA384.hexdigest(sql)]
+            )
+          end
           c.exec("COMMIT")
         rescue StandardError
           c.exec("ROLLBACK")
@@ -40,7 +68,7 @@ module Stampeded
           DROP TABLE IF EXISTS stampeded_cron_occurrences, stampeded_schedules,
                                stampeded_queue_policies, stampeded_queue_counts,
                                stampeded_jobs, stampeded_workers,
-                               stampeded_schema CASCADE;
+                               stampeded_schema, stampeded_migrations CASCADE;
           DO $$
           DECLARE r record;
           BEGIN

data/lib/stampeded/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Stampeded
-  VERSION = "0.4.0" # x-release-please-version
+  VERSION = "0.4.1" # x-release-please-version
 end

data/schema/{schema.sql → 0001_init.sql}

@@ -65,20 +65,15 @@ CREATE TABLE stampeded_jobs (
     dedup_period    text        CHECK (dedup_period IN
                                   ('minute','hour','day','week','month')),
 
-    -- Scope, period and bucket: buckets of two periods start together (an
-    -- hour and its day at midnight UTC), so the period keeps them apart.
-    -- Joined as bytea with a raw zero byte: no `text` value can ever decode
-    -- to one (chr(0) itself is refused), so a crafted dedup_scope can never
-    -- forge the separator and so never forge another row's key.
     dedup_key       bytea       GENERATED ALWAYS AS (
-        sha256(
-            dedup_scope::bytea || '\x00'::bytea || coalesce(
-                dedup_period::bytea || '\x00'::bytea || extract(
+        sha256((
+            dedup_scope || coalesce(
+                ':' || extract(
                     epoch FROM (date_trunc(dedup_period, inserted_at, 'UTC') - to_timestamp(0))
-                )::bigint::text::bytea,
-                ''::bytea
+                )::bigint::text,
+                ''
             )
-        )
+        )::bytea)
     ) STORED,
 
     failure_kind    text,
@@ -1052,13 +1047,3 @@ CREATE TABLE stampeded_queue_policies (
     created_at     timestamptz NOT NULL DEFAULT now(),
     updated_at     timestamptz NOT NULL DEFAULT now()
 );
-
--- Which schema this database holds, one row: `stampeded::schema::migrate`
--- upgrades from it and refuses one newer than itself. Raised by each upgrade.
-CREATE TABLE stampeded_schema (
-    version     integer     NOT NULL CHECK (version > 0),
-    changed_at  timestamptz NOT NULL DEFAULT now()
-);
-CREATE UNIQUE INDEX stampeded_schema_one_row ON stampeded_schema ((true));
--- Must match schema::VERSION.
-INSERT INTO stampeded_schema (version) VALUES (3);

data/schema/0002_dedup_key_period.sql

@@ -0,0 +1,14 @@
+-- Schema 2: the dedup key names its period. Without it, an hourly and a
+-- daily unique job of the same scope shared a key whenever their buckets
+-- started together (every midnight UTC, Mondays, the 1st of a month) and
+-- refused each other. The new key is strictly finer: no row can collide.
+ALTER TABLE stampeded_jobs ALTER COLUMN dedup_key SET EXPRESSION AS (
+    sha256((
+        dedup_scope || coalesce(
+            ':' || dedup_period || ':' || extract(
+                epoch FROM (date_trunc(dedup_period, inserted_at, 'UTC') - to_timestamp(0))
+            )::bigint::text,
+            ''
+        )
+    )::bytea)
+);

data/schema/0003_dedup_key_separator.sql

@@ -0,0 +1,17 @@
+-- Schema 3: the dedup key joins scope/period/bucket as bytea with a raw
+-- zero byte. The old ':' join let a crafted dedup_scope (e.g.
+-- 'a:hour:1700000000') forge the exact key of an unrelated periodic job
+-- (scope 'a', period 'hour', that epoch) -- same sha256, same unique index,
+-- wrongly rejected as a duplicate. No `text` value can ever decode to a
+-- zero byte (chr(0) itself is refused by Postgres), so the new separator
+-- can never be forged.
+ALTER TABLE stampeded_jobs ALTER COLUMN dedup_key SET EXPRESSION AS (
+    sha256(
+        dedup_scope::bytea || '\x00'::bytea || coalesce(
+            dedup_period::bytea || '\x00'::bytea || extract(
+                epoch FROM (date_trunc(dedup_period, inserted_at, 'UTC') - to_timestamp(0))
+            )::bigint::text::bytea,
+            ''::bytea
+        )
+    )
+);

data/schema/0004_dedup_key_raw_scope.sql

@@ -0,0 +1,12 @@
+-- Backslashes are doubled so `::bytea` keeps the scope's own bytes instead
+-- of reading escapes: `a` and `\x61` are distinct scopes.
+ALTER TABLE stampeded_jobs ALTER COLUMN dedup_key SET EXPRESSION AS (
+    sha256(
+        replace(dedup_scope, '\', '\\')::bytea || '\x00'::bytea || coalesce(
+            dedup_period::bytea || '\x00'::bytea || extract(
+                epoch FROM (date_trunc(dedup_period, inserted_at, 'UTC') - to_timestamp(0))
+            )::bigint::text::bytea,
+            ''::bytea
+        )
+    )
+);

data/schema/0005_schema_version_guard.sql

@@ -0,0 +1,9 @@
+-- stampeded 0.4 reads this and refuses a version above 3, rather than
+-- mistaking this database for an unversioned one and rewriting it.
+CREATE TABLE IF NOT EXISTS stampeded_schema (
+    version     integer     NOT NULL CHECK (version > 0),
+    changed_at  timestamptz NOT NULL DEFAULT now()
+);
+CREATE UNIQUE INDEX IF NOT EXISTS stampeded_schema_one_row ON stampeded_schema ((true));
+DELETE FROM stampeded_schema;
+INSERT INTO stampeded_schema (version) VALUES (5);

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: stampeded
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.4.1
 platform: ruby
 authors:
 - Abdelkader Boudih
@@ -81,7 +81,11 @@ files:
 - lib/stampeded/tasks/stampeded.rake
 - lib/stampeded/version.rb
 - lib/stampeded/worker.rb
-- schema/schema.sql
+- schema/0001_init.sql
+- schema/0002_dedup_key_period.sql
+- schema/0003_dedup_key_separator.sql
+- schema/0004_dedup_key_raw_scope.sql
+- schema/0005_schema_version_guard.sql
 homepage: https://github.com/seuros/stampeded
 licenses:
 - BSD-3-Clause