stampeded 0.4.1 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d3ce5382689990aa3c0d7280ec0e86af16f18a7494a1fd96603bd6942c467c4b
-  data.tar.gz: 9e055b934cca7b96d9a7b464235d9d6b515937dc9c8d53618e2461a12d4134f4
+  metadata.gz: db2468be1b5b64e89bd00f2bebccbafce4faf21c875065d91d5f43a394bdbd9c
+  data.tar.gz: 5f5bc056343dff4ac2c83eec8eda56334e9d1423cc22acdae97aaba046f9672d
 SHA512:
-  metadata.gz: aaa523d7d9dd7d567bd1818da6a9c0064b8cc1582f76cb9279ce6f2b2b7c209f75e21aa36185f4e8a0faef2033f1f732d3c48846232f2576ffec25729ad271c4
-  data.tar.gz: dbba8d9afc205878ae795aad19953daaabb080c593769c30ab54f2c363f6d06abff8e0c41d7592a9b31e0ebe12c300df86fed9caaa4d6bc7f2de47b499f039e0
+  metadata.gz: e5d126e89b18e370b2b9cee271d025eb2f68a7967aaaea185423854d99f314c88266a06dbb7b54ee3b6b3659319ff7a169c553bd3f61f08e144b182b9b06cb01
+  data.tar.gz: e77790d2458e5be336950f1590958afc7ebc52bc03be7b9c0db0a40d173ddc91b0b8c4ac092cfb8606a6d5ecd79e54aa73a42f7fee4755aabeaa3ea895789c0a

data/README.md

@@ -135,7 +135,8 @@ Stampeded::Schedule.recurring(
 
 The daemon fires it; a new schedule first runs at its next occurrence. An
 expression that names its hours runs once on a DST fall-back day; one using `*`
-or a step runs on real elapsed time. `Stampeded::Schedule.pause`, `.remove` and
+or a step runs on real elapsed time. The queue is `queue:`, or the job's
+`queue_as` evaluated with `args`. `Stampeded::Schedule.pause`, `.remove` and
 `.all` manage them.
 
 ## Workers
@@ -164,8 +165,9 @@ config.except_queues = %w[payments_dlq]    # with :all
 - `SIGTERM` lets in-flight jobs finish before exiting. Jobs still running after
   `shutdown_timeout` are reclaimed by the daemon.
 - The worker runs on `libstampeded_ffi`, the same core every runtime's worker
-  uses; see [The binaries](#the-binaries). `STAMPEDED_LOG=1` logs each job;
-  `STAMPEDED_LOG=debug` (or `info`, `warn`, ...) also sets the core's level.
+  uses; see [The binaries](#the-binaries). Failures to claim or report go to
+  stderr always; `STAMPEDED_LOG=1` also logs each job, and
+  `STAMPEDED_LOG=debug` (or `info`, `warn`, ...) sets the core's level.
 
 ## Queue policies
 
@@ -178,6 +180,14 @@ The daemon turns these into a verdict per queue every second; the worker claims
 rushed queues first and skips held ones. `max_running` also limits what one
 worker process holds from that queue, with or without the daemon.
 
+```ruby
+Stampeded::QueuePolicy.pause("mail")   # no worker in any language claims from it
+Stampeded::QueuePolicy.resume("mail")
+```
+
+Jobs already running finish, and `perform_later` and schedules keep queueing
+behind the pause. `set` keeps a pause; `remove` lifts it.
+
 ## Without Rails
 
 ```ruby

data/lib/stampeded/envelope.rb

@@ -21,17 +21,22 @@ module Stampeded
     end
 
     def from_active_job(job)
-      task = job.class.try(:stampeded_task)
-      context = job.serialize.except("arguments", "provider_job_id")
-      kwargs = standard_arguments(job.arguments)
+      build(job.class, job.arguments, job.serialize.except("arguments", "provider_job_id"))
+    end
+
+    # [task, args, meta] for `arguments` as `perform` takes them: plain keywords
+    # become portable `args`, anything else rides in ActiveJob's own format.
+    def build(job_class, arguments, context)
+      task = job_class.try(:stampeded_task)
+      kwargs = standard_arguments(arguments)
 
       if kwargs
-        [task || job.class.name, kwargs, { "activejob" => context }]
+        [task || job_class.name, kwargs, { "activejob" => context }]
       elsif task
-        raise PortableArgumentsError.new(job.class.name, task)
+        raise PortableArgumentsError.new(job_class.name, task)
       else
-        context["arguments"] = ActiveJob::Arguments.serialize(job.arguments)
-        [job.class.name, {}, { "activejob" => context }]
+        context["arguments"] = ActiveJob::Arguments.serialize(arguments)
+        [job_class.name, {}, { "activejob" => context }]
       end
     end
 

data/lib/stampeded/lifecycle.rb

@@ -18,7 +18,8 @@ module Stampeded
 
     def events_from(state) = events.select { legal?(state, it) }
 
-    def terminal?(state) = events_from(state).empty?
+    # No worker moves it again; only an operator's resurrect does.
+    def terminal?(state) = events_from(state).all?(:resurrect)
 
     def claimable?(state) = legal?(state, :claim)
 

data/lib/stampeded/queue_policy.rb

@@ -21,6 +21,34 @@ module Stampeded
       queue.to_s
     end
 
+    # No worker, in any runtime, claims from a paused queue; running jobs finish
+    # and enqueueing carries on. `set` keeps a pause; `remove` lifts it.
+    def pause(queue)
+      raise Error, "a queue policy needs a queue name" if queue.to_s.strip.empty?
+
+      Connection.translating_errors do
+        Connection.with do |c|
+          c.exec_params(<<~SQL, [queue.to_s])
+            INSERT INTO stampeded_queue_policies (queue, paused) VALUES ($1, true)
+            ON CONFLICT (queue) DO UPDATE SET paused = true, updated_at = now()
+          SQL
+        end
+      end
+      queue.to_s
+    end
+
+    def resume(queue)
+      Connection.translating_errors do
+        Connection.with do |c|
+          c.exec_params(<<~SQL, [queue.to_s])
+            UPDATE stampeded_queue_policies SET paused = false, updated_at = now()
+             WHERE queue = $1 AND paused
+          SQL
+        end
+      end
+      queue.to_s
+    end
+
     def remove(queue)
       Connection.with { |c| c.exec_params("DELETE FROM stampeded_queue_policies WHERE queue = $1", [queue.to_s]) }
     end
@@ -29,7 +57,7 @@ module Stampeded
       Connection.translating_errors do
         Connection.with do |c|
           c.exec(<<~SQL).to_a
-            SELECT queue, max_running, max_wait_secs, verdict, reason, decided_at
+            SELECT queue, max_running, max_wait_secs, paused, verdict, reason, decided_at
               FROM stampeded_queue_policies ORDER BY queue
           SQL
         end

data/lib/stampeded/schedule.rb

@@ -13,9 +13,11 @@ module Stampeded
       suggested = check_cron!(cron, timezone)
       dst_policy ||= suggested
       job_class = job.is_a?(Class) ? job : job.class
-      queue ||= resolve_queue_name(job_class) || Stampeded.config.default_queue
+      # As `perform` takes them: a Hash is keywords, anything else positional.
+      arguments = args.is_a?(Hash) ? [Hash.ruby2_keywords_hash(args)] : Array(args)
+      queue ||= resolve_queue_name(job_class, arguments) || Stampeded.config.default_queue
 
-      task, standard_args, meta = envelope(job_class, args, queue, priority)
+      task, standard_args, meta = envelope(job_class, arguments, queue, priority)
 
       Connection.with do |c|
         c.exec_params(<<~SQL, [key, queue, task, JSON.generate(standard_args), JSON.generate(meta), cron, timezone, dst_policy, priority])
@@ -68,18 +70,19 @@ module Stampeded
       MSG
     end
 
-    def resolve_queue_name(job_class)
+    # A `queue_as` block sees the schedule's arguments, which are the same every run.
+    def resolve_queue_name(job_class, arguments)
       return nil unless job_class.respond_to?(:queue_name)
 
       name = job_class.queue_name
-      name = job_class.new.queue_name if name.respond_to?(:call)
+      name = job_class.new(*arguments).queue_name if name.respond_to?(:call)
       name&.to_s
-    rescue StandardError
-      nil
+    rescue StandardError => e
+      raise Error, "#{job_class.name}'s queue_as raised #{e.class}: #{e.message}; " \
+                   "pass queue: to name the schedule's queue"
     end
 
-    def envelope(job_class, args, queue, priority)
-      task = job_class.try(:stampeded_task)
+    def envelope(job_class, arguments, queue, priority)
       context = {
         "job_class" => job_class.name,
         "queue_name" => queue,
@@ -88,15 +91,7 @@ module Stampeded
         "timezone" => defined?(Time.zone) && Time.zone ? Time.zone.name : "UTC"
       }
 
-      kwargs = args.is_a?(Hash) ? args : ({} if Array(args).empty?)
-      if kwargs && Envelope.plain?(kwargs)
-        [task || job_class.name, JSON.parse(JSON.generate(kwargs)), { "activejob" => context }]
-      elsif task
-        raise PortableArgumentsError.new(job_class.name, task)
-      else
-        context["arguments"] = ActiveJob::Arguments.serialize(Array(args))
-        [job_class.name, {}, { "activejob" => context }]
-      end
+      Envelope.build(job_class, arguments, context)
     end
   end
 end

data/lib/stampeded/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Stampeded
-  VERSION = "0.4.1" # x-release-please-version
+  VERSION = "0.5.0" # x-release-please-version
 end

data/lib/stampeded/worker.rb

@@ -17,6 +17,8 @@ module Stampeded
       @batch_size   = positive!(batch_size || Stampeded.config.batch_size, "batch_size")
       @poll_interval = poll_interval || Stampeded.config.poll_interval
       @handlers     = {}
+      @warned       = {}
+      @warned_lock  = Mutex.new
       @stopping     = false
       @wake_reader, @wake_writer = IO.pipe
     end
@@ -57,8 +59,8 @@ module Stampeded
 
       stragglers = @threads.count(&:alive?)
       if stragglers.positive?
-        log "#{stragglers} job(s) did not finish in #{Stampeded.config.shutdown_timeout}s; " \
-            "the reaper will reclaim them"
+        problem "#{stragglers} job(s) did not finish in #{Stampeded.config.shutdown_timeout}s; " \
+                "the reaper will reclaim them"
       end
       log "stopped"
     ensure
@@ -66,7 +68,7 @@ module Stampeded
       begin
         puller.stop(0) if puller && !stopped
       rescue Error => e
-        log "could not stop: #{e.message}"
+        problem "could not stop: #{e.message}"
       end
       @threads&.each { |t| t.join(@poll_interval * 2) }
       puller&.close if @threads.nil? || @threads.none?(&:alive?)
@@ -105,7 +107,7 @@ module Stampeded
             begin
               puller.next(@poll_interval)
             rescue Error => e
-              log "claim failed: #{e.message}"
+              problem "claim failed: #{e.message}"
               sleep @poll_interval
               next
             end
@@ -146,7 +148,7 @@ module Stampeded
       else lease.fail!(message, retry_in_ms: error.is_a?(RetryIn) ? error.delay_ms : nil)
       end
     rescue Error => e
-      log "could not report #{lease.row["id"]}: #{e.message}"
+      problem "could not report #{lease.row["id"]}: #{e.message}"
     end
 
     def with_job_timeout(job, &block)
@@ -155,7 +157,9 @@ module Stampeded
 
       expire = (job.row["expire_seconds"] || 3600).to_i
       if limit >= expire
-        log "job_timeout (#{limit}s) is not shorter than #{job.task}'s expire_seconds (#{expire}s): it may run twice"
+        once([job.task, expire]) do
+          problem "job_timeout (#{limit}s) is not shorter than #{job.task}'s expire_seconds (#{expire}s): it may run twice"
+        end
       end
 
       Timeout.timeout(limit, Timeout::Error, "job exceeded job_timeout of #{limit}s", &block)
@@ -215,10 +219,19 @@ module Stampeded
       log? ? "info" : "warn"
     end
 
+    # Progress: each job, start and stop. Only when asked for, or on a terminal.
     def log(message)
       return unless log?
 
       warn "[stampeded] #{message}"
     end
+
+    # Something an operator must see, which a service manager's stdout is not.
+    def problem(message) = warn("[stampeded] #{message}")
+
+    def once(key)
+      first = @warned_lock.synchronize { @warned[key] ? false : (@warned[key] = true) }
+      yield if first
+    end
   end
 end

data/schema/0006_schedule_unknown_timezone.sql

@@ -0,0 +1,58 @@
+-- Schema 6: a wall-clock occurrence of a schedule whose zone Postgres does not
+-- know is keyed in UTC, as the scheduler fires it. Before, the key raised, and
+-- the error rolled back the whole tick: one mistyped zone stopped every schedule.
+CREATE OR REPLACE FUNCTION stampeded_fire_schedule(p_key text, p_next_run_at timestamptz)
+RETURNS uuid
+LANGUAGE plpgsql AS $$
+DECLARE
+    s        stampeded_schedules%ROWTYPE;
+    v_token  text;
+    v_job    uuid;
+    -- Separate from v_job: the ledger's id is bigserial, the job's is uuid.
+    -- Reusing one variable for both is a runtime cast error, not a compile one.
+    v_occ    bigint;
+BEGIN
+    SELECT * INTO s FROM stampeded_schedules WHERE key = p_key;
+    IF NOT FOUND THEN
+        RETURN NULL;
+    END IF;
+
+    IF s.dst_policy = 'wall_clock' THEN
+        BEGIN
+            v_token := to_char(s.next_run_at AT TIME ZONE s.timezone, 'YYYY-MM-DD HH24:MI');
+        EXCEPTION WHEN invalid_parameter_value THEN
+            v_token := to_char(s.next_run_at AT TIME ZONE 'UTC', 'YYYY-MM-DD HH24:MI');
+        END;
+    ELSE
+        v_token := extract(epoch FROM s.next_run_at)::bigint::text;
+    END IF;
+
+    INSERT INTO stampeded_cron_occurrences (cron_key, occurrence_token)
+    VALUES (s.key, v_token)
+    ON CONFLICT (cron_key, occurrence_token) DO NOTHING
+    RETURNING id INTO v_occ;
+
+    IF v_occ IS NULL THEN
+        UPDATE stampeded_schedules
+           SET next_run_at = p_next_run_at, updated_at = now()
+         WHERE key = s.key;
+        RETURN NULL;
+    END IF;
+
+    INSERT INTO stampeded_jobs (queue, task, args, meta, tenant_id, priority)
+    VALUES (s.queue, s.task, s.args, s.meta, s.tenant_id, s.priority)
+    RETURNING id INTO v_job;
+
+    UPDATE stampeded_cron_occurrences
+       SET job_id = v_job
+     WHERE cron_key = s.key AND occurrence_token = v_token;
+
+    UPDATE stampeded_schedules
+       SET next_run_at = p_next_run_at, last_run_at = now(), updated_at = now()
+     WHERE key = s.key;
+
+    RETURN v_job;
+END;
+$$;
+
+UPDATE stampeded_schema SET version = 6, changed_at = now();

data/schema/0007_queue_pause.sql

@@ -0,0 +1,83 @@
+-- Schema 7: a queue can be paused. A paused queue's jobs stay where they are
+-- and enqueueing still works; stampeded_claim hands out nothing from it, so
+-- the pause holds for every runtime and without a daemon. Jobs already
+-- running finish.
+ALTER TABLE stampeded_queue_policies
+    ADD COLUMN IF NOT EXISTS paused boolean NOT NULL DEFAULT false;
+
+-- Whether p_queue is paused; false, not an error, without the policies table,
+-- which workers already treat as optional. plpgsql so the table is looked up
+-- only once it is known to exist.
+CREATE OR REPLACE FUNCTION stampeded_queue_paused(p_queue text)
+RETURNS boolean
+LANGUAGE plpgsql STABLE AS $$
+BEGIN
+    IF to_regclass('stampeded_queue_policies') IS NULL THEN
+        RETURN false;
+    END IF;
+    RETURN EXISTS (SELECT 1 FROM stampeded_queue_policies
+                    WHERE queue = p_queue AND paused);
+END;
+$$;
+
+CREATE OR REPLACE FUNCTION stampeded_claim(
+    p_queue     text,
+    p_worker_id uuid,
+    p_limit     integer DEFAULT 1,
+
+    p_now       timestamptz DEFAULT NULL
+) RETURNS SETOF stampeded_jobs
+LANGUAGE sql AS $$
+    WITH candidate AS MATERIALIZED (
+        SELECT id
+          FROM stampeded_jobs
+         WHERE queue        = p_queue
+           -- 'retryable' is claimable once its backoff has elapsed. That can be
+           -- handled by a claim predicate, a separate promotion pass, or both.
+           -- The predicate is chosen here for having no moving parts -- a
+           -- promotion pass is another process that can fall behind or die.
+           --
+           -- Omitting 'retryable' here is not a missing feature, it is a silent
+           -- poison-job bug: the reaper parks a crashed job in 'retryable', a
+           -- claim that only reads 'available' never picks it up again, the
+           -- attempt budget never walks down, and max_attempts is never
+           -- reached. Found by test, not by reading.
+           AND state        IN ('available','retryable')
+           AND scheduled_at <= coalesce(p_now, now())
+           AND attempt      < max_attempts
+           -- Reads no job column, so it is a one-time filter: a paused queue
+           -- skips the index scan rather than filtering every row it reads.
+           AND NOT stampeded_queue_paused(p_queue)
+         -- Must match stampeded_jobs_claim_idx EXACTLY. `id` is not a
+         -- tiebreaker here: adding one reintroduces a sort unless it is also
+         -- added to the index, and in the index it costs 8.5x the size by
+         -- defeating btree deduplication. See the claim index note in NOTES.md.
+         ORDER BY priority, scheduled_at
+         LIMIT p_limit
+         FOR UPDATE SKIP LOCKED
+    )
+    UPDATE stampeded_jobs j
+       SET state        = 'executing',
+           attempt      = j.attempt + 1,
+           -- Seed the job heartbeat at claim so stampeded_expire has a baseline
+           -- for a job that never calls stampeded_touch. Left NULL it would fall
+           -- back to claimed_at, which is the same thing -- but only by accident,
+           -- and the expiry predicate should not depend on that coincidence.
+           heartbeat_at = coalesce(p_now, now()),
+           claimed_by   = p_worker_id,
+           claimed_at   = coalesce(p_now, now()),
+           -- Bounded ring buffer. Unbounded attempt history is a slow leak on
+           -- heavily-retried jobs; keep the last 10.
+           attempted_by = array_append(
+               CASE WHEN coalesce(array_length(j.attempted_by, 1), 0) >= 10
+                    THEN j.attempted_by[coalesce(array_length(j.attempted_by,1),0) - 8 :]
+                    ELSE j.attempted_by
+               END,
+               p_worker_id::text)
+      FROM candidate c
+     WHERE j.queue = p_queue
+       AND j.id    = c.id
+    RETURNING j.*;
+$$;
+
+UPDATE stampeded_schema SET version = 7, changed_at = now();

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: stampeded
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.5.0
 platform: ruby
 authors:
 - Abdelkader Boudih
@@ -86,6 +86,8 @@ files:
 - schema/0003_dedup_key_separator.sql
 - schema/0004_dedup_key_raw_scope.sql
 - schema/0005_schema_version_guard.sql
+- schema/0006_schedule_unknown_timezone.sql
+- schema/0007_queue_pause.sql
 homepage: https://github.com/seuros/stampeded
 licenses:
 - BSD-3-Clause