zeridion-flare 0.2.0

data/lib/zeridion_flare/worker/cancellation_token.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+module Zeridion
+  module Flare
+    module Worker
+      # Cooperative cancellation handle handed to every job via JobContext.
+      #
+      # Cancellation in Ruby is COOPERATIVE ONLY (decision D6): the worker never
+      # uses Thread#raise, Thread#kill, or Timeout.timeout — those are unsafe
+      # around Net::HTTP / pg / redis and can corrupt connection state. A
+      # CPU-bound or blocking handler that never checks the token is
+      # uninterruptible from the SDK and is bounded only by the server reaper.
+      #
+      # Handlers cooperate by polling #cancelled? / calling #check! at safe
+      # points, and by using #wait(seconds) instead of Kernel#sleep so a pending
+      # sleep wakes immediately on cancel. Cancellation fires on: a server
+      # "cancel" heartbeat response, host shutdown (SIGTERM/SIGINT), or the
+      # local per-job timeout. In every case the job ultimately acks "failed"
+      # (D1), never "cancelled".
+      class CancellationToken
+        def initialize
+          @mutex = Mutex.new
+          @cond = ConditionVariable.new
+          @cancelled = false
+        end
+
+        # @return [Boolean] true once cancellation has been requested.
+        def cancelled?
+          @mutex.synchronize { @cancelled }
+        end
+
+        # Raise JobCancelledError if cancellation has been requested. Call at
+        # natural checkpoints in a long handler.
+        def check!
+          raise JobCancelledError if cancelled?
+        end
+
+        # Interruptible, monotonic sleep. Returns early (true) the moment
+        # cancellation is requested; otherwise sleeps up to `seconds` and
+        # returns false. Uses ConditionVariable#wait, which is monotonic-clock
+        # based, so it is immune to wall-clock jumps. seconds <= 0 returns
+        # immediately.
+        #
+        # An infinite `seconds` (Float::INFINITY) means "block until cancelled":
+        # ConditionVariable#wait rejects a non-finite timeout ("Inf out of Time
+        # range"), so we pass nil (wait forever) instead of an out-of-range
+        # deadline. #run relies on this to park the main thread until a signal.
+        #
+        # @param seconds [Numeric]
+        # @return [Boolean] true if it woke due to cancellation, false on timeout.
+        def wait(seconds)
+          infinite = seconds.respond_to?(:infinite?) && seconds.infinite?
+          deadline = infinite ? nil : monotonic_now + seconds.to_f
+          @mutex.synchronize do
+            loop do
+              return true if @cancelled
+
+              if deadline.nil?
+                # Block until #cancel! broadcasts (no timeout).
+                @cond.wait(@mutex)
+                next
+              end
+
+              remaining = deadline - monotonic_now
+              return false if remaining <= 0
+
+              @cond.wait(@mutex, remaining)
+            end
+          end
+        end
+
+        # Request cancellation and wake any thread blocked in #wait. Idempotent.
+        # Internal — called by the runtime, not by handlers.
+        def cancel!
+          @mutex.synchronize do
+            @cancelled = true
+            @cond.broadcast
+          end
+        end
+
+        private
+
+        def monotonic_now
+          Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        end
+      end
+    end
+  end
+end

data/lib/zeridion_flare/worker/configuration.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+module Zeridion
+  module Flare
+    module Worker
+      # Resolved worker configuration. One bootstrap object — there is no
+      # separate "Configuration class vs Worker class" split; Worker.new builds
+      # this from its keyword args and normalizes defaults here.
+      class Configuration
+        DEFAULT_QUEUES = ["default"].freeze
+        DEFAULT_CONCURRENCY = 10
+        DEFAULT_POLL_INTERVAL_S = 2.0
+        DEFAULT_TIMEOUT_S = 1800
+        DEFAULT_MAX_ATTEMPTS = 3
+        DEFAULT_SHUTDOWN_GRACE_S = 30.0
+
+        # The worker owns its own HTTP client with a poll read-timeout > 30 s so
+        # the ~30 s long-poll never trips the client's own read deadline
+        # (decision D5). 45 s leaves margin for the server's long-poll plus slack.
+        DEFAULT_POLL_READ_TIMEOUT_S = 45.0
+        # Heartbeat / ack use a short read-timeout (these are quick round-trips).
+        DEFAULT_RPC_TIMEOUT_S = 15.0
+
+        # Floor on the heartbeat cadence in milliseconds (10 s, per the protocol).
+        # Shrinkable for deterministic tests via FLARE_HEARTBEAT_MIN_MS (test
+        # seam D8) or the explicit heartbeat_min_ms: keyword.
+        HEARTBEAT_FLOOR_MS = 10_000
+
+        attr_reader :queues, :concurrency, :poll_interval_s, :default_timeout_s,
+                    :default_max_attempts, :shutdown_grace_s, :poll_read_timeout_s,
+                    :rpc_timeout_s, :heartbeat_min_ms, :hostname, :logger
+
+        # @param queues [Array<String>, nil] queues to poll. nil → the union of
+        #   every registered job's queue (falls back to ["default"]).
+        # @param concurrency [Integer] max simultaneous jobs (clamped >= 1).
+        # @param poll_interval_s [Numeric] idle delay between empty polls.
+        # @param default_timeout_s [Integer] timeout when a job item omits one.
+        # @param default_max_attempts [Integer]
+        # @param shutdown_grace_s [Numeric] max time to await in-flight jobs on stop.
+        # @param poll_read_timeout_s [Numeric] poll HTTP read timeout (> 30 s).
+        # @param rpc_timeout_s [Numeric] heartbeat/ack HTTP read timeout.
+        # @param hostname [String, nil] register hostname (default: machine name).
+        # @param logger [Logger, nil] runtime logger (nil → silent).
+        # @param heartbeat_min_ms [Integer, nil] override the 10 s heartbeat floor
+        #   (test seam). Falls back to FLARE_HEARTBEAT_MIN_MS, then 10_000.
+        def initialize(
+          queues: nil,
+          concurrency: DEFAULT_CONCURRENCY,
+          poll_interval_s: DEFAULT_POLL_INTERVAL_S,
+          default_timeout_s: DEFAULT_TIMEOUT_S,
+          default_max_attempts: DEFAULT_MAX_ATTEMPTS,
+          shutdown_grace_s: DEFAULT_SHUTDOWN_GRACE_S,
+          poll_read_timeout_s: DEFAULT_POLL_READ_TIMEOUT_S,
+          rpc_timeout_s: DEFAULT_RPC_TIMEOUT_S,
+          hostname: nil,
+          logger: nil,
+          heartbeat_min_ms: nil
+        )
+          @queues = normalize_queues(queues)
+          @concurrency = [concurrency.to_i, 1].max
+          @poll_interval_s = poll_interval_s.to_f
+          @default_timeout_s = [default_timeout_s.to_i, 1].max
+          @default_max_attempts = [default_max_attempts.to_i, 1].max
+          @shutdown_grace_s = shutdown_grace_s.to_f
+          @poll_read_timeout_s = poll_read_timeout_s.to_f
+          @rpc_timeout_s = rpc_timeout_s.to_f
+          @hostname = hostname
+          @logger = logger
+          @heartbeat_min_ms = resolve_heartbeat_min_ms(heartbeat_min_ms)
+        end
+
+        # Whether the user pinned an explicit queue list (vs deriving from jobs).
+        def explicit_queues?
+          !@queues.nil?
+        end
+
+        private
+
+        def normalize_queues(queues)
+          return nil if queues.nil?
+
+          list = Array(queues).map(&:to_s).reject(&:empty?).uniq
+          list.empty? ? nil : list
+        end
+
+        def resolve_heartbeat_min_ms(explicit)
+          return [explicit.to_i, 1].max unless explicit.nil?
+
+          env = ENV["FLARE_HEARTBEAT_MIN_MS"]
+          return [env.to_i, 1].max if env && env.match?(/\A\d+\z/)
+
+          HEARTBEAT_FLOOR_MS
+        end
+      end
+    end
+  end
+end

data/lib/zeridion_flare/worker/errors.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Zeridion
+  module Flare
+    module Worker
+      # Raised at wiring time when two job classes claim the same job_type. A
+      # duplicate type on a shared queue would make routing ambiguous, so the
+      # registry refuses it loudly rather than silently last-writer-wins.
+      class DuplicateJobTypeError < ::Zeridion::Flare::FlareError
+        def initialize(job_type)
+          super(
+            "Duplicate job_type #{job_type.inspect}: two job classes cannot share the same job_type.",
+            status_code: 0,
+            code: "duplicate_job_type",
+            request_id: nil,
+          )
+        end
+      end
+
+      # Raised when a polled job_type has no registered handler. The worker turns
+      # this into an ack "failed" with a recognizable error.type rather than
+      # crashing the poll loop.
+      class UnknownJobTypeError < ::Zeridion::Flare::FlareError
+        def initialize(job_type)
+          super(
+            "No job type registered for #{job_type.inspect}.",
+            status_code: 0,
+            code: "unknown_job_type",
+            request_id: nil,
+          )
+        end
+      end
+
+      # Internal sentinel: the in-flight job was cancelled (server "cancel"
+      # response, host shutdown, or local timeout). Per decision D1 this still
+      # acks "failed" — never "cancelled".
+      class JobCancelledError < StandardError
+        def initialize(msg = "Job cancelled.")
+          super
+        end
+      end
+    end
+  end
+end

data/lib/zeridion_flare/worker/executor.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+module Zeridion
+  module Flare
+    module Worker
+      # Executes a single polled job: resolves its definition, decodes the
+      # payload, builds the JobContext, invokes #perform, and reports the
+      # outcome. Mirrors the reference SDK's JobExecutor.
+      #
+      # Ack-status mapping (decision D1): a normal return → "succeeded"; ANY
+      # raised exception — including cancellation, timeout, or an unknown job
+      # type — → "failed". The worker NEVER acks "cancelled". The error.type is
+      # the exception's class name (one-level cause-unwrap), matching the
+      # reference's fully-qualified error type.
+      class Executor
+        # Outcome of one execution. error is nil on success, else a wire-ready
+        # Hash {"type", "message", "stack_trace"} (already nil-compacted).
+        Outcome = Struct.new(:succeeded, :error, keyword_init: true) do
+          def succeeded?
+            succeeded
+          end
+        end
+
+        def initialize(registry:, logger: nil)
+          @registry = registry
+          @logger = logger
+        end
+
+        # @param item [Hash] the polled job item (string keys: "id", "job_type",
+        #   "payload", "attempt", "max_attempts", "timeout_seconds", "enqueued_at")
+        # @param context [JobContext]
+        # @return [Outcome]
+        def execute(item, context)
+          definition = @registry.lookup(item["job_type"])
+          if definition.nil?
+            return Outcome.new(
+              succeeded: false,
+              error: error_hash(UnknownJobTypeError.new(item["job_type"])),
+            )
+          end
+
+          begin
+            invoke(definition, item, context)
+            Outcome.new(succeeded: true, error: nil)
+          rescue StandardError => e
+            Outcome.new(succeeded: false, error: error_hash(unwrap(e)))
+          end
+        end
+
+        private
+
+        def invoke(definition, item, context)
+          instance = definition.job_class.new
+
+          if definition.recurring?
+            instance.perform(context)
+          else
+            payload = coerce_payload(item["payload"], definition.payload_struct)
+            instance.perform(payload, context)
+          end
+        end
+
+        # Decode the wire payload. JSON is already parsed into a Hash/Array/
+        # scalar by the transport, so by default we pass the Hash through with
+        # string keys. With payload_struct: configured, hydrate a typed value via
+        # keyword init (symbolized keys). nil / non-Hash → passed through.
+        def coerce_payload(payload, payload_struct)
+          return payload if payload_struct.nil?
+          return payload unless payload.is_a?(Hash)
+
+          symbolized = payload.transform_keys(&:to_sym)
+          payload_struct.new(**symbolized)
+        end
+
+        # One-level unwrap: prefer the underlying cause when an exception wraps
+        # another (mirrors the reference's TargetInvocationException unwrap).
+        def unwrap(exception)
+          exception.cause || exception
+        end
+
+        def error_hash(exception)
+          {
+            "type" => exception.class.name,
+            "message" => exception.message,
+            "stack_trace" => format_backtrace(exception),
+          }.compact
+        end
+
+        def format_backtrace(exception)
+          bt = exception.backtrace
+          return nil if bt.nil? || bt.empty?
+
+          bt.join("\n")
+        end
+      end
+    end
+  end
+end

data/lib/zeridion_flare/worker/heartbeat.rb

@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+module Zeridion
+  module Flare
+    module Worker
+      # One heartbeat pump per in-flight job, run on its own thread.
+      #
+      # Cadence (frozen protocol §3): every max(10s, timeout_seconds / 3),
+      # integer floor-div on the SECONDS, with the 10 s floor shrinkable for
+      # tests via Configuration#heartbeat_min_ms (FLARE_HEARTBEAT_MIN_MS seam).
+      #
+      # A heartbeat is emitted IMMEDIATELY on claim (the first beat is not
+      # delayed) so a long job lands in the server reaper's forgiving "heartbeated
+      # + silent for GREATEST(120s, timeout × ⅔)" window rather than the strict
+      # "never-heartbeated + StartedAt + timeout" branch.
+      #
+      # The pump keeps beating until the HANDLER finishes (the `stop` signal is
+      # raised only when #perform returns or raises). On a server "cancel"
+      # response it trips the cancellation token (cooperative cancel) but KEEPS
+      # beating while the handler winds down, so a cancelled-but-not-yet-returned
+      # job isn't reaped mid-unwind. Heartbeats are best-effort: a failed beat is
+      # swallowed (the job keeps running); only a "cancel" status or host
+      # shutdown stops execution.
+      class Heartbeat
+        # @param client [Object] worker HTTP client exposing #heartbeat(body) (strict)
+        # @param worker_id [String]
+        # @param job_id [String]
+        # @param timeout_seconds [Integer] the job item's timeout (cadence base)
+        # @param progress_slot [ProgressSlot]
+        # @param cancellation [CancellationToken] tripped on a "cancel" response
+        # @param stop [CancellationToken] internal latch; cancel!'d when the
+        #   handler finishes so the pump exits
+        # @param heartbeat_min_ms [Integer] floor on the cadence (ms)
+        # @param logger [Logger, nil]
+        def initialize(client:, worker_id:, job_id:, timeout_seconds:, progress_slot:,
+                       cancellation:, stop:, heartbeat_min_ms:, logger: nil)
+          @client = client
+          @worker_id = worker_id
+          @job_id = job_id
+          @timeout_seconds = timeout_seconds
+          @progress_slot = progress_slot
+          @cancellation = cancellation
+          @stop = stop
+          @heartbeat_min_ms = heartbeat_min_ms
+          @logger = logger
+          @thread = nil
+        end
+
+        # The cadence in seconds: max(floor, timeout/3) where floor honors the
+        # heartbeat_min_ms seam. Pure / side-effect-free so tests can assert it
+        # without spinning a thread.
+        def interval_seconds
+          floor = @heartbeat_min_ms / 1000.0
+          third = @timeout_seconds.to_i / 3 # integer floor-div on seconds, per spec
+          [floor, third].max
+        end
+
+        # Start the pump thread. Emits the immediate beat, then loops on the
+        # cadence until `stop` is signalled.
+        def start
+          @thread = Thread.new do
+            Thread.current.name = "flare-heartbeat-#{@job_id}" if Thread.current.respond_to?(:name=)
+            run
+          end
+          self
+        end
+
+        # Signal the pump to stop (handler finished) and join its thread.
+        def stop_and_join(timeout: nil)
+          @stop.cancel!
+          @thread&.join(timeout)
+          nil
+        end
+
+        private
+
+        def run
+          # Immediate beat on claim — lands the job in the forgiving reaper window.
+          return if beat_once == :cancel && stopped?
+
+          interval = interval_seconds
+          loop do
+            break if @stop.wait(interval) # woke because the handler finished
+
+            beat_once
+          end
+        rescue StandardError
+          # Pump must never crash the process; a beat error is non-fatal.
+          nil
+        end
+
+        # Send one heartbeat. Best-effort: swallow transport/HTTP errors. A
+        # "cancel" status trips the cancellation token (but does NOT stop the
+        # pump — it keeps beating while the handler unwinds). Returns :cancel,
+        # :ok, or :error.
+        def beat_once
+          body = { "job_id" => @job_id, "worker_id" => @worker_id }
+          progress = @progress_slot.get
+          body["progress"] = progress unless progress.nil?
+
+          response = @client.heartbeat(body)
+          if response.is_a?(Hash) && response["status"] == "cancel"
+            @cancellation.cancel!
+            log_cancel
+            :cancel
+          else
+            :ok
+          end
+        rescue StandardError
+          :error
+        end
+
+        def stopped?
+          @stop.cancelled?
+        end
+
+        def log_cancel
+          @logger&.warn("heartbeat_cancel job_id=#{@job_id}")
+        end
+      end
+    end
+  end
+end

data/lib/zeridion_flare/worker/identity.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require "securerandom"
+require "socket"
+
+module Zeridion
+  module Flare
+    module Worker
+      # Builds the worker id: wrk_{host}_{pid}_{rand}
+      #
+      #   * host is SANITIZED to [A-Za-z0-9_-] (replace "." with "-", strip
+      #     anything else) so a raw FQDN like "web-01.prod.internal" can't
+      #     violate the server's `^[A-Za-z0-9_\-]+$` regex — which would 422 and
+      #     silently kill BOTH register and poll.
+      #   * rand is >= 12 hex chars (>= 48 bits) from a CSPRNG, so two workers
+      #     sharing a container hostname (PID 1) don't collide and cross-talk.
+      #   * the whole id is truncated to <= 200 chars (register's cap). The
+      #     random suffix is preserved; the HOST portion is trimmed if needed so
+      #     uniqueness survives truncation.
+      module Identity
+        MAX_LEN = 200
+        RAND_HEX = 12 # 48 bits of entropy
+
+        module_function
+
+        # @param hostname [String, nil] override host (default: machine hostname)
+        # @param pid [Integer] override pid (default: this process)
+        # @param rand_hex [Integer] hex chars of CSPRNG entropy (>= 12)
+        # @return [String]
+        def generate(hostname: nil, pid: Process.pid, rand_hex: RAND_HEX)
+          host = sanitize(hostname || default_hostname)
+          rand = secure_hex([rand_hex, RAND_HEX].max)
+          suffix = "_#{pid}_#{rand}"
+          prefix = "wrk_"
+
+          # Trim the host so prefix + host + suffix <= MAX_LEN, never sacrificing
+          # the random suffix (uniqueness lives there).
+          budget = MAX_LEN - prefix.length - suffix.length
+          host = "w" if host.empty? # never emit "wrk__<pid>_<rand>" with a blank host
+          host = host[0, budget] if budget.positive? && host.length > budget
+          host = "w" if budget <= 0
+
+          "#{prefix}#{host}#{suffix}"
+        end
+
+        # Sanitize a hostname to the server-allowed alphabet.
+        def sanitize(raw)
+          s = raw.to_s
+          s = s.tr(".", "-")        # dotted FQDN segments become dashes
+          s = s.gsub(/[^A-Za-z0-9_-]/, "") # strip everything else
+          s
+        end
+
+        def default_hostname
+          Socket.gethostname
+        rescue StandardError
+          "host"
+        end
+
+        # n hex characters from a CSPRNG (n rounded up to an even count).
+        def secure_hex(n)
+          bytes = (n + 1) / 2
+          SecureRandom.hex(bytes)[0, n]
+        end
+      end
+    end
+  end
+end

data/lib/zeridion_flare/worker/job.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+module Zeridion
+  module Flare
+    # Mixin marking a class as a Flare payload job. Include it, optionally set
+    # defaults with the `flare_options` macro, and implement `#perform`:
+    #
+    #   class SendWelcomeEmail
+    #     include Zeridion::Flare::Job
+    #     flare_options queue: "email", max_attempts: 5, timeout: 120
+    #
+    #     def perform(payload, ctx)
+    #       return if ctx.cancelled?
+    #       return if AlreadySent.exists?(ctx.job_id)   # at-least-once → idempotent
+    #       Mailer.welcome(payload["email"]).deliver_now
+    #       ctx.report_progress(1.0)
+    #     end
+    #   end
+    #
+    # The `included` hook registers the class into the module-level worker
+    # registry at definition time (no ObjectSpace scan). job_type defaults to the
+    # class's fully-qualified name; override it with `flare_options job_type:`.
+    # Two classes claiming the same job_type raise DuplicateJobTypeError.
+    #
+    # payload is the decoded JSON Hash with STRING keys by default. Opt into a
+    # typed payload with `flare_options payload_struct: MyStruct` — the worker
+    # then calls `MyStruct.new(**symbolized_hash)` (works with Struct / Data /
+    # any keyword-initializable class).
+    module Job
+      def self.included(base)
+        base.extend(ClassMacros)
+        base.include(InstanceShape)
+        Worker::Registry.register_payload(base)
+      end
+
+      # `flare_options ...` and the resolved config readers, mixed into the
+      # including class as class methods.
+      module ClassMacros
+        # Set per-class defaults. All keys optional.
+        #
+        # @param job_type [String, nil] explicit routing type (default: class FQN)
+        # @param queue [String]
+        # @param max_attempts [Integer]
+        # @param timeout [Integer] seconds
+        # @param payload_struct [Class, nil] keyword-initializable payload type
+        def flare_options(job_type: nil, queue: nil, max_attempts: nil, timeout: nil, payload_struct: nil)
+          @flare_job_type = job_type unless job_type.nil?
+          @flare_queue = queue unless queue.nil?
+          @flare_max_attempts = max_attempts unless max_attempts.nil?
+          @flare_timeout = timeout unless timeout.nil?
+          @flare_payload_struct = payload_struct unless payload_struct.nil?
+          nil
+        end
+
+        def flare_job_type
+          @flare_job_type || name ||
+            raise(ArgumentError, "anonymous job class needs an explicit flare_options(job_type:)")
+        end
+
+        def flare_queue
+          @flare_queue || "default"
+        end
+
+        def flare_max_attempts
+          @flare_max_attempts || 3
+        end
+
+        def flare_timeout
+          @flare_timeout || 1800
+        end
+
+        def flare_payload_struct
+          @flare_payload_struct
+        end
+      end
+
+      # Guards against forgetting `#perform`.
+      module InstanceShape
+        def perform(_payload, _ctx)
+          raise NotImplementedError, "#{self.class.name} must implement #perform(payload, ctx)"
+        end
+      end
+    end
+  end
+end