flexiq-executor 2.1.0

data/lib/flexiq/executor/session.rb

@@ -0,0 +1,230 @@
+# frozen_string_literal: true
+
+require "English"
+
+module FlexiQ
+  module Executor
+    # One attach: a bidirectional stream from `hello` to its end, and the jobs it dispatched.
+    #
+    # `run` returns how the stream ended. A Worker reconnects after `:rotated` and `:error`, and
+    # stops after `:shutdown` and `:stopped`.
+    class Session
+      SESSION_METADATA_KEY = "flexiq-attach-session-bin"
+      # How long the writer gets to send what is queued once the session stops taking work.
+      FLUSH_BUDGET = 5
+      # How long a half-closed stream waits for the scheduler to end it before it is cancelled.
+      CLOSE_GRACE = 5
+
+      Ending = Data.define(:reason, :attached, :error)
+
+      def initialize(config, stub, handlers, slots)
+        @config = config
+        @log = config.logger
+        @stub = stub
+        @handlers = handlers
+        @slots = slots
+        @mutex = Mutex.new
+        @running = {}
+        @warned = {}
+        @acked = Queue.new
+        @heartbeat_stop = Queue.new
+        @done = Queue.new
+        @outbox = Outbox.new
+      end
+
+      # Attaches and serves the stream until it ends. Never raises for a stream failure: the
+      # failure comes back in the Ending.
+      def run
+        @slots.resume
+        @outbox.push(hello)
+        @op = @stub.attach(@outbox.frames(-> { @finished || !@op&.status.nil? }),
+                           metadata: metadata, return_op: true)
+        watch_handshake
+        @op.execute.each { |response| receive(response) }
+        ending(@reason || :rotated)
+      rescue AttachError => e
+        ending(@reason || :error, e)
+      rescue GRPC::BadStatus => e
+        ending(@reason || :error, @refusal || AttachError.from(e))
+      ensure
+        teardown
+      end
+
+      # Stops taking work, lets running jobs finish within the drain budget, flushes their
+      # results and half-closes the stream. Idempotent; returns at once.
+      def begin_drain(reason)
+        @mutex.synchronize do
+          return if @drainer
+
+          @reason ||= reason
+          @drainer = Thread.new { drain }
+        end
+      end
+
+      private
+
+      def metadata = { "authorization" => @config.bearer }
+
+      def hello
+        V1::AttachRequest.new(hello: V1::HelloFrame.new(
+          executor_id: @config.id, sdk: @config.sdk, version: @config.version, tasks: @handlers.keys.sort,
+          slots: @config.slots, protocol_version: PROTOCOL_VERSION, capabilities: CAPABILITIES
+        ))
+      end
+
+      # A `Heartbeat` that overtakes `hello_ack` is read as the handshake and refuses the attach,
+      # so nothing else is sent until the acknowledgement arrives, and only so long for it.
+      def watch_handshake
+        Thread.new do
+          # nil is the timeout; true or false means the handshake already resolved.
+          next unless @acked.pop(timeout: @config.handshake_timeout).nil?
+
+          @refusal = AttachError.new(GRPC::Core::StatusCodes::DEADLINE_EXCEEDED,
+                                     "no hello acknowledgement within #{@config.handshake_timeout}s",
+                                     permanent: false)
+          @op.cancel
+        end
+      end
+
+      def receive(response)
+        frame = response.frame
+        return handshake(response.hello_ack) if frame == :hello_ack && !@attached
+
+        unless @attached
+          raise AttachError.new(GRPC::Core::StatusCodes::FAILED_PRECONDITION,
+                                "the scheduler's first frame was not a hello acknowledgement", permanent: true)
+        end
+
+        case frame
+        when :job then dispatch(response.job)
+        when :cancel then cancel(response.cancel.job_id)
+        when :shutdown then begin_drain(:shutdown)
+        when :hello_ack then once(:hello_ack, "a second hello acknowledgement on an attached stream; ignoring it")
+        when :job_steps, :step_ack then once(:steps, "a durable-step frame this client did not ask for; ignoring it")
+        else once(:unknown, "the scheduler sent a frame this build does not recognise; ignoring it")
+        end
+      end
+
+      def handshake(ack)
+        unless ack.protocol_version == PROTOCOL_VERSION
+          @acked << true
+          raise AttachError.new(GRPC::Core::StatusCodes::FAILED_PRECONDITION,
+                                "protocol version mismatch: we speak #{PROTOCOL_VERSION}, " \
+                                "the scheduler speaks #{ack.protocol_version}", permanent: true)
+        end
+
+        @token = @op.metadata&.fetch(SESSION_METADATA_KEY, nil)
+        @log.warn("the attach response carried no #{SESSION_METADATA_KEY}; heartbeats will be refused") unless @token
+        @side_on = ack.capabilities.include?(CAP_SIDE_CHANNEL)
+        @attached = true
+        @acked << true
+        @log.info("attached as #{@config.id} to #{ack.scheduler_id} (slots=#{@config.slots}, " \
+                  "tasks=#{@handlers.size}, side_channel=#{@side_on}, lease=#{ack.capabilities.include?(CAP_LEASE)})")
+        @heartbeat = Thread.new { heartbeat_loop }
+      end
+
+      def dispatch(frame)
+        job = Job.new(frame, side: @side_on ? @outbox : nil)
+        handler = @handlers[job.task_name]
+        unless handler
+          return refuse(job, "TaskNotRegistered", "task not registered: #{job.task_name}", should_retry: false)
+        end
+        unless @slots.acquire
+          return refuse(job, "NoCapacity", "executor did not run '#{job.task_name}': no free slot", should_retry: true)
+        end
+
+        # Booked before the thread starts: a `cancel` can follow its `job` frame at once.
+        @mutex.synchronize { @running[job.id] = job }
+        Thread.new { execute(job, handler) }
+      end
+
+      def refuse(job, errtype, message, should_retry:)
+        @outbox.push(Outcome.failure(job, TaskError.encode(errtype, message), should_retry: should_retry))
+      end
+
+      def execute(job, handler)
+        settled = false
+        started = Clock.now
+        value = error = nil
+        begin
+          value = handler.call(job)
+        rescue StandardError, ScriptError => e
+          error = e
+        end
+        settle(job, Outcome.new(value: value, error: error, wall: Clock.now - started,
+                                timed_out: job.timed_out?, cancelled: job.cancelled?).frame(job))
+        settled = true
+      ensure
+        # Exactly one settling frame per job, even for a thread that was killed or raised past
+        # StandardError: a job never settled is a job the scheduler waits on until its timeout.
+        unless settled
+          message = "the handler ended without returning: #{$ERROR_INFO&.class || "thread killed"}"
+          settle(job, Outcome.failure(job, TaskError.encode("HandlerAborted", message), should_retry: true))
+        end
+        @mutex.synchronize { @running.delete(job.id) }
+        @slots.release
+      end
+
+      def settle(job, frame)
+        @outbox.promote(job.id)
+        @outbox.push(frame)
+      end
+
+      def cancel(job_id)
+        @mutex.synchronize { @running[job_id] }&.request_cancel!
+      end
+
+      def heartbeat_loop
+        heartbeat(@slots.available) until @heartbeat_stop.pop(timeout: @config.heartbeat_interval)
+      end
+
+      # Unary and off the stream, so it carries the session token rather than the executor id:
+      # an id is a name this executor picked, and could be another executor's.
+      def heartbeat(free_slots)
+        return unless @token
+
+        # The interval bounds the call, but never below a second, so a short interval in a test
+        # cannot make every heartbeat time out.
+        deadline = Time.now + [@config.heartbeat_interval, 1].max
+        @stub.heartbeat(V1::HeartbeatRequest.new(session: @token, free_slots: free_slots),
+                        metadata: metadata, deadline: deadline)
+      rescue GRPC::BadStatus => e
+        @log.debug("heartbeat failed: #{e.message}")
+      end
+
+      def drain
+        @slots.drain
+        # Zero capacity first, so the scheduler stops matching work to this stream.
+        heartbeat(0) if @attached
+        unless @slots.await_idle(@config.shutdown_drain)
+          @log.warn("#{@slots.in_flight} job(s) still running after the #{@config.shutdown_drain}s drain")
+        end
+        @outbox.close
+        @log.warn("gave up flushing results after #{FLUSH_BUDGET}s") unless @outbox.await_drained(FLUSH_BUDGET)
+        # The scheduler ends the stream once it reads the half-close; cancel only if it does not.
+        @op&.cancel unless @done.pop(timeout: CLOSE_GRACE)
+      end
+
+      def teardown
+        @done << true
+        @drainer&.join
+        @finished = true
+        @outbox.close
+        @heartbeat_stop << true
+        @heartbeat&.join
+        @acked << false
+        @mutex.synchronize { @running.each_value(&:request_cancel!) }
+        @op&.cancel
+      end
+
+      def ending(reason, error = nil) = Ending.new(reason: reason, attached: @attached == true, error: error)
+
+      def once(key, message)
+        return if @warned[key]
+
+        @warned[key] = true
+        @log.warn(message)
+      end
+    end
+  end
+end

data/lib/flexiq/executor/slots.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module FlexiQ
+  module Executor
+    # The executor's concurrency: how many jobs may run at once, shared across reconnects so a
+    # job still running from the last stream keeps its slot.
+    class Slots
+      attr_reader :total
+
+      def initialize(total)
+        @total = total
+        @free = total
+        @draining = false
+        @mutex = Mutex.new
+        @changed = ConditionVariable.new
+      end
+
+      # Takes a slot; false when none is free or the session is draining.
+      def acquire
+        @mutex.synchronize do
+          return false if @draining || @free <= 0
+
+          @free -= 1
+          true
+        end
+      end
+
+      def release
+        @mutex.synchronize do
+          @free = [@free + 1, total].min
+          @changed.broadcast
+        end
+      end
+
+      # Free capacity as a heartbeat reports it: zero while draining, so no new work is matched.
+      def available = @mutex.synchronize { @draining ? 0 : @free }
+
+      def in_flight = @mutex.synchronize { total - @free }
+
+      def drain = @mutex.synchronize { @draining = true }
+
+      def resume = @mutex.synchronize { @draining = false }
+
+      # Waits for every running job to finish, up to `budget` seconds.
+      def await_idle(budget)
+        deadline = Clock.now + budget
+        @mutex.synchronize do
+          until @free >= total
+            remaining = deadline - Clock.now
+            return false unless remaining.positive?
+
+            @changed.wait(@mutex, remaining)
+          end
+          true
+        end
+      end
+    end
+  end
+end

data/lib/flexiq/executor/v1/executor_service_pb.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: flexiq/executor/v1/executor_service.proto
+
+require 'google/protobuf'
+
+require 'google/protobuf/duration_pb'
+require 'google/protobuf/timestamp_pb'
+
+
+descriptor_data = "\n)flexiq/executor/v1/executor_service.proto\x12\x12\x66lexiq.executor.v1\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"\x90\x04\n\rAttachRequest\x12\x36\n\x05hello\x18\x01 \x01(\x0b\x32\x1e.flexiq.executor.v1.HelloFrameH\x00R\x05hello\x12<\n\x07success\x18\x02 \x01(\x0b\x32 .flexiq.executor.v1.SuccessFrameH\x00R\x07success\x12<\n\x07\x66\x61ilure\x18\x03 \x01(\x0b\x32 .flexiq.executor.v1.FailureFrameH\x00R\x07\x66\x61ilure\x12\x42\n\tcancelled\x18\x04 \x01(\x0b\x32\".flexiq.executor.v1.CancelledFrameH\x00R\tcancelled\x12\x36\n\x05slept\x18\x05 \x01(\x0b\x32\x1e.flexiq.executor.v1.SleptFrameH\x00R\x05slept\x12?\n\x08progress\x18\x06 \x01(\x0b\x32!.flexiq.executor.v1.ProgressFrameH\x00R\x08progress\x12=\n\x08task_log\x18\x07 \x01(\x0b\x32 .flexiq.executor.v1.TaskLogFrameH\x00R\x07taskLog\x12\x46\n\x0bstep_commit\x18\x08 \x01(\x0b\x32#.flexiq.executor.v1.StepCommitFrameH\x00R\nstepCommitB\x07\n\x05\x66rame\"\x8a\x03\n\x0e\x41ttachResponse\x12@\n\thello_ack\x18\x01 \x01(\x0b\x32!.flexiq.executor.v1.HelloAckFrameH\x00R\x08helloAck\x12\x30\n\x03job\x18\x02 \x01(\x0b\x32\x1c.flexiq.executor.v1.JobFrameH\x00R\x03job\x12@\n\tjob_steps\x18\x03 \x01(\x0b\x32!.flexiq.executor.v1.JobStepsFrameH\x00R\x08jobSteps\x12=\n\x08step_ack\x18\x04 \x01(\x0b\x32 .flexiq.executor.v1.StepAckFrameH\x00R\x07stepAck\x12\x39\n\x06\x63\x61ncel\x18\x05 \x01(\x0b\x32\x1f.flexiq.executor.v1.CancelFrameH\x00R\x06\x63\x61ncel\x12?\n\x08shutdown\x18\x06 \x01(\x0b\x32!.flexiq.executor.v1.ShutdownFrameH\x00R\x08shutdownB\x07\n\x05\x66rame\"\xd4\x01\n\nHelloFrame\x12\x1f\n\x0b\x65xecutor_id\x18\x01 \x01(\tR\nexecutorId\x12\x10\n\x03sdk\x18\x02 \x01(\tR\x03sdk\x12\x18\n\x07version\x18\x03 \x01(\tR\x07version\x12\x14\n\x05tasks\x18\x04 \x03(\tR\x05tasks\x12\x14\n\x05slots\x18\x05 \x01(\rR\x05slots\x12)\n\x10protocol_version\x18\x06 \x01(\rR\x0fprotocolVersion\x12\"\n\x0c\x63\x61pabilities\x18\x07 \x03(\tR\x0c\x63\x61pabilities\"\x81\x01\n\rHelloAckFrame\x12!\n\x0cscheduler_id\x18\x01 \x01(\tR\x0bschedulerId\x12)\n\x10protocol_version\x18\x02 \x01(\rR\x0fprotocolVersion\x12\"\n\x0c\x63\x61pabilities\x18\x03 \x03(\tR\x0c\x63\x61pabilities\"\x93\x03\n\x08JobFrame\x12\x0e\n\x02id\x18\x01 \x01(\tR\x02id\x12\x1b\n\ttask_name\x18\x02 \x01(\tR\x08taskName\x12\x18\n\x07payload\x18\x03 \x01(\x0cR\x07payload\x12\x1f\n\x0bretry_count\x18\x04 \x01(\x05R\nretryCount\x12\x1f\n\x0bmax_retries\x18\x05 \x01(\x05R\nmaxRetries\x12\x14\n\x05queue\x18\x06 \x01(\tR\x05queue\x12\x33\n\x07timeout\x18\x07 \x01(\x0b\x32\x19.google.protobuf.DurationR\x07timeout\x12!\n\tnamespace\x18\x08 \x01(\tH\x00R\tnamespace\x88\x01\x01\x12/\n\x13\x64isabled_middleware\x18\t \x03(\tR\x12\x64isabledMiddleware\x12\x1f\n\x08metadata\x18\n \x01(\tH\x01R\x08metadata\x88\x01\x01\x12\x19\n\x05lease\x18\x0b \x01(\x0cH\x02R\x05lease\x88\x01\x01\x42\x0c\n\n_namespaceB\x0b\n\t_metadataB\x08\n\x06_lease\"B\n\rJobStepsFrame\x12\x15\n\x06job_id\x18\x01 \x01(\tR\x05jobId\x12\x1a\n\x08snapshot\x18\x02 \x01(\x0cR\x08snapshot\"\xf6\x01\n\x0cStepAckFrame\x12\x15\n\x06job_id\x18\x01 \x01(\tR\x05jobId\x12\x10\n\x03seq\x18\x02 \x01(\x05R\x03seq\x12\x0e\n\x02ok\x18\x03 \x01(\x08R\x02ok\x12\x18\n\x07\x61lready\x18\x04 \x01(\x08R\x07\x61lready\x12\x33\n\x07wake_at\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\x06wakeAt\x12\x19\n\x05\x65rror\x18\x06 \x01(\tH\x00R\x05\x65rror\x88\x01\x01\x12\x39\n\x07\x66\x61ilure\x18\x07 \x01(\x0e\x32\x1f.flexiq.executor.v1.StepFailureR\x07\x66\x61ilureB\x08\n\x06_error\"$\n\x0b\x43\x61ncelFrame\x12\x15\n\x06job_id\x18\x01 \x01(\tR\x05jobId\"\x0f\n\rShutdownFrame\"g\n\rProgressFrame\x12\x15\n\x06job_id\x18\x01 \x01(\tR\x05jobId\x12\x1a\n\x08progress\x18\x02 \x01(\x05R\x08progress\x12\x19\n\x05lease\x18\x03 \x01(\x0cH\x00R\x05lease\x88\x01\x01\x42\x08\n\x06_lease\"\xbc\x01\n\x0cTaskLogFrame\x12\x15\n\x06job_id\x18\x01 \x01(\tR\x05jobId\x12\x1b\n\ttask_name\x18\x02 \x01(\tR\x08taskName\x12\x14\n\x05level\x18\x03 \x01(\tR\x05level\x12\x18\n\x07message\x18\x04 \x01(\tR\x07message\x12\x19\n\x05\x65xtra\x18\x05 \x01(\x0cH\x00R\x05\x65xtra\x88\x01\x01\x12\x19\n\x05lease\x18\x06 \x01(\x0cH\x01R\x05lease\x88\x01\x01\x42\x08\n\x06_extraB\x08\n\x06_lease\"\xc7\x01\n\x0cSuccessFrame\x12\x15\n\x06job_id\x18\x01 \x01(\tR\x05jobId\x12\x1b\n\ttask_name\x18\x02 \x01(\tR\x08taskName\x12\x1b\n\x06result\x18\x03 \x01(\x0cH\x00R\x06result\x88\x01\x01\x12\x36\n\twall_time\x18\x04 \x01(\x0b\x32\x19.google.protobuf.DurationR\x08wallTime\x12\x19\n\x05lease\x18\x05 \x01(\x0cH\x01R\x05lease\x88\x01\x01\x42\t\n\x07_resultB\x08\n\x06_lease\"\xb7\x02\n\x0c\x46\x61ilureFrame\x12\x15\n\x06job_id\x18\x01 \x01(\tR\x05jobId\x12\x1b\n\ttask_name\x18\x02 \x01(\tR\x08taskName\x12\x14\n\x05\x65rror\x18\x03 \x01(\tR\x05\x65rror\x12\x1f\n\x0bretry_count\x18\x04 \x01(\x05R\nretryCount\x12\x1f\n\x0bmax_retries\x18\x05 \x01(\x05R\nmaxRetries\x12\x36\n\twall_time\x18\x06 \x01(\x0b\x32\x19.google.protobuf.DurationR\x08wallTime\x12!\n\x0cshould_retry\x18\x07 \x01(\x08R\x0bshouldRetry\x12\x1b\n\ttimed_out\x18\x08 \x01(\x08R\x08timedOut\x12\x19\n\x05lease\x18\t \x01(\x0cH\x00R\x05lease\x88\x01\x01\x42\x08\n\x06_lease\"\xa1\x01\n\x0e\x43\x61ncelledFrame\x12\x15\n\x06job_id\x18\x01 \x01(\tR\x05jobId\x12\x1b\n\ttask_name\x18\x02 \x01(\tR\x08taskName\x12\x36\n\twall_time\x18\x03 \x01(\x0b\x32\x19.google.protobuf.DurationR\x08wallTime\x12\x19\n\x05lease\x18\x04 \x01(\x0cH\x00R\x05lease\x88\x01\x01\x42\x08\n\x06_lease\"\xfb\x01\n\x0fStepCommitFrame\x12\x15\n\x06job_id\x18\x01 \x01(\tR\x05jobId\x12\x10\n\x03seq\x18\x02 \x01(\x05R\x03seq\x12\x19\n\x08step_key\x18\x03 \x01(\tR\x07stepKey\x12\x30\n\x04kind\x18\x04 \x01(\x0e\x32\x1c.flexiq.executor.v1.StepKindR\x04kind\x12\x18\n\x07payload\x18\x05 \x01(\x0cR\x07payload\x12\x33\n\x07wake_at\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\x06wakeAt\x12\x19\n\x05lease\x18\x07 \x01(\x0cH\x00R\x05lease\x88\x01\x01\x42\x08\n\x06_lease\"\xd2\x01\n\nSleptFrame\x12\x15\n\x06job_id\x18\x01 \x01(\tR\x05jobId\x12\x1b\n\ttask_name\x18\x02 \x01(\tR\x08taskName\x12\x33\n\x07wake_at\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\x06wakeAt\x12\x36\n\twall_time\x18\x04 \x01(\x0b\x32\x19.google.protobuf.DurationR\x08wallTime\x12\x19\n\x05lease\x18\x05 \x01(\x0cH\x00R\x05lease\x88\x01\x01\x42\x08\n\x06_lease\"\xda\x01\n\rSettleRequest\x12<\n\x07success\x18\x01 \x01(\x0b\x32 .flexiq.executor.v1.SuccessFrameH\x00R\x07success\x12<\n\x07\x66\x61ilure\x18\x02 \x01(\x0b\x32 .flexiq.executor.v1.FailureFrameH\x00R\x07\x66\x61ilure\x12\x42\n\tcancelled\x18\x03 \x01(\x0b\x32\".flexiq.executor.v1.CancelledFrameH\x00R\tcancelledB\t\n\x07outcome\"\x10\n\x0eSettleResponse\"y\n\x12\x45xtendLeaseRequest\x12\x15\n\x06job_id\x18\x01 \x01(\tR\x05jobId\x12\x14\n\x05lease\x18\x02 \x01(\x0cR\x05lease\x12\x36\n\textend_by\x18\x03 \x01(\x0b\x32\x19.google.protobuf.DurationR\x08\x65xtendBy\"M\n\x13\x45xtendLeaseResponse\x12\x36\n\x08\x64\x65\x61\x64line\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\x08\x64\x65\x61\x64line\"V\n\x15ReportProgressRequest\x12=\n\x08progress\x18\x01 \x01(\x0b\x32!.flexiq.executor.v1.ProgressFrameR\x08progress\"\x18\n\x16ReportProgressResponse\"R\n\x13WriteTaskLogRequest\x12;\n\x08task_log\x18\x01 \x01(\x0b\x32 .flexiq.executor.v1.TaskLogFrameR\x07taskLog\"\x16\n\x14WriteTaskLogResponse\"K\n\x10HeartbeatRequest\x12\x18\n\x07session\x18\x01 \x01(\x0cR\x07session\x12\x1d\n\nfree_slots\x18\x02 \x01(\rR\tfreeSlots\"\x13\n\x11HeartbeatResponse*M\n\x08StepKind\x12\x19\n\x15STEP_KIND_UNSPECIFIED\x10\x00\x12\x11\n\rSTEP_KIND_RUN\x10\x01\x12\x13\n\x0fSTEP_KIND_SLEEP\x10\x02*\x80\x01\n\x0bStepFailure\x12\x1c\n\x18STEP_FAILURE_UNSPECIFIED\x10\x00\x12\x1a\n\x16STEP_FAILURE_RETRYABLE\x10\x01\x12\x1a\n\x16STEP_FAILURE_PERMANENT\x10\x02\x12\x1b\n\x17STEP_FAILURE_SUPERSEDED\x10\x03\x32\xbd\x04\n\x0f\x45xecutorService\x12S\n\x06\x41ttach\x12!.flexiq.executor.v1.AttachRequest\x1a\".flexiq.executor.v1.AttachResponse(\x01\x30\x01\x12X\n\tHeartbeat\x12$.flexiq.executor.v1.HeartbeatRequest\x1a%.flexiq.executor.v1.HeartbeatResponse\x12O\n\x06Settle\x12!.flexiq.executor.v1.SettleRequest\x1a\".flexiq.executor.v1.SettleResponse\x12^\n\x0b\x45xtendLease\x12&.flexiq.executor.v1.ExtendLeaseRequest\x1a\'.flexiq.executor.v1.ExtendLeaseResponse\x12g\n\x0eReportProgress\x12).flexiq.executor.v1.ReportProgressRequest\x1a*.flexiq.executor.v1.ReportProgressResponse\x12\x61\n\x0cWriteTaskLog\x12\'.flexiq.executor.v1.WriteTaskLogRequest\x1a(.flexiq.executor.v1.WriteTaskLogResponseB\x98\x01\n\x16\x63om.flexiq.executor.v1B\x14\x45xecutorServiceProtoP\x01\xa2\x02\x03\x46\x45X\xaa\x02\x12\x46lexiq.Executor.V1\xca\x02\x12\x46lexiq\\Executor\\V1\xe2\x02\x1e\x46lexiq\\Executor\\V1\\GPBMetadata\xea\x02\x14\x46lexiQ::Executor::V1b\x06proto3"
+
+pool = ::Google::Protobuf::DescriptorPool.generated_pool
+pool.add_serialized_file(descriptor_data)
+
+module FlexiQ
+  module Executor
+    module V1
+      AttachRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("flexiq.executor.v1.AttachRequest").msgclass
+      AttachResponse = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("flexiq.executor.v1.AttachResponse").msgclass
+      HelloFrame = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("flexiq.executor.v1.HelloFrame").msgclass
+      HelloAckFrame = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("flexiq.executor.v1.HelloAckFrame").msgclass
+      JobFrame = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("flexiq.executor.v1.JobFrame").msgclass
+      JobStepsFrame = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("flexiq.executor.v1.JobStepsFrame").msgclass
+      StepAckFrame = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("flexiq.executor.v1.StepAckFrame").msgclass
+      CancelFrame = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("flexiq.executor.v1.CancelFrame").msgclass
+      ShutdownFrame = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("flexiq.executor.v1.ShutdownFrame").msgclass
+      ProgressFrame = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("flexiq.executor.v1.ProgressFrame").msgclass
+      TaskLogFrame = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("flexiq.executor.v1.TaskLogFrame").msgclass
+      SuccessFrame = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("flexiq.executor.v1.SuccessFrame").msgclass
+      FailureFrame = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("flexiq.executor.v1.FailureFrame").msgclass
+      CancelledFrame = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("flexiq.executor.v1.CancelledFrame").msgclass
+      StepCommitFrame = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("flexiq.executor.v1.StepCommitFrame").msgclass
+      SleptFrame = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("flexiq.executor.v1.SleptFrame").msgclass
+      SettleRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("flexiq.executor.v1.SettleRequest").msgclass
+      SettleResponse = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("flexiq.executor.v1.SettleResponse").msgclass
+      ExtendLeaseRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("flexiq.executor.v1.ExtendLeaseRequest").msgclass
+      ExtendLeaseResponse = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("flexiq.executor.v1.ExtendLeaseResponse").msgclass
+      ReportProgressRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("flexiq.executor.v1.ReportProgressRequest").msgclass
+      ReportProgressResponse = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("flexiq.executor.v1.ReportProgressResponse").msgclass
+      WriteTaskLogRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("flexiq.executor.v1.WriteTaskLogRequest").msgclass
+      WriteTaskLogResponse = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("flexiq.executor.v1.WriteTaskLogResponse").msgclass
+      HeartbeatRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("flexiq.executor.v1.HeartbeatRequest").msgclass
+      HeartbeatResponse = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("flexiq.executor.v1.HeartbeatResponse").msgclass
+      StepKind = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("flexiq.executor.v1.StepKind").enummodule
+      StepFailure = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("flexiq.executor.v1.StepFailure").enummodule
+    end
+  end
+end

data/lib/flexiq/executor/v1/executor_service_services_pb.rb

@@ -0,0 +1,139 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# Source: flexiq/executor/v1/executor_service.proto for package 'FlexiQ.Executor.V1'
+# Original file comments:
+# STABLE as of 2.0.0, the release that first shipped the gRPC role. Field
+# numbers, field names and RPC shapes are permanent from here (§2.2 of
+# tasks/specs/2026-09-01-flexiq-v1-proto-design.md): the `proto-breaking`
+# label is for reverting a mistake made in the same release train, never for
+# changing a shipped shape.
+#
+# The rules this package is held to, because a client reads the .proto and not
+# the design document:
+#
+#   1. Field numbers are never reused and never renumbered.
+#   2. Fields are deprecated in place, and on removal both the number and the
+#      name are reserved.
+#   3. New RPCs, messages, fields and enum values are additive and never bump
+#      the package version.
+#   4. A field's meaning and units are frozen with its number.
+#   5. Readers tolerate what they do not know: unknown fields, unknown enum
+#      values, unknown oneof arms.
+#
+# This package is deliberately separate from flexiq.v1. The audiences,
+# credentials and exposure differ: a producer submits work, an executor runs
+# it. flexiq.executor.v1 may import flexiq.v1; the reverse import is forbidden,
+# so the package a third party generates a producer client from stays
+# compilable and reviewable on its own.
+#
+
+require 'grpc'
+require 'flexiq/executor/v1/executor_service_pb'
+
+module FlexiQ
+  module Executor
+    module V1
+      module ExecutorService
+        # The executor door: attach, run what you are given, report what happened.
+        #
+        # An attached executor holds no database credential. It never reads a job row,
+        # never writes a result, and never resolves its own namespace, owner or attempt
+        # — the scheduler applies every one of those from the dispatch it recorded.
+        # That is what makes this a door an application container may dial out through.
+        #
+        # The frames below are the worker frame protocol in BINDING_CONTRACT.md, with
+        # the same ordering rules and the same meanings. This service is a fourth
+        # transport for that protocol, not a second one: an executor attached over gRPC
+        # and an executor attached over a socket are the same executor to the
+        # scheduler, are placed by the same rules, and appear side by side in the same
+        # registry.
+        #
+        # What this service deliberately does not have:
+        #
+        #   * No HTTP binding, ever. The JSON facade serves flexiq.v1 and nothing else.
+        #   * No admin surface, and no producer surface. A credential scoped to this
+        #     package may run work; it may not submit any.
+        #   * No way to name a namespace, an owner, an attempt or a resource cap.
+        #     Anything an executor could name is something an executor could forge, so
+        #     the scheduler resolves all of them itself.
+        #   * No registry fingerprint on the wire. `hello` carries `tasks` and the
+        #     scheduler derives the fingerprint from it; a wire field would be a second
+        #     copy of one fact, free to disagree with the first.
+        #
+        # One thing a client must do for itself: **raise its own message-size limits.**
+        # gRPC libraries default to 4 MiB in each direction, and this door carries the
+        # payloads the worker frame protocol allows — up to 64 MiB, plus envelope. An
+        # executor that leaves the default in place attaches fine and then fails on its
+        # first large job, which is the worst time to find out.
+        class Service
+
+          include ::GRPC::GenericService
+
+          self.marshal_class_method = :encode
+          self.unmarshal_class_method = :decode
+          self.service_name = 'flexiq.executor.v1.ExecutorService'
+
+          # Attach for the lifetime of one stream: `hello` in, `hello_ack` out, then
+          # jobs out and results in until either side ends it.
+          #
+          # **The stream is bounded on purpose.** A stream cannot be load balanced once
+          # it has started, so the scheduler ends it on a timer and expects the
+          # executor to reconnect. Before it does, it stops matching new work to the
+          # stream and waits for what the executor already holds, so a rotation never
+          # costs a job. A clean end means "reconnect"; a `shutdown` frame means "stop".
+          rpc :Attach, stream(::FlexiQ::Executor::V1::AttachRequest), stream(::FlexiQ::Executor::V1::AttachResponse)
+          # Report liveness and current free capacity.
+          #
+          # Off the dispatch stream on purpose. A heartbeat sharing the stream it is
+          # meant to report on cannot distinguish a busy stream from a dead peer, and
+          # it is the first thing a stream-per-connection design has to take back out.
+          #
+          # `session` comes from the `flexiq-attach-session-bin` metadata on the Attach
+          # response. It is the scheduler's own value handed back: an executor naming
+          # itself here could shrink a peer's advertised capacity by claiming its id.
+          #
+          # The metadata arrives before the handshake finishes, but this RPC is still
+          # ordered behind it: `hello` is the first thing the scheduler reads on a
+          # stream, and a heartbeat that overtakes it is read *as* the handshake and
+          # refuses the attach. Wait for `hello_ack`.
+          rpc :Heartbeat, ::FlexiQ::Executor::V1::HeartbeatRequest, ::FlexiQ::Executor::V1::HeartbeatResponse
+          # Report the outcome of a dispatch that outlived the request it arrived on.
+          #
+          # For a push target, which is handed a job over HTTP and answers `202
+          # Accepted` because the work will not finish inside the platform's request
+          # deadline. The connection that started the job is gone by the time this is
+          # called; the lease inside the frame is what says which dispatch of which
+          # attempt is being settled.
+          #
+          # **Single-use, and fenced.** The scheduler records one settle marker per
+          # accepted dispatch and removes it in the same statement that tests it, so
+          # exactly one of this RPC, the same RPC on another replica, and the settle
+          # deadline passing can win. A call that lost that race is
+          # FAILED_PRECONDITION — never ABORTED, which sits in the retry-with-backoff
+          # class: a settle that lost its fence must not be resent, because resending
+          # it is the double execution the fence exists to refuse.
+          rpc :Settle, ::FlexiQ::Executor::V1::SettleRequest, ::FlexiQ::Executor::V1::SettleResponse
+          # Ask for longer before the attempt is considered lost.
+          #
+          # An accepted dispatch is waited on until a deadline, and a target that
+          # needs longer than the job's timeout must be able to say so or the job is
+          # retried under it. Repeatable: the ceiling is per call, not in total, so a
+          # target that needs six hours asks six times and its asking is the liveness
+          # signal. A target that stops asking is one the deadline will collect.
+          rpc :ExtendLease, ::FlexiQ::Executor::V1::ExtendLeaseRequest, ::FlexiQ::Executor::V1::ExtendLeaseResponse
+          # Report progress for an accepted dispatch.
+          #
+          # The frame the attach stream carries, off the stream — a push target has no
+          # stream to put it on. Fire and forget, as it is there: a task that only
+          # wanted to report progress must never block on the scheduler to do it, and
+          # an empty response says the frame was taken, not that a row was written.
+          rpc :ReportProgress, ::FlexiQ::Executor::V1::ReportProgressRequest, ::FlexiQ::Executor::V1::ReportProgressResponse
+          # Write one structured log line for an accepted dispatch. As
+          # ReportProgress, and with the same fire-and-forget meaning.
+          rpc :WriteTaskLog, ::FlexiQ::Executor::V1::WriteTaskLogRequest, ::FlexiQ::Executor::V1::WriteTaskLogResponse
+        end
+
+        Stub = Service.rpc_stub_class
+      end
+    end
+  end
+end

data/lib/flexiq/executor/version.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module FlexiQ
+  module Executor
+    # Mirrored from the workspace version by scripts/version.mjs; never hand-edit.
+    VERSION = "2.1.0"
+  end
+end

data/lib/flexiq/executor/worker.rb

@@ -0,0 +1,134 @@
+# frozen_string_literal: true
+
+module FlexiQ
+  module Executor
+    # Attaches to a flexiq-server's executor door and runs the jobs it dispatches.
+    #
+    #   worker = FlexiQ::Executor::Worker.new("queue.internal:50051", token: ENV.fetch("FLEXIQ_EXECUTE_TOKEN"))
+    #   worker.handle("billing.charge") { |job| Receipt.for(job.args.first) }
+    #   trap("TERM") { worker.stop }
+    #   worker.run
+    class Worker
+      # Options (all keywords, all but `token:` optional):
+      #
+      # token::              bearer token carrying the `execute` scope
+      # id::                 executor id, unique among attached executors; host and pid by default
+      # slots::              jobs run at once; the processor count by default
+      # insecure::           plaintext gRPC, for loopback and tests only
+      # credentials::        a GRPC::Core::ChannelCredentials; the system roots by default
+      # max_message_bytes::  68 MiB by default, the door's own ceiling
+      # user_agent::         prepended to this gem's own
+      # handshake_timeout::  seconds to wait for `hello_ack` (10)
+      # heartbeat_interval:: seconds between heartbeats (5)
+      # shutdown_drain::     seconds running jobs get to finish on a drain (30)
+      # backoff_min, backoff_max:: reconnect delay bounds in seconds (0.25, 30)
+      # logger::             a Logger; $stderr at INFO by default
+      def initialize(target, **)
+        @config = Config.new(**)
+        @stub = V1::ExecutorService::Stub.new(target, @config.channel_credentials,
+                                              channel_args: @config.channel_args)
+        @handlers = {}
+        @mutex = Mutex.new
+        @wakeup = Queue.new
+        @started = false
+        @stopping = false
+      end
+
+      # Registers the handler for one task name: a block, or anything responding to `call`. It is
+      # given a Job; what it returns is the result, what it raises is the failure.
+      def handle(task_name, callable = nil, &block)
+        handler = callable || block
+        raise ArgumentError, "task name must not be empty" if task_name.to_s.empty?
+        raise ArgumentError, "handler for #{task_name.inspect} must respond to call" unless handler.respond_to?(:call)
+
+        @mutex.synchronize do
+          raise FlexiQ::Error, "cannot register #{task_name.inspect} after run has started" if @started
+          raise ArgumentError, "task #{task_name.inspect} is already registered" if @handlers.key?(task_name.to_s)
+
+          @handlers[task_name.to_s] = handler
+        end
+        self
+      end
+
+      # Attaches and serves until the scheduler sends `shutdown` or `stop` is called. A stream
+      # the scheduler ends is a rotation, not a failure: it reconnects. Raises AttachError for a
+      # refusal that reconnecting cannot fix.
+      def run
+        handlers = begin_run
+        slots = Slots.new(@config.slots)
+        backoff = Backoff.new(@config.backoff_min, @config.backoff_max)
+
+        while (session = next_session(handlers, slots))
+          ending = session.run
+          backoff.reset if ending.attached
+          return if finished?(ending)
+          return if retry_after?(ending) && pause(backoff.next_delay)
+        end
+      end
+
+      # Drains and ends `run`: no new work, running jobs get `shutdown_drain` seconds, their
+      # results are flushed. Returns at once, and is safe to call from a signal handler.
+      def stop
+        Thread.new do
+          session = @mutex.synchronize do
+            @stopping = true
+            @session
+          end
+          @wakeup << true
+          session&.begin_drain(:stopped)
+        end
+        nil
+      end
+
+      private
+
+      def begin_run
+        @mutex.synchronize do
+          raise FlexiQ::Error, "run has already been called on this worker" if @started
+          if @handlers.empty?
+            raise FlexiQ::Error, "no handlers registered: an executor advertising no tasks is sent no work"
+          end
+
+          @started = true
+          @handlers.dup.freeze
+        end
+      end
+
+      def next_session(handlers, slots)
+        @mutex.synchronize do
+          @session = @stopping ? nil : Session.new(@config, @stub, handlers, slots)
+        end
+      end
+
+      def finished?(ending)
+        case ending.reason
+        when :shutdown
+          @config.logger.info("the scheduler sent shutdown; stopping")
+          true
+        when :stopped then true
+        else false
+        end
+      end
+
+      # True when the next attach should wait first. A rotation reconnects at once; a failure,
+      # or a stream that ended before it ever attached, backs off.
+      def retry_after?(ending)
+        if ending.reason == :rotated && ending.attached
+          @config.logger.info("the scheduler ended the stream; reconnecting")
+          return false
+        end
+        error = ending.error
+        raise error if error&.permanent?
+
+        @config.logger.warn("attach failed; retrying: #{error&.message || "the stream ended before hello_ack"}")
+        true
+      end
+
+      # Sleeps `delay` seconds; true when `stop` cut the wait short.
+      def pause(delay)
+        @wakeup.pop(timeout: delay)
+        @mutex.synchronize { @stopping }
+      end
+    end
+  end
+end

data/lib/flexiq/executor.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+# FlexiQ executor client over a flexiq-server's executor door (flexiq.executor.v1).
+# The wire contract is contracts/REMOTE_SDK_CONTRACT.md in the FlexiQ repository.
+#
+# An executor cannot enqueue: the door has no enqueue-shaped RPC. A task that fans out goes back
+# through the producer door with the `flexiq` gem and a produce-scoped credential of its own.
+
+require "flexiq"
+require "grpc"
+
+require_relative "executor/version"
+require_relative "executor/v1/executor_service_services_pb"
+
+require_relative "executor/clock"
+require_relative "executor/errors"
+require_relative "executor/config"
+require_relative "executor/job"
+require_relative "executor/outcome"
+require_relative "executor/outbox"
+require_relative "executor/slots"
+require_relative "executor/backoff"
+require_relative "executor/session"
+require_relative "executor/worker"