flezha 0.1.0-x86_64-linux

data/lib/flezha/executor.rb

@@ -0,0 +1,329 @@
+# frozen_string_literal: true
+
+require "etc"
+
+require_relative "dispatch"
+require_relative "logger"
+
+module Flezha
+  # The concurrency host: what runs `poll -> dispatch -> complete`, and on what.
+  #
+  # Two members in v1, and the seam between them may never observe or affect
+  # per-cell ordering. Exclusivity is the core's handout guarantee — one
+  # outstanding activation per Cell — and an activation is completed only
+  # after its dispatch has returned, so ordering holds for any executor that
+  # does not complete early. That invariant is this seam's entire contract.
+  module Executor
+    # What `pool_supervise` answers: spawn a base worker, stop, spawn a grown
+    # one; anything else is nothing to do this wait.
+    POOL_SPAWN_BASE = 1
+    POOL_STOP = 2
+    POOL_SPAWN_GROWN = 3
+
+    module_function
+
+    # One fetch loop, and the one place the three answers `poll` can give are
+    # interpreted.
+    #
+    # Both executors fetch, and the deferred Ractor one will too — so what a
+    # timeout, an activation and a shutdown each mean lives here rather than in
+    # one correct copy per executor. The block is yielded each activation. The
+    # loop ends when `stopping` answers true, when the core says it is shutting
+    # down, and after `retire_ms` of timed-out polls when that is positive: the
+    # governor's retire time, for a grown worker.
+    def fetch(core, poll_ms, stopping, retire_ms = 0)
+      idle = 0
+      until stopping.call
+        activation = core.poll(poll_ms)
+        return :shutting_down if activation == :shutting_down
+
+        if activation.nil?
+          idle += poll_ms
+          return :retired if retire_ms.positive? && idle >= retire_ms
+        else
+          idle = 0
+          yield activation
+        end
+      end
+      :stopped
+    end
+
+    # When a drain of `seconds` must be finished by.
+    def deadline(seconds) = Process.clock_gettime(Process::CLOCK_MONOTONIC) + seconds
+
+    # How long is left of it, never negative — `Thread#join(0)` polls rather
+    # than blocking, which is what a passed deadline should do.
+    def remaining(deadline)
+      [deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC), 0].max
+    end
+
+    # Say what the drain did not finish. Silence here is the failure mode the
+    # bound exists to avoid reintroducing: a runner that leaves without
+    # reporting what it abandoned looks identical to one that drained.
+    def report_undrained(count, seconds)
+      return if count.zero?
+
+      Flezha.log.warn(
+        "#{count} activation(s) were still running after the #{seconds}s drain; the " \
+        "runner is exiting anyway. The core re-hands unanswered work on the next bind — find " \
+        "the handler that does not return."
+      )
+    end
+
+    # What both executors share: the core's pool governor sizes the pool over
+    # this machine's processor count, and its supervisor's answers are what
+    # this host spawns. A host only brings the threads.
+    module Pooled
+      attr_reader :base, :ceiling
+
+      private
+
+      def open_pool(core)
+        @core = core
+        @base, @ceiling, @poll_ms, @retire_ms, drain_ms = core.pool_open(Etc.nprocessors)
+        @drain_s = drain_ms / 1000.0
+      end
+
+      # The base workers into the seats the governor reserved, then its supervisor.
+      def start_pool
+        @base.times { start_worker(false) }
+        @supervisor = Thread.new { supervise }
+        @supervisor.name = "flezha-supervisor"
+      end
+
+      # One governor answer per wait, at most a second apart, and the core's
+      # notices written on each, so an idle runner's notices still reach its log.
+      def supervise
+        until @stopping
+          answer = @core.pool_supervise
+          Flezha.log_notices(@core.take_notices)
+          break if answer == POOL_STOP || @stopping
+
+          start_worker(answer == POOL_SPAWN_GROWN) if [POOL_SPAWN_BASE, POOL_SPAWN_GROWN].include?(answer)
+        end
+      rescue StandardError => e
+        Flezha.log.error("the pool supervisor ended on #{e.class}: #{e.message}")
+      end
+
+      # A seat the governor reserved; a thread that cannot start gives it back.
+      def start_worker(grown)
+        spawn_member(grown)
+      rescue ThreadError => e
+        @core.pool_left
+        Flezha.log.warn("could not start a worker thread: #{e.message}")
+      end
+
+      # A worker that dispatches on itself: the thread executor's every worker,
+      # and a grown seat under either, because a parked completion needs a
+      # thread and a stalled scheduler cannot supply one.
+      def run_worker(grown)
+        Executor.fetch(@core, @poll_ms, -> { @stopping }, grown ? @retire_ms : 0) do |activation|
+          Dispatch.dispatch(activation, @core, @deps)
+        end
+      rescue StandardError => e
+        # A worker that dies silently is a pool that shrinks with no report;
+        # the governor replaces it, but only this line says why it left.
+        Flezha.log.error("a worker thread ended on #{e.class}: #{e.message}")
+      ensure
+        @core.pool_left
+      end
+    end
+  end
+
+  # The default executor: a base pool of Ruby threads each looping
+  # `poll -> dispatch -> complete` on itself, plus the core pool governor's
+  # supervisor, whose answers grow the pool under starvation and restore it
+  # when a worker left.
+  #
+  # This is the Rust host's model rather than Java's hand-off, and the reason
+  # is the reason behind both shipped hosts: Java hands off because a parked
+  # virtual thread costs a stack and nothing else — which is also exactly why
+  # Java's fetchers never wait on user code. An MRI thread is a
+  # real OS thread with a real stack, so hand-off would buy nothing and cost a
+  # thread per activation up to the core's in-flight cap.
+  #
+  # Concurrency is bounded by the core's `FLEZHA_MAX_INFLIGHT_ACTIVATIONS`,
+  # never by the pool size. The pool is a servicing detail.
+  class ThreadExecutor
+    include Executor::Pooled
+
+    def initialize(core, deps)
+      open_pool(core)
+      @deps = deps
+      @workers = []
+      @supervisor = nil
+      @lock = Mutex.new
+      @stopping = false
+    end
+
+    def name = "threads"
+
+    # How many workers are alive now. Reported in the startup line and read by
+    # the pool-growth tests.
+    def size = @lock.synchronize { @workers.count(&:alive?) }
+
+    def start
+      start_pool
+      self
+    end
+
+    # Stop fetching without the core's word. A local run ends while the
+    # process-wide poll core keeps running — there is no controller session to
+    # shut down, and shutting the core would stop every later run in the
+    # process — so the flag is raised first and the workers leave on their next
+    # poll timeout instead of on the drain deadline.
+    def stop_fetching
+      @stopping = true
+    end
+
+    # Stop fetching and drain. The core says when: a worker leaves on the poll
+    # that reports the core shutting down, which the core reports only once
+    # the shutdown sequence has shipped every Cell's state through these
+    # same workers; a worker inside a handler finishes it, completes it, and
+    # only then observes it. Bounded by the governor's drain bound — a handler
+    # that never returns must not keep the runner alive forever — and the flag
+    # is raised after the join so a worker spawned late still leaves.
+    def stop
+      deadline = Executor.deadline(@drain_s)
+      @lock.synchronize { @workers.dup }.each { |worker| worker.join(Executor.remaining(deadline)) }
+      @stopping = true
+      @supervisor&.join(Executor.remaining(deadline))
+      undrained = @lock.synchronize { @workers.count(&:alive?) }
+      Executor.report_undrained(undrained, @drain_s)
+      @lock.synchronize { @workers.clear }
+      Flezha.log_notices(@core.take_notices)
+      undrained
+    end
+
+    private
+
+    def spawn_member(grown)
+      @lock.synchronize do
+        @workers.select!(&:alive?)
+        worker = Thread.new { run_worker(grown) }
+        worker.name = "flezha-worker-#{@workers.size}"
+        @workers << worker
+      end
+    end
+  end
+
+  # The opt-in executor for I/O-heavy handlers: real-thread fetchers hand
+  # activations to fibers through a `Thread::Queue`, which is scheduler-aware,
+  # so a handler's I/O yields instead of blocking a thread.
+  #
+  # The fetcher stays a real thread under this executor too. `poll` blocks in
+  # native code, and `docs/internal/sdk/poll_core.md` is explicit that fetching on a
+  # runtime's shared I/O pool starves everything else that pool serves.
+  #
+  # Completion happens on the fiber that dispatched, not on the fetcher: the
+  # ABI accepts a completion from any thread, and the ordering guarantee is the
+  # core's handout, not this executor's.
+  class FiberExecutor
+    include Executor::Pooled
+
+    # The queue depth between the fetchers and the fibers. Bounded so a fetcher
+    # cannot run arbitrarily far ahead of the handlers; the core's in-flight
+    # cap bounds the total either way.
+    QUEUE_DEPTH = 64
+
+    def initialize(core, deps, scheduler: nil)
+      open_pool(core)
+      @deps = deps
+      @scheduler = scheduler
+      @queue = Thread::SizedQueue.new(QUEUE_DEPTH)
+      @threads = []
+      @stopping = false
+    end
+
+    def name = "fibers"
+    def size = @threads.count(&:alive?)
+
+    def start
+      @dispatcher = Thread.new { run_dispatcher }
+      @dispatcher.name = "flezha-fibers"
+      start_pool
+      self
+    end
+
+    # Stop fetching without the core's word; see the thread executor's.
+    def stop_fetching
+      @stopping = true
+    end
+
+    # The fetchers leave on the core's word, as the thread executor's workers
+    # do; the flag is raised after their join.
+    def stop
+      deadline = Executor.deadline(@drain_s)
+      @threads.dup.each { |thread| thread.join(Executor.remaining(deadline)) }
+      @stopping = true
+      @supervisor&.join(Executor.remaining(deadline))
+      # Closing the queue is what ends the dispatcher: a scheduler-aware pop
+      # answers nil on a closed queue rather than parking forever.
+      @queue.close
+      # Bounded for the reason the thread executor's join is: closing a
+      # scheduler runs its outstanding fibers to completion, and a fiber that
+      # never returns would hold the drain open forever.
+      @dispatcher&.join(Executor.remaining(deadline))
+      undrained = [@dispatcher, *@threads].count { |thread| thread&.alive? }
+      Executor.report_undrained(undrained, @drain_s)
+      Flezha.log_notices(@core.take_notices)
+      undrained
+    end
+
+    private
+
+    # A fetcher for a base seat; a grown seat is a worker that dispatches on
+    # itself, since a scheduler stalled in a parked completion cannot rescue itself.
+    def spawn_member(grown)
+      @threads.select!(&:alive?)
+      thread = Thread.new { grown ? run_worker(true) : run_fetcher }
+      thread.name = "flezha-fetcher-#{@threads.size}"
+      @threads << thread
+    end
+
+    def run_fetcher
+      Executor.fetch(@core, @poll_ms, -> { @stopping }) { |activation| @queue.push(activation) }
+    rescue ClosedQueueError
+      nil
+    rescue StandardError => e
+      Flezha.log.error("a fetcher thread ended on #{e.class}: #{e.message}")
+    ensure
+      @core.pool_left
+    end
+
+    # The pop runs in a scheduled fiber, so a quiet queue yields to the event
+    # loop and a handler fiber is resumed as soon as its I/O is ready. That loop
+    # is the scheduler's close, which Ruby runs when this thread ends and which
+    # runs every fiber to completion, the pop's included: a drain here means
+    # what it means under the thread executor. Closing the scheduler by hand as
+    # well would be a second close, which the scheduler refuses.
+    def run_dispatcher
+      Fiber.set_scheduler(build_scheduler)
+      Fiber.schedule { pop_and_dispatch }
+    rescue StandardError => e
+      Flezha.log.error("the fiber dispatcher ended on #{e.class}: #{e.message}")
+    end
+
+    def pop_and_dispatch
+      while (activation = @queue.pop)
+        Fiber.schedule { Dispatch.dispatch(activation, @core, @deps) }
+      end
+    rescue StandardError => e
+      Flezha.log.error("the fiber dispatcher ended on #{e.class}: #{e.message}")
+    end
+
+    def build_scheduler
+      return @scheduler.call if @scheduler
+
+      begin
+        require "async"
+      rescue LoadError
+        raise Error.new("runtime.executor_unavailable",
+                        executor: "fibers",
+                        reason: "it needs a Fiber::Scheduler; add the `async` gem, or pass `scheduler:` " \
+                                "a callable answering one")
+      end
+      Async::Scheduler.new
+    end
+  end
+end

data/lib/flezha/graph.rb

@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+
+require "json"
+
+require_relative "builder"
+require_relative "errors"
+
+module Flezha
+  # The `HostClassDecl` the core is handed, and the verdict it answers.
+  #
+  # The topology is judged in ONE place, `flz_core::host::register::check`:
+  # this module records what the author declared, submits it, and the core's
+  # refusal reaches the author as the `Flezha::Error` it raises.
+  module Graph
+    # The Ruby SDK's byte in the core's `sdk_lang` allocation table.
+    SDK_LANG_RUBY = 5
+
+    # This SDK's own released version, stamped into every durable record it
+    # writes; the core packs it. `Flezha::VERSION` is the single literal
+    # `lib/flezha/version.rb` holds and the gemspec already reads, so the gem
+    # cannot ship a version its records disagree with.
+    SDK_VERSION = Flezha::VERSION
+
+    # MessagePack, shared with every SDK whose codec is MessagePack.
+    FORMAT_ID_MSGPACK = 5
+
+    module_function
+
+    # Judge one declaration and answer what the core derived from it: the
+    # channel names, kinds, consumers, output names and acknowledgement
+    # granularity it read off the record.
+    #
+    # Judge, register and declare one class, in the one order that works.
+    #
+    # `validate` writes the derived channel names and consumer edges back into
+    # the spec, so a caller that skipped it would register a spec whose edges
+    # route nowhere. Both the runner and local mode reach the core through
+    # here, so there is one such order rather than two that agree.
+    def declare(spec, core, registry)
+      validate(spec, core)
+      registry.add(spec)
+      core.declare_class(JSON.generate(to_host_class_decl(spec)))
+    end
+
+    def validate(spec, core)
+      derived = submit(spec, core)
+      bind_edges(spec, derived)
+      derived
+    end
+
+    # The record, over the ABI, and the verdict back. A refusal raises the
+    # core's `Flezha::Error`: its code is the first issue's, and
+    # `details["issues"]` lists every issue as `{code, message, details}`.
+    def submit(spec, core)
+      JSON.parse(core.check_class(JSON.generate(to_host_class_decl(spec))))
+    end
+
+    # The consumers the CORE derived, each carrying the closures of the chain
+    # that produced it: the builder held them aside per chain, and the derived
+    # edge names the chain. Which edge survives a re-declaration is therefore
+    # the core's answer, not a rule this SDK repeats.
+    def bind_edges(spec, derived)
+      derived["channels"].each do |channel|
+        target = spec.channels[channel["index"]]
+        target.consumers = channel["consumers"].map { |edge| target.chain_edges[edge["chain"]] }
+        target.derived_kind = channel["kind"]
+      end
+    end
+
+    # Drop the keys whose value is nil, so the JSON omits them.
+    def compact(hash) = hash.reject { |_key, value| value.nil? }
+
+    # The channel as the record carries it: the kind the author declared and
+    # the sequences declared off its handle. The consumers, the plugin kind and
+    # the output name are the core's to DERIVE from those chains.
+    def channel_json(channel)
+      compact(
+        "name" => channel.name,
+        "kind" => channel.kind.to_s,
+        "producer" => channel.producer,
+        "chains" => channel.chains,
+        "retry_max" => channel.retry_max,
+        "retry_backoff_ms" => channel.retry_backoff_ms,
+        "overflow" => channel.overflow.to_s,
+        "keyed" => channel.keyed,
+        "codec" => channel.codec,
+        "schema" => channel.schema,
+        "description" => channel.description
+      )
+    end
+
+    def tasklet_json(node)
+      definition = node.definition
+      compact(
+        "name" => node.name,
+        "cell_keys" => definition.cell_keys || [],
+        "idle_ttl_ms" => definition.idle_ttl_ms,
+        "max_cells" => definition.max_cells,
+        "cells_for_overridden" => definition.cells_for?,
+        "compensate_overridden" => definition.compensate?,
+        "clock" => definition.clock.to_s,
+        "mailbox_capacity" => definition.mailbox_capacity,
+        "max_timers_per_cell" => definition.max_timers_per_cell,
+        # A tasklet with no state journals nothing, so a logical clock over
+        # it never advances. Ruby has no per-tasklet Change type; having a
+        # state IS the same statement.
+        "change_is_nochange" => !definition.stateful?,
+        "schema_version" => definition.schema_version || 1,
+        "min_readable_schema_version" => definition.min_readable_schema_version,
+        "migrates" => definition.migrate_state? || definition.migrate_change?
+      )
+    end
+
+    # The declaration the core deserializes. The settings tree goes as the
+    # author wrote it: the core judges it and refuses a misplaced setting by
+    # name. Every key is pinned by
+    # `test/fixtures/host_class_decl_keys.txt`, which this gem's own test
+    # reads: a key the core learns that Ruby cannot spell fails that test.
+    def to_host_class_decl(spec)
+      compact(
+        "factory_id" => spec.factory_id,
+        "description" => spec.description,
+        "max_duration_ms" => spec.max_duration_ms,
+        "run" => spec.run,
+        "durable" => spec.durable,
+        "sdk_lang" => SDK_LANG_RUBY,
+        "sdk_version" => SDK_VERSION,
+        "change_format_id" => FORMAT_ID_MSGPACK,
+        "compact_after_bytes" => spec.tasklets.filter_map { |node| node.definition.compact_after_bytes }.min,
+        "channels" => spec.channels.map { |channel| channel_json(channel) },
+        "tasklets" => spec.tasklets.map { |node| tasklet_json(node) }
+      )
+    end
+  end
+end

data/lib/flezha/identity.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Flezha
+  # The replay-stable identifier of a cell.
+  #
+  # The algorithm is fixed by the SDK contract rather than by a library, so
+  # every SDK derives the same value from the same cell: FNV-1a over
+  # `identity | change seq | per-invocation ordinal`, where the identity is
+  # the bytes the core hands with the cell's first activation.
+  module Identity
+    FNV128_OFFSET_BASIS = 0x6c62272e07bb014262b821756295c58d
+    FNV128_PRIME = 0x0000000001000000000000000000013b
+    MASK128 = (1 << 128) - 1
+
+    module_function
+
+    def fnv1a128(seed, bytes)
+      hash = seed
+      bytes.each_byte do |byte|
+        hash = (hash ^ byte) & MASK128
+        hash = (hash * FNV128_PRIME) & MASK128
+      end
+      hash
+    end
+
+    # Thirty-two lowercase hex characters. Repeated calls in one invocation
+    # differ; the n-th call of a replayed invocation equals the n-th call of the
+    # original, which is what makes it the replacement for a random UUID.
+    def stable_id(identity, change_seq, ordinal)
+      hash = fnv1a128(FNV128_OFFSET_BASIS, identity)
+      hash = fnv1a128(hash, [change_seq & 0xffffffffffffffff].pack("Q>"))
+      hash = fnv1a128(hash, [ordinal & 0xffffffff].pack("N"))
+      format("%032x", hash)
+    end
+  end
+end

data/lib/flezha/local.rb

@@ -0,0 +1,146 @@
+# frozen_string_literal: true
+
+module Flezha
+  # One value a graph released on a declared output. `metadata` is what a
+  # deployed run would have put on the wire, `flz.*` stamps included.
+  LocalOutput = Struct.new(:channel, :payload, :metadata, keyword_init: true)
+
+  # Running a declared relay graph in this process, with no controller.
+  #
+  # `LocalRelay.start` takes the relay class you deploy and runs the
+  # `channels` graph it already declares — no local-mode member on the class,
+  # no second declaration. What runs is the deployed machinery: entry decode,
+  # edge transforms and keying, one Cell per resolved key with its own
+  # mailbox, the retry ladder, and inter-tasklet delivery performed by the
+  # runtime.
+  #
+  #   run = Flezha::LocalRelay.start(WordCount)
+  #   run.on_output("totals") { |output| puts output.payload }
+  #   run.feed("lines", "the quick brown fox")
+  #   run.run_until_idle(5_000)
+  #   run.close
+  #
+  # State is held in memory and nothing is durable. Durability, plugins,
+  # placement and migration, the dead-letter queue, observability export,
+  # authz and multi-runner scale all need a controller; the run says so through
+  # the SDK's logger every time it starts. The "Local mode" section of
+  # https://github.com/flezha/flezha/blob/main/docs/public/topics/core/running_and_testing.md#local-mode
+  # walks through it.
+  class LocalRelay
+    # Bounded so a stopped run's drain thread leaves promptly.
+    OUTPUT_POLL_MS = 100
+
+    # Register +klass+'s declared graph and start it locally. Raises naming
+    # the capability when the declaration needs a controller.
+    def self.start(klass)
+      core = Core.create
+      begin
+        spec = klass.spec
+        registry = Registry.new
+        Graph.declare(spec, core, registry)
+        # Not `start`, which is the one call that dials a controller.
+        core.local_start(spec.factory_id, "#{spec.factory_id}-local")
+        deps = DispatchDeps.new(registry: registry, bridge: core)
+        executor = ThreadExecutor.new(core, deps)
+        executor.start
+      rescue Exception # rubocop:disable Lint/RescueException
+        core.free
+        raise
+      end
+      new(core, executor)
+    end
+
+    def initialize(core, executor)
+      @core = core
+      @executor = executor
+      @sinks = {}
+      @closed = false
+      @drain = Thread.new { drain_outputs }
+      @drain.name = "flz-local-output"
+    end
+
+    # Deliver every value released on the output +channel+ to the block, in
+    # release order. An output channel takes the producing tasklet's name
+    # unless the declaration named it.
+    def on_output(channel, &callback)
+      # Registered with the core first: a channel the declaration does not
+      # carry is refused there, naming it, rather than silently accepted.
+      @core.local_on_sink(channel)
+      @sinks[channel] = callback
+      self
+    end
+
+    # Feed one value's wire bytes to a declared entry channel.
+    def feed(channel, payload)
+      @core.local_feed(channel, payload.to_s)
+      self
+    end
+
+    # Block until the graph has nothing left to do. A budget that elapses
+    # first is reported, never passed off as done.
+    def run_until_idle(budget_ms)
+      @core.local_run_until_idle(budget_ms)
+    end
+
+    # End the run and release its Cells. Idempotent.
+    def close
+      return if @closed
+
+      @closed = true
+      begin
+        # `join` re-raises whatever the drain thread died of, so the teardown
+        # below is an `ensure`: a failed delivery must not strand the context.
+        @drain.join(OUTPUT_POLL_MS / 1000.0 * 20)
+        # Stop the run first, then take what it released and the drain thread
+        # had not reached: a value can be queued for up to one poll interval,
+        # and ending the run while it sits there is how an author loses output
+        # the graph did produce. Stopping drops the sender, so this ends at the
+        # first empty read rather than waiting.
+        @core.local_stop
+        while (output = @core.local_next_output(0))
+          deliver(output)
+        end
+      ensure
+        # Not `Core#shutdown`: that stops the process-wide poll core, and a
+        # process may start another local run after this one. The workers
+        # leave on the flag instead, which is why it is raised before the
+        # drain rather than after it.
+        @executor.stop_fetching
+        undrained = @executor.stop
+        # As the runner does: a worker the bounded drain gave up on may yet
+        # call the core, and freeing the context under it is a use-after-free.
+        @core.free if undrained.zero?
+      end
+    end
+
+    private
+
+    # One released value per pass, handed to whoever registered its channel.
+    def drain_outputs
+      until @closed
+        begin
+          output = @core.local_next_output(OUTPUT_POLL_MS)
+        rescue StandardError
+          # The run ended under us; the close path owns the teardown.
+          return
+        end
+        deliver(output) if output
+      end
+    end
+
+    def deliver(output)
+      unless output[:target] == :plugin
+        # The core queues what an output released and nothing else; an edge
+        # output is an in-process mailbox push that never leaves.
+        raise "local mode released a #{output[:target]} output"
+      end
+
+      channel = output[:plugin_channel]
+      callback = @sinks[channel]
+      return if callback.nil?
+
+      callback.call(LocalOutput.new(channel: channel, payload: output[:payload],
+                                    metadata: (output[:metadata] || []).to_h))
+    end
+  end
+end