flezha 0.1.0-x86_64-linux

data/ext/flz_core_ffi/Cargo.toml

@@ -0,0 +1,29 @@
+# The wrapper crate flezha's native binary is built from when it is built from
+# source outside the core's repository.
+#
+# It carries no code: the veneer is a feature of `flz_core_ffi`, which
+# `sdk-core-abi` makes a requirement rather than a preference. All this
+# manifest does is name the core at a tag, turn on the one feature the gem's native extension
+# needs, and ask for a `cdylib`. Switching to a registry is a change to the
+# dependency line and nothing else.
+[package]
+name = "flezha-ruby-native"
+version = "0.1.0"
+edition = "2021"
+publish = false
+
+# Detached from any workspace. In this repository the file sits inside the
+# core's own cargo workspace, and a member of it would inherit a lockfile and
+# a target directory an SDK checkout does not have.
+[workspace]
+
+# The artefact keeps the core's own name, so the build scripts that stage it
+# and the loaders that find it read one file name whichever side built it.
+[lib]
+name = "flz_core_ffi"
+crate-type = ["cdylib"]
+
+[dependencies]
+# Renamed, because a dependency spelled like this crate's own lib target is
+# ambiguous at the glob below.
+veneer = { package = "flz_core_ffi", version = "0.1.0", git = "https://github.com/flezha/flezha", tag = "v0.1.0", features = ["ruby"] }

data/ext/flz_core_ffi/extconf.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+# Build the native veneer for the source gem.
+#
+# A precompiled platform gem ships the library and needs no toolchain at all;
+# that is the primary distribution, and this file is the fallback for platforms
+# it does not cover. The fallback builds the wrapper crate beside this file,
+# whose one dependency is the core at a git tag, so it needs cargo and git and
+# says so rather than failing inside cargo.
+
+require "mkmf"
+
+ext_dir = __dir__
+prebuilt = File.expand_path("../../lib/flezha/flz_core_ffi.#{RbConfig::CONFIG['DLEXT']}", ext_dir)
+
+# A checkout whose library is already built, installed in place by Bundler.
+# Only in place: rake-compiler builds in a directory of its own and copies what
+# make produced, so a Makefile that produces nothing would fail it.
+if File.exist?(prebuilt) && File.expand_path(Dir.pwd) == ext_dir
+  File.write("Makefile", "all:\n\t@true\ninstall:\n\t@true\nclean:\n\t@true\n")
+  exit 0
+end
+
+missing = %w[cargo git].reject { |tool| find_executable(tool) }
+unless missing.empty?
+  abort <<~MESSAGE
+    flezha: building the source gem needs cargo and git (#{missing.join(' and ')} not found):
+    it compiles the core from its git tag. Install a Rust toolchain and git, or install
+    the precompiled platform gem instead: gem install flezha --platform #{Gem::Platform.local}
+  MESSAGE
+end
+
+require "rb_sys/mkmf"
+
+create_rust_makefile("flezha/flz_core_ffi") do |r|
+  # rb-sys runs bindgen over ruby.h, and bindgen needs clang's builtin headers.
+  # Without libclang-dev they are absent, and gcc's builtin include directory
+  # carries the same ones.
+  if ENV["BINDGEN_EXTRA_CLANG_ARGS"].nil? && Dir.glob("/usr/lib/llvm-*/lib/clang/*/include/stdarg.h").empty?
+    gcc_include = `gcc -print-file-name=include 2>/dev/null`.strip
+    r.env = { "BINDGEN_EXTRA_CLANG_ARGS" => "-I#{gcc_include}" } if File.file?(File.join(gcc_include, "stdarg.h"))
+  end
+end

data/ext/flz_core_ffi/src/lib.rs

@@ -0,0 +1,7 @@
+//! Nothing of this crate's own: the gem's native extension IS `flz_core_ffi` built with one
+//! feature, and this re-export is what puts its entry points in a `cdylib`'s
+//! export table. A `cdylib` exports no `#[no_mangle]` symbol of a dependency
+//! it never names. Measured, not assumed: without the glob the binary
+//! links and exports nothing at all.
+
+pub use ::veneer::*;

data/lib/flezha/builder.rb

@@ -0,0 +1,504 @@
+# frozen_string_literal: true
+
+require_relative "errors"
+
+module Flezha
+  # A tasklet as the graph holds it, with its position in the declaration.
+  # `output` is the one channel it produces on, created on first wiring.
+  TaskNode = Struct.new(:index, :name, :klass, :definition, :output)
+
+  # One consumer of a channel, with the transforms and key of its own edge.
+  ConsumerEdge = Struct.new(:tasklet, :transforms, :key_by, keyword_init: true)
+
+  # A channel's call end: the transforms and key of the chain that closed on
+  # it, applied to a value before it becomes a request, and the request's
+  # encoder.
+  CallEnd = Struct.new(:transforms, :key_by, :encode, keyword_init: true)
+
+  # One fluent sequence declared off a channel handle, as the record carries
+  # it: `links` in the order the author called them, for the core to judge.
+  # Every transform or `key_by` answers a new sequence over the previous,
+  # which is what lets a shared prefix branch without the arms colliding;
+  # `superseded` marks the prefix a later link built on, so only a tip reaches
+  # the record. `edge` is the consumer an `into` sequence closes on, held aside
+  # for the core to name back.
+  Chain = Struct.new(:source, :links, :superseded, :edge, keyword_init: true)
+
+  # A channel as the graph holds it. Its position in the list IS its id.
+  # `output` is the channel's name once a plugin end, an output or a call, is
+  # attached; nil means none, and the core's answer names no output for a call.
+  # `sink_encode` is that end's payload encoder and belongs to the
+  # plugin frame alone: an inter-tasklet frame is decoded with the class codec,
+  # so encoding one with it hands the consuming tasklet bytes it cannot read.
+  # `call` is the channel's call end, when a chain closes on one.
+  # `codec` is the format the relay's API announces (an entry's input, an
+  # output's value), and `schema` and `description` an entry's own.
+  #
+  # `chain_edges` runs parallel to `chains`: the closures each sequence closed
+  # on, at the position the core reads that sequence at. `consumers` and
+  # `derived_kind` are filled from the derived declaration — the core says
+  # which chains became edges and what kind the channel is.
+  ChannelSpec = Struct.new(
+    :index, :name, :kind, :derived_kind, :producer, :consumers, :chains, :chain_edges, :overflow,
+    :retry_max, :retry_backoff_ms, :keyed, :sink, :decode, :sink_encode, :call,
+    :codec, :schema, :description,
+    keyword_init: true
+  )
+
+  # Everything the graph knows about one declared class.
+  # `run` and `durable` are the tree's JSON as written, `nil` when unwritten.
+  ClassSpec = Struct.new(
+    :factory_id, :description, :max_duration_ms, :run, :durable,
+    :tasklets, :channels,
+    keyword_init: true
+  ) do
+    # Whether this class runs in steps, whose producers release their values
+    # untransformed.
+    #
+    # A steps stage releases ONE value per emit and the driver fans it out
+    # to every consumer itself, so each consumer edge (its transforms and its
+    # key) belongs to the consumer, which applies it on the input the core
+    # marks untransformed.
+    def runs_in_steps? = ClassSpec.branch(run) == "steps"
+
+    # The branch a tree setting names: its bare name, or its one key.
+    def self.branch(tree)
+      return "false" if tree == false
+      return tree if tree.is_a?(String)
+
+      tree.keys.first if tree.is_a?(Hash) && tree.size == 1
+    end
+  end
+
+  # Run an edge's transform chain over one value and answer the values that
+  # come out of it. Every link answers the list one value becomes: none when a
+  # filter dropped it, one when it was mapped, several when a flat_map fanned
+  # it. A later link runs over every value an earlier one yielded, and a value
+  # that is itself an Array stays one value.
+  #
+  # One implementation for the three places a chain runs — an entry payload
+  # before its handler, a payload at the prepare crossing, and an emit on its
+  # way out — so a filter cannot mean one thing on one path and another
+  # elsewhere.
+  def self.apply_transforms(transforms, value)
+    transforms.reduce([value]) do |values, transform|
+      values.flat_map { |current| transform.call(current) }
+    end
+  end
+
+  # The fluent channel builder a relay's `channels` block receives.
+  class ChannelBuilder
+    attr_reader :factory_id, :tasklets, :channels
+
+    def initialize(factory_id)
+      @factory_id = factory_id
+      @tasklets = []
+      @channels = []
+      # The links each fluent sequence was declared with, in declaration
+      # order. Recorded, never judged: the core reads them and answers.
+      @chains = []
+      @merge_groups = 0
+    end
+
+    # Declare a controller-fed entry channel. `decode` turns the wire bytes
+    # into the value a handler sees, `codec` names a format from
+    # `Codec::CHANNEL_CODECS` instead, and with neither the bytes are JSON.
+    # `description` and a `json` channel's JSON Schema `schema` are announced
+    # in the relay's API.
+    def input(name, decode: nil, codec: nil, description: nil, schema: nil)
+      channel = ChannelSpec.new(
+        index: @channels.size, name: name.to_s, kind: :entry, consumers: [],
+        chains: [], chain_edges: [], overflow: :drop_nothing, keyed: false,
+        description: description, schema: schema
+      )
+      named = channel_codec(channel, codec, decode)
+      channel.decode = decode || Codec.channel_decoder(named, channel)
+      channel.codec = api_codec(named, decode)
+      @channels << channel
+      Stream.new(self, [:entry, channel], {})
+    end
+
+    # The codec a channel names, refused when it is unknown or named beside
+    # the channel's own decoder or encoder; nil when it names none.
+    def channel_codec(channel, codec, callable)
+      return nil if codec.nil?
+
+      raise Error.new("topology.codec_with_callable", channel: channel.name) if callable
+
+      known = Codec::CHANNEL_CODECS.find { |name| name.to_s == codec.to_s }
+      return known if known
+
+      raise Error.new("topology.unknown_codec", channel: channel.name, codec: codec.to_s,
+                                                codecs: Codec::CHANNEL_CODECS.map(&:to_s))
+    end
+
+    # The format the relay's API announces for a channel: the codec it names,
+    # `custom` for its own decoder or encoder, else JSON.
+    def api_codec(named, callable) = callable ? "custom" : (named || :json).to_s
+
+    # Merge two or more sources into one tasklet: see `Merge`.
+    def merge = Merge.new(self)
+
+    # The id that ties one merge's sources together in the record.
+    def next_merge_group
+      group = @merge_groups
+      @merge_groups += 1
+      group
+    end
+
+    # Turn a relay-wide document stream into per-key deliveries: every key
+    # seen on `traffic` receives each distinct document once, as
+    # `[key, document]`, on that key's own next traffic item. The block answers
+    # a traffic item's key.
+    #
+    # A key never seen in traffic receives nothing, and a key learns of a new
+    # document one traffic item late. Fanning a document out over a remembered
+    # key set instead would emit keys this member may no longer own after a
+    # rescale; a key its own traffic just reached is one it owns.
+    #
+    # Sugar over a merge into one packaged `RouteLatest` node named
+    # `route_latest`, so a second router in one relay names its own node
+    # with `route_latest_named`. Answers the router's output edge.
+    def route_latest(documents, traffic, &key) = route_latest_named(documents, traffic, nil, &key)
+
+    def route_latest_named(documents, traffic, node_name, &key)
+      merge
+        .from(documents) { |document| { document: document } }
+        .from(traffic) { |item| { traffic: key.call(item).to_s } }
+        .into(RouteLatest, node_name)
+    end
+
+    # The node for a tasklet. Two unnamed mentions of one tasklet are one
+    # node — the naming convention every SDK mirrors — so a diamond that fans
+    # out and back in has one instance, not two.
+    def node_for(klass, name = nil)
+      definition = klass.definition
+      identity = (name || definition.name).to_s
+      existing = @tasklets.find { |node| node.name == identity && node.klass == klass }
+      return existing if existing
+
+      if @tasklets.any? { |node| node.name == identity }
+        # The node table is keyed by identity and physically cannot hold two,
+        # so the record the core would judge is unwritable. The refusal answers
+        # under the core's own name all the same.
+        raise Error.new("topology.node_identity_collision", name: identity)
+      end
+
+      node = TaskNode.new(@tasklets.size, identity, klass, definition, nil)
+      @tasklets << node
+      node
+    end
+
+    # The channel this node produces on, created on first use.
+    #
+    # Lazily, so a leaf node never carries a phantom channel. Named by the
+    # node's identity until a `named(...)` override, and reused by every later
+    # call on an edge whose source is this node — which is what makes a fan-out
+    # one channel with two consumers rather than two channels claiming one
+    # producer.
+    def output_channel_for(node)
+      return node.output if node.output
+
+      channel = ChannelSpec.new(
+        index: @channels.size, name: node.name, kind: :inter_task,
+        producer: node.index, consumers: [], chains: [], chain_edges: [],
+        overflow: :drop_nothing, keyed: false
+      )
+      @channels << channel
+      node.output = channel
+      channel
+    end
+
+    # Record one more link on the sequence at `base`, or open one on `source`,
+    # and answer the new sequence's id.
+    #
+    # The links are copied rather than appended in place: a handle held over a
+    # branch point extends the prefix twice, and two sequences sharing one
+    # array would each see the other's links.
+    def link_chain(source, base, link, edge: nil)
+      links = base.nil? ? [] : @chains[base].links
+      @chains[base].superseded = true unless base.nil?
+      @chains << Chain.new(source: source, links: links + [link], superseded: false, edge: edge)
+      @chains.size - 1
+    end
+
+    # File every sequence on the channel it was declared off, in declaration
+    # order — the `chains` the record carries.
+    #
+    # A channel named only by a sequence no terminator closed is created here,
+    # so a dangling chain is submitted rather than dropped on the way to the
+    # core. Idempotent: a spec is assembled once, but a builder is reachable.
+    def seal
+      return self if @sealed
+
+      @sealed = true
+      @chains.each do |chain|
+        next if chain.superseded
+
+        channel = channel_of(chain.source)
+        channel.chains << chain.links
+        channel.chain_edges << chain.edge
+      end
+      self
+    end
+
+    # The channel an edge's values leave on: the entry's own, or the producing
+    # node's, created on first use.
+    def channel_of(source)
+      source[0] == :entry ? source[1] : output_channel_for(source[1])
+    end
+  end
+
+  # One edge under construction. Immutable: every fluent call answers a new
+  # edge, so a shared prefix can be branched without the arms colliding.
+  class Stream
+    # The two links a fluent call adds. Frozen literals rather than a hash per
+    # call: every `map` on every channel records the same one.
+    TRANSFORM = { "link" => "transform" }.freeze
+    KEY = { "link" => "key" }.freeze
+
+    def initialize(builder, from, pending)
+      @builder = builder
+      @from = from
+      @pending = pending
+    end
+
+    # Override the channel name this edge creates. The default is derived from
+    # the producing node's identity.
+    def named(name) = with(channel_name: name.to_s)
+
+    # Transform every value on this edge. Evaluated producer-side.
+    def map(&transform) = flat_map { |value| [transform.call(value)] }
+
+    # Drop the values a predicate rejects, before this edge's key is computed.
+    # A dropped value is consumed and reaches no handler.
+    def filter(&predicate) = flat_map { |value| predicate.call(value) ? [value] : [] }
+
+    # Turn every value on this edge into the values the block answers, each
+    # keyed and delivered on its own. An empty answer drops the value. Every
+    # transform link is one of these: `map` and `filter` are its one-value and
+    # one-or-none cases.
+    def flat_map(&fan)
+      relink(TRANSFORM, transforms: transforms + [->(value) { fan.call(value).to_a }])
+    end
+
+    # Route by a computed key: the consumer becomes a dynamic keyed tasklet.
+    def key_by(&key) = relink(KEY, key_by: key)
+
+    def on_full(policy) = with(overflow: policy.to_sym)
+    def retry_max(attempts) = with(retry_max: attempts)
+    def retry_backoff_ms(milliseconds) = with(retry_backoff_ms: milliseconds)
+
+    # Stamp `flz.cell` / `flz.seq` / `flz.key` onto this channel's plugin frames.
+    def keyed = with(keyed: true)
+
+    # Fan out: each arm receives this same edge, so `branch` builds exactly the
+    # topology the separate `into` statements would.
+    #
+    # The arity is recorded as a sequence of its own, so the arms extend the
+    # prefix this edge already carries rather than the branch's own record.
+    def branch(*arms)
+      @builder.link_chain(@from, nil, { "link" => "branch", "arms" => arms.size })
+      arms.each { |arm| arm.call(self) }
+      self
+    end
+
+    # Wire this edge into a tasklet.
+    def into(klass, node_name = nil)
+      node = @builder.node_for(klass, node_name)
+      close_on(node, { "link" => "into", "tasklet" => node.index })
+    end
+
+    # Close this edge's chain on a resolved node with `link`, and answer the
+    # node's output edge. The one body behind `into` and every merge source.
+    def close_on(node, link)
+      # Creates this edge's channel and applies its pending options, before the
+      # link that closes on it is recorded.
+      channel
+      # The transforms and the key belong to this edge, not to the channel: one
+      # channel may feed several tasklets, and a chain declared before one
+      # `into` must not reach another. They ride the chain that closes on them,
+      # and the core names that chain back on the edge it derived — so whether
+      # a re-declaration is the legal fan-in no-op or `EdgeAlreadyWired` is
+      # answered there and nowhere here.
+      @builder.link_chain(
+        @from, @pending[:chain], link,
+        edge: ConsumerEdge.new(
+          tasklet: node.index, transforms: transforms, key_by: @pending[:key_by]
+        )
+      )
+      Stream.new(@builder, [:tasklet, node], {})
+    end
+    # `Merge` reaches it through `send`, as a friend of the builder.
+    private :close_on
+
+    # Terminate this edge's channel on the plugin layer, and answer the edge.
+    # A name written here IS the channel's name, everywhere one appears;
+    # `output` with none leaves it derived as it would be without an output at all.
+    #
+    # `output(...).into(...)` wires a consumer behind the output, so one emit
+    # reaches both. A channel with no consumer behind it is the plugin-bound
+    # terminal — terminality is the absence of further wiring, not a different
+    # method.
+    # The one refusal left here: a channel with no producer has no value to
+    # serialize, and the core reads an output as the terminal of the channel it
+    # sits on — it cannot see that this one is fed by the controller.
+    #
+    # `encode` turns a value into the plugin frame's bytes, and `codec` names a
+    # format instead. With neither, a String is sent as its bytes and any other
+    # value as JSON, so an output needs neither for a string, bytes or a JSON value.
+    def output(name = nil, encode: nil, codec: nil) = attach_sink(name&.to_s, encode, codec)
+
+    # End this edge's chain in a call to another relay, and answer the edge
+    # of its Response channel: the entry channel `<channel>.response`, whose
+    # values are `Flezha::Response`s. Every value reaching the call, after this
+    # chain's transforms and key, is one request. `target` names an instance,
+    # or a partition group whose member the key chooses; `factory:` instead
+    # names a factory whose instance the key names, started when absent unless
+    # `create: false`. `channel:` names the callee's entry channel, and
+    # `answer_channel:` the callee's channel whose output answers (its first
+    # output for the request when unset). `codec` names the format of both the
+    # request and the answer, JSON by default. A call is not an edge, so the
+    # Response channel may feed this tasklet. A channel ends in one output or one
+    # call.
+    def call(target = nil, factory: nil, deadline_ms:, create: true, channel: nil,
+             answer_channel: nil, codec: nil)
+      raise Error.new("topology.output_on_entry", channel: @from[1].name) unless @from[0] == :tasklet
+
+      source = self.channel
+      named = @builder.channel_codec(source, codec || :json, nil)
+      link = call_link(target, factory, deadline_ms, create, channel, answer_channel, source)
+      @builder.link_chain(@from, @pending[:chain], link)
+      # First call wins, as for an output; a second end is the core's to refuse.
+      source.call ||= CallEnd.new(transforms: transforms, key_by: @pending[:key_by],
+                                  encode: Codec.channel_encoder(named, source))
+      decode = Codec.channel_decoder(named, source)
+      @builder.input("#{source.name}.response", decode: ->(bytes) { Response.decode(bytes, decode) })
+    end
+
+    protected
+
+    # Attach an output, which leaves as a plugin output; the core refuses a second
+    # end on a channel.
+    def attach_sink(name, encode, codec)
+      unless @from[0] == :tasklet
+        raise Error.new("topology.output_on_entry", channel: name || "<derived>")
+      end
+
+      target = channel
+      # The name written here IS the channel's name; `output` with none leaves it
+      # derived as it would be without an output at all.
+      target.name = name unless name.nil?
+      named = @builder.channel_codec(target, codec, encode)
+      link = { "link" => "sink" }
+      link["name"] = name unless name.nil?
+      @builder.link_chain(@from, @pending[:chain], link)
+      # First call wins on the builder's own side, as the reference does; a
+      # second end is what the core refuses the class for.
+      if target.sink.nil?
+        target.sink = target.name
+        target.sink_encode = encode || Codec.channel_encoder(named, target)
+        target.codec = @builder.api_codec(named, encode)
+      end
+      self
+    end
+
+    # The channel this edge's values leave on, with the pending options
+    # applied. An entry edge already has one; a tasklet edge's is created on
+    # first use and reused after.
+    def channel
+      if @from[0] == :entry
+        target = @from[1]
+      else
+        target = @builder.output_channel_for(@from[1])
+        rename = @pending[:channel_name]
+        # Recorded where it is APPLIED, so the core reads the overrides in the
+        # order this builder took them and the two cannot disagree about which
+        # name a channel ends up with.
+        if rename && target.name != rename
+          target.name = rename
+          @builder.link_chain(@from, nil, { "link" => "named", "name" => rename })
+        end
+      end
+      apply_pending(target)
+      target
+    end
+
+    private
+
+    def transforms = @pending[:transforms] || []
+
+    # The `call` link as the record carries it; which of `target` and `factory`
+    # is named, and whether exactly one is, is the core's to judge.
+    def call_link(target, factory, deadline_ms, create, channel, answer_channel, source)
+      link = { "link" => "call" }
+      link["target"] = target.to_s unless target.nil?
+      link["factory"] = factory.to_s unless factory.nil?
+      link.merge!("deadline_ms" => deadline_ms, "create" => create)
+      link["channel"] = channel.to_s unless channel.nil?
+      link["answer_channel"] = answer_channel.to_s unless answer_channel.nil?
+      link.merge("response" => "#{source.name}.response")
+    end
+
+    # Applied now, as the reference applies it: an option no terminator follows
+    # would otherwise be dropped. This call is why `channel` is protected.
+    def with(**next_pending)
+      edge = Stream.new(@builder, @from, @pending.merge(next_pending))
+      edge.channel
+      edge
+    end
+
+    # This edge with one more chain link — the one place a transform's or a
+    # key's result edge is built, so every link reaches the record and a chain
+    # no `into(...)` closes is submitted rather than silently discarded.
+    def relink(link, **next_pending)
+      chain = @builder.link_chain(@from, @pending[:chain], link)
+      Stream.new(@builder, @from, @pending.merge(next_pending, { chain: chain }))
+    end
+
+    def apply_pending(channel)
+      channel.overflow = @pending[:overflow] if @pending[:overflow]
+      channel.retry_max = @pending[:retry_max] unless @pending[:retry_max].nil?
+      channel.retry_backoff_ms = @pending[:retry_backoff_ms] unless @pending[:retry_backoff_ms].nil?
+      channel.keyed = true if @pending[:keyed]
+    end
+  end
+
+  # Several sources into one tasklet node, each lifted into the merged value
+  # by its own mapping. Recorded, never judged: a merge of one source, or of
+  # one source twice, is the core's to refuse.
+  class Merge
+    def initialize(builder)
+      @builder = builder
+      @sources = []
+      @keys = []
+    end
+
+    # One source and the block that maps its values into the merged value. The
+    # mapping is that source's own edge transform, after any the handle
+    # already carries, so it runs on the edge the core derives for the source.
+    def from(source, &mapping)
+      @sources << source.map(&mapping)
+      self
+    end
+
+    # Route the merged value by a computed key, evaluated on the mapped value.
+    def key_by(&key)
+      @keys << key
+      self
+    end
+
+    # Close every source on one node under one merge group, and answer the
+    # node's output edge.
+    def into(klass, node_name = nil)
+      node = @builder.node_for(klass, node_name)
+      link = { "link" => "merge", "tasklet" => node.index, "group" => @builder.next_merge_group }
+      # The key rides each source's chain after its mapping; a second `key_by`
+      # is one more link, which the core refuses as it would on one edge.
+      @sources.each do |source|
+        @keys.reduce(source) { |edge, key| edge.key_by(&key) }.send(:close_on, node, link)
+      end
+      Stream.new(@builder, [:tasklet, node], {})
+    end
+  end
+end