async-graph 0.1.1 → 0.1.2

data/lib/async-graph/graph.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative "graph_validation"
+
 module AsyncGraph
   FINISH = :__finish__
 
@@ -7,21 +9,8 @@ module AsyncGraph
   Edge = Struct.new(:to, :branch, keyword_init: true)
   AwaitSignal = Struct.new(:requests, keyword_init: true)
 
-  def self.symbolize(object)
-    case object
-    when Array then object.map { |value| symbolize(value) }
-    when Hash then object.each_with_object({}) { |(key, value), memo| memo[key.to_sym] = symbolize(value) }
-    else object
-    end
-  end
-
-  def self.stringify(object)
-    case object
-    when Array then object.map { |value| stringify(value) }
-    when Hash then object.each_with_object({}) { |(key, value), memo| memo[key.to_s] = stringify(value) }
-    else object
-    end
-  end
+  class ValidationError < StandardError; end
+  class JoinConflictError < StandardError; end
 
   class Command < Struct.new(:update, :goto, keyword_init: true)
     def self.goto(node) = new(goto: node)
@@ -44,6 +33,16 @@ module AsyncGraph
     def finished? = true
   end
 
+  class JoinParked < Struct.new(:joins, keyword_init: true)
+    def parked? = true
+    def released? = false
+  end
+
+  class JoinReleased < Struct.new(:token, :joins, keyword_init: true)
+    def parked? = false
+    def released? = true
+  end
+
   # Interesting options for issuing multiple external jobs from one logical step:
   # 1. Model fan-out explicitly in the graph as a diamond.
   # 2. Use await.all(...) inside one node to queue a batch and suspend once.
@@ -80,46 +79,97 @@ module AsyncGraph
   end
 
   class Graph
+    include GraphValidation
+
     attr_reader :entry
 
     def initialize(&block)
       @nodes = {}
       @edges = Hash.new { |hash, key| hash[key] = [] }
       @join_expects = {}
+      @validated = false
       instance_eval(&block) if block
     end
 
-    def node(name, &block) = @nodes[name.to_sym] = block
+    def node(name, &block)
+      name = name.to_sym
+      raise ValidationError, "Node #{name} requires a block" unless block
+      raise ValidationError, "Node #{name} is already defined" if @nodes.key?(name)
+
+      invalidate_validation!
+      @nodes[name] = block
+    end
 
     def edge(from, to, branch: nil)
       if from.is_a?(Array)
-        @join_expects[to.to_sym] = from.map(&:to_sym)
-        from.each { |item| edge(item, to, branch: item) }
+        sources = from.map(&:to_sym)
+        validate_join_sources!(to, sources)
+
+        target = to.to_sym
+        raise ValidationError, "Join target #{target} is already defined" if @join_expects.key?(target)
+
+        invalidate_validation!
+        @join_expects[target] = sources
+        sources.each { |item| edge(item, target, branch: item) }
       else
-        @edges[from.to_sym] << Edge.new(to: to.to_sym, branch: branch&.to_sym)
+        invalidate_validation!
+        add_edge(from.to_sym, Edge.new(to: to.to_sym, branch: branch&.to_sym))
       end
     end
 
-    def set_entry_point(name) = @entry = name.to_sym
+    def set_entry_point(name)
+      invalidate_validation!
+      @entry = name.to_sym
+    end
+
     def set_finish_point(name) = edge(name, FINISH)
 
     def step(state:, node:, resolved: {})
+      validate!
+
       current = node.to_sym
       return Finished.new(state:) if current == FINISH
 
+      validate_known_node!(current, context: "Step node")
       await = Await.new(resolved)
-      result = catch(:await) { [:ok, call_node(@nodes.fetch(current), state, await)] }
+      result = catch(:await) { [:ok, call_node(@nodes[current], state, await)] }
       return Suspended.new(state:, node: current, requests: result.requests) if result.is_a?(AwaitSignal)
 
       Advanced.new(**advance(current, state, result.last))
     end
 
     def edges_from(node)
-      @edges[node.to_sym].yield_self { |edges| edges.empty? ? [Edge.new(to: FINISH)] : edges }
+      validate!
+
+      current = node.to_sym
+      validate_known_node!(current, context: "Node")
+      @edges[current].yield_self { |edges| edges.empty? ? [Edge.new(to: FINISH)] : edges }
     end
 
-    def join?(node) = @join_expects.key?(node.to_sym)
-    def join_for(node) = @join_expects.fetch(node.to_sym)
+    def join?(node)
+      validate!
+      @join_expects.key?(node.to_sym)
+    end
+
+    def join_for(node)
+      validate!
+      @join_expects.fetch(node.to_sym) { raise ValidationError, "Node #{node} is not a join node" }
+    end
+
+    def process_join(token:, joins:)
+      validate!
+
+      current, expects, fork_uid, source = join_context(token)
+      bucket_key = join_bucket_key(fork_uid, current)
+      current_joins = joins.dup
+      state = token.fetch(:state, {})
+      states = next_join_states(current, source, current_joins[bucket_key], state)
+
+      return park_join(current_joins, bucket_key, current, states) unless join_complete?(states, expects)
+
+      current_joins.delete(bucket_key)
+      release_join(current_joins, current, fork_uid, expects, states)
+    end
 
     private
 
@@ -136,13 +186,98 @@ module AsyncGraph
       when Hash
         { state: state.merge(result), destinations: edges_from(node) }
       when Command
+        destinations =
+          if result.goto
+            target = result.goto.to_sym
+            validate_destination!(target, context: "Goto target")
+            [Edge.new(to: target)]
+          else
+            edges_from(node)
+          end
+
         {
           state: state.merge(result.update || {}),
-          destinations: result.goto ? [Edge.new(to: result.goto.to_sym)] : edges_from(node)
+          destinations: destinations
         }
       else
         { state: state, destinations: edges_from(node) }
       end
     end
+
+    def add_edge(from, edge)
+      @edges[from] << edge unless @edges[from].any? { |existing| existing.to == edge.to && existing.branch == edge.branch }
+    end
+
+    def join_context(token)
+      current = token.fetch(:node)&.to_sym
+      validate_known_node!(current, context: "Join node")
+      raise ValidationError, "Node #{current} is not a join node" unless join?(current)
+
+      fork_uid = token.fetch(:fork_uid)
+      raise ValidationError, "Join token for #{current} is missing fork_uid" if fork_uid.nil? || fork_uid.to_s.empty?
+
+      source = token.fetch(:source_node)&.to_sym
+      raise ValidationError, "Join token for #{current} is missing source_node" unless source
+
+      expects = join_for(current)
+      unless expects.include?(source)
+        raise ValidationError, "Join token for #{current} arrived from unexpected source #{source}"
+      end
+
+      [current, expects, fork_uid, source]
+    end
+
+    def join_bucket_key(fork_uid, node)
+      :"#{fork_uid}:#{node}"
+    end
+
+    def next_join_states(current, source, bucket, state)
+      states = (bucket || {join_node: current, states: {}}).fetch(:states, {}).dup
+      if states.key?(source) && states[source] != state
+        raise JoinConflictError, "Join node #{current} received conflicting state for source #{source}"
+      end
+
+      states[source] = state
+      states
+    end
+
+    def join_complete?(states, expects)
+      (expects - states.keys).empty?
+    end
+
+    def park_join(current_joins, bucket_key, current, states)
+      current_joins[bucket_key] = {
+        join_node: current,
+        states: states
+      }
+      JoinParked.new(joins: current_joins)
+    end
+
+    def release_join(current_joins, current, fork_uid, expects, states)
+      JoinReleased.new(
+        token: {
+          token_uid: "#{fork_uid}.join",
+          node: current,
+          state: merge_join_states(states, expects, current),
+          fork_uid: nil,
+          branch: nil,
+          source_node: nil,
+          awaits: {}
+        },
+        joins: current_joins
+      )
+    end
+
+    def merge_join_states(states, expects, current)
+      expects.each_with_object({}) do |source, memo|
+        state = states.fetch(source)
+        conflicting = (memo.keys & state.keys).reject { |key| memo[key] == state[key] }
+        unless conflicting.empty?
+          raise JoinConflictError, "Join node #{current} has conflicting values for: #{conflicting.join(', ')}"
+        end
+
+        memo.merge!(state)
+      end
+    end
   end
 end

data/lib/async-graph/graph_validation.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module AsyncGraph
+  module GraphValidation
+    def validate!
+      return self if @validated
+
+      errors = validation_errors
+      raise ValidationError, errors.uniq.join("; ") unless errors.empty?
+
+      @validated = true
+      self
+    end
+
+    private
+
+    def validation_errors
+      [].tap do |errors|
+        add_entry_validation_errors(errors)
+        add_edge_validation_errors(errors)
+        add_join_validation_errors(errors)
+      end.uniq
+    end
+
+    def add_entry_validation_errors(errors)
+      errors << "Entry point is not set" unless @entry
+      return unless @entry && !@nodes.key?(@entry)
+
+      errors << "Entry point #{@entry} is not defined"
+    end
+
+    def add_edge_validation_errors(errors)
+      @edges.each do |from, edges|
+        errors << "Edge source #{from} is not defined" unless @nodes.key?(from)
+        add_edge_target_validation_errors(errors, edges)
+      end
+    end
+
+    def add_edge_target_validation_errors(errors, edges)
+      edges.each do |edge|
+        next if edge.to == FINISH || @nodes.key?(edge.to)
+
+        errors << "Edge target #{edge.to} is not defined"
+      end
+    end
+
+    def add_join_validation_errors(errors)
+      @join_expects.each do |target, sources|
+        errors << "Join target #{target} is not defined" unless @nodes.key?(target)
+        sources.each do |source|
+          errors << "Join source #{source} for #{target} is not defined" unless @nodes.key?(source)
+        end
+      end
+    end
+
+    def invalidate_validation!
+      @validated = false
+    end
+
+    def validate_join_sources!(target, sources)
+      target = target.to_sym
+      raise ValidationError, "Join target #{target} requires at least one source" if sources.empty?
+
+      duplicates = sources.group_by(&:itself).filter_map { |source, items| source if items.size > 1 }
+      return if duplicates.empty?
+
+      raise ValidationError, "Join target #{target} has duplicate sources: #{duplicates.join(', ')}"
+    end
+
+    def validate_known_node!(node, context:)
+      raise ValidationError, "#{context} is not set" unless node
+      return if @nodes.key?(node)
+
+      raise ValidationError, "#{context} #{node} is not defined"
+    end
+
+    def validate_destination!(node, context:)
+      return if node == FINISH
+
+      validate_known_node!(node, context:)
+    end
+  end
+end

data/lib/async-graph/runner.rb

@@ -0,0 +1,255 @@
+# frozen_string_literal: true
+
+module AsyncGraph
+  class Runner
+    Run = Struct.new(:status, :tokens, :joins, :result, keyword_init: true) do
+      def running? = status.to_sym == :running
+      def finished? = status.to_sym == :finished
+
+      def to_h
+        {
+          status: status.to_s,
+          tokens: tokens,
+          joins: joins,
+          result: result
+        }
+      end
+    end
+
+    Result = Struct.new(:status, :tokens, :joins, :state, :requests, :request_refs, keyword_init: true) do
+      def token = tokens.first
+      def parked? = status == :parked
+      def released? = status == :released
+      def suspended? = status == :suspended
+      def advanced? = status == :advanced
+      def finished? = status == :finished
+    end
+
+    def initialize(graph)
+      @graph = graph
+    end
+
+    def start(state:, token_uid: "t1")
+      @graph.validate!
+
+      {
+        token_uid: token_uid,
+        node: @graph.entry,
+        state: state,
+        fork_uid: nil,
+        branch: nil,
+        source_node: nil,
+        awaits: {}
+      }
+    end
+
+    def start_run(state:, token_uid: "t1")
+      Run.new(
+        status: :running,
+        tokens: [start(state:, token_uid:)],
+        joins: {},
+        result: nil
+      )
+    end
+
+    def step(token:, joins:, resolved: {}, &)
+      return build_join_result(@graph.process_join(token:, joins:)) if join_token?(token)
+
+      graph_step = @graph.step(
+        state: token.fetch(:state),
+        node: token.fetch(:node),
+        resolved: resolved
+      )
+
+      case graph_step
+      when Suspended
+        build_suspended_result(token, joins, graph_step, &)
+      when Advanced
+        build_advanced_result(token, joins, graph_step)
+      when Finished
+        build_finished_result(joins, graph_step.state)
+      else
+        raise ValidationError, "Unsupported graph step result #{graph_step.class}"
+      end
+    end
+
+    def advance_run(run:, resolved_for: nil, &)
+      current_run = normalize_run(run)
+      return current_run if current_run.finished?
+
+      next_tokens, joins, final_state = advance_tokens(
+        current_run,
+        resolved_for: resolved_for,
+        &
+      )
+      Run.new(
+        status: run_status(final_state, next_tokens, joins),
+        tokens: next_tokens,
+        joins: joins,
+        result: final_state
+      )
+    end
+
+    private
+
+    def normalize_run(run)
+      return run if run.is_a?(Run)
+
+      Run.new(
+        status: run.fetch(:status).to_sym,
+        tokens: run.fetch(:tokens, []),
+        joins: run.fetch(:joins, {}),
+        result: run[:result]
+      )
+    end
+
+    def join_token?(token)
+      token[:source_node] && @graph.join?(token.fetch(:node))
+    end
+
+    def build_join_result(join_result)
+      case join_result
+      when JoinParked
+        Result.new(
+          status: :parked,
+          tokens: [],
+          joins: join_result.joins,
+          state: nil,
+          requests: [],
+          request_refs: {}
+        )
+      when JoinReleased
+        Result.new(
+          status: :released,
+          tokens: [join_result.token],
+          joins: join_result.joins,
+          state: join_result.token.fetch(:state),
+          requests: [],
+          request_refs: {}
+        )
+      else
+        raise ValidationError, "Unsupported join result #{join_result.class}"
+      end
+    end
+
+    def build_suspended_result(token, joins, graph_step, &)
+      awaits = token.fetch(:awaits, {}).dup
+      request_refs = graph_step.requests.to_h do |request|
+        key = request.key.to_sym
+        [key, awaits[key] ||= bind_request(request, &)]
+      end
+
+      Result.new(
+        status: :suspended,
+        tokens: [
+          {
+            token_uid: token.fetch(:token_uid),
+            node: graph_step.node,
+            state: graph_step.state,
+            fork_uid: token[:fork_uid],
+            branch: token[:branch],
+            source_node: token[:source_node],
+            awaits: awaits
+          }
+        ],
+        joins: joins,
+        state: graph_step.state,
+        requests: graph_step.requests,
+        request_refs: request_refs
+      )
+    end
+
+    def build_advanced_result(token, joins, graph_step)
+      destinations = graph_step.destinations
+      if destinations.size > 1
+        fork_uid = fork_uid_for(token.fetch(:token_uid))
+        tokens = destinations.map do |edge|
+          suffix = edge.branch || edge.to
+          {
+            token_uid: "#{token.fetch(:token_uid)}.#{suffix}",
+            node: edge.to,
+            state: graph_step.state,
+            fork_uid: fork_uid,
+            branch: edge.branch,
+            source_node: token.fetch(:node),
+            awaits: {}
+          }
+        end
+
+        return Result.new(
+          status: :advanced,
+          tokens: tokens,
+          joins: joins,
+          state: graph_step.state,
+          requests: [],
+          request_refs: {}
+        )
+      end
+
+      edge = destinations.first
+      return build_finished_result(joins, graph_step.state) if edge.to == FINISH
+
+      Result.new(
+        status: :advanced,
+        tokens: [
+          {
+            token_uid: token.fetch(:token_uid),
+            node: edge.to,
+            state: graph_step.state,
+            fork_uid: token[:fork_uid],
+            branch: token[:branch],
+            source_node: token.fetch(:node),
+            awaits: {}
+          }
+        ],
+        joins: joins,
+        state: graph_step.state,
+        requests: [],
+        request_refs: {}
+      )
+    end
+
+    def build_finished_result(joins, state)
+      Result.new(
+        status: :finished,
+        tokens: [],
+        joins: joins,
+        state: state,
+        requests: [],
+        request_refs: {}
+      )
+    end
+
+    def bind_request(request, &)
+      return yield(request) if block_given?
+
+      raise ArgumentError, "Runner#step requires a block to bind external IDs for new requests"
+    end
+
+    def advance_tokens(run, resolved_for:, &)
+      next_tokens = []
+      joins = run.joins
+      final_state = run.result
+
+      run.tokens.each do |token|
+        outcome = step(
+          token: token,
+          joins: joins,
+          resolved: resolved_for&.call(token) || {},
+          &
+        )
+        joins = outcome.joins
+        next_tokens.concat(outcome.tokens)
+        final_state = outcome.state if outcome.finished?
+      end
+
+      [next_tokens, joins, final_state]
+    end
+
+    def run_status(final_state, next_tokens, joins)
+      final_state && next_tokens.empty? && joins.empty? ? :finished : :running
+    end
+
+    def fork_uid_for(token_uid) = "fork-#{token_uid}"
+  end
+end

data/lib/async-graph/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module AsyncGraph
-  VERSION = "0.1.1"
+  VERSION = "0.1.2"
 end

data/lib/async-graph.rb

@@ -2,3 +2,4 @@
 
 require_relative "async-graph/version"
 require_relative "async-graph/graph"
+require_relative "async-graph/runner"