igniter 0.2.0 → 0.3.1

data/lib/igniter/model/output_node.rb

@@ -5,16 +5,36 @@ module Igniter
     class OutputNode < Node
       attr_reader :source
 
-      def initialize(id:, name:, source:, metadata: {})
+      def initialize(id:, name:, source:, path: nil, metadata: {})
+        normalized_source = source.to_s
+
         super(
           id: id,
           kind: :output,
           name: name,
-          path: "output.#{name}",
-          dependencies: [source],
+          path: (path || "output.#{name}"),
+          dependencies: [normalized_source.split(".").first],
           metadata: metadata
         )
-        @source = source.to_sym
+        @source = normalized_source.include?(".") ? normalized_source : normalized_source.to_sym
+      end
+
+      def source_root
+        source.to_s.split(".").first.to_sym
+      end
+
+      def composition_output?
+        source.to_s.include?(".")
+      end
+
+      def type
+        metadata[:type]
+      end
+
+      def child_output_name
+        return unless composition_output?
+
+        source.to_s.split(".", 2).last.to_sym
       end
     end
   end

data/lib/igniter/model.rb

@@ -5,6 +5,8 @@ require_relative "model/graph"
 require_relative "model/input_node"
 require_relative "model/compute_node"
 require_relative "model/composition_node"
+require_relative "model/branch_node"
+require_relative "model/collection_node"
 require_relative "model/output_node"
 
 module Igniter

data/lib/igniter/runtime/cache.rb

@@ -5,47 +5,86 @@ module Igniter
     class Cache
       def initialize
         @states = {}
+        @mutex = Mutex.new
+        @condition = ConditionVariable.new
       end
 
       def fetch(node_name)
-        @states[node_name.to_sym]
+        @mutex.synchronize { @states[node_name.to_sym] }
       end
 
       def write(state)
-        current = fetch(state.node.name)
-        version = state.version || next_version(current)
-        @states[state.node.name] = NodeState.new(
-          node: state.node,
-          status: state.status,
-          value: state.value,
-          error: state.error,
-          version: version,
-          resolved_at: state.resolved_at,
-          invalidated_by: state.invalidated_by
-        )
+        @mutex.synchronize do
+          current = @states[state.node.name]
+          version = state.version || (current&.running? ? current.version : next_version(current))
+          @states[state.node.name] = NodeState.new(
+            node: state.node,
+            status: state.status,
+            value: state.value,
+            error: state.error,
+            version: version,
+            resolved_at: state.resolved_at,
+            invalidated_by: state.invalidated_by
+          )
+          @condition.broadcast
+        end
+      end
+
+      def begin_resolution(node)
+        @mutex.synchronize do
+          loop do
+            current = @states[node.name]
+            return [:cached, current] if current && !current.stale? && !current.running?
+
+            unless current&.running?
+              @states[node.name] = NodeState.new(
+                node: node,
+                status: :running,
+                value: current&.value,
+                error: current&.error,
+                version: next_version(current),
+                resolved_at: current&.resolved_at || Time.now.utc,
+                invalidated_by: nil
+              )
+              return [:started, @states[node.name]]
+            end
+
+            @condition.wait(@mutex)
+          end
+        end
       end
 
       def stale!(node, invalidated_by:)
-        current = fetch(node.name)
-        return unless current
+        @mutex.synchronize do
+          current = @states[node.name]
+          return unless current
 
-        @states[node.name] = NodeState.new(
-          node: node,
-          status: :stale,
-          value: current.value,
-          error: current.error,
-          version: current.version + 1,
-          resolved_at: current.resolved_at,
-          invalidated_by: invalidated_by
-        )
+          @states[node.name] = NodeState.new(
+            node: node,
+            status: :stale,
+            value: current.value,
+            error: current.error,
+            version: current.version + 1,
+            resolved_at: current.resolved_at,
+            invalidated_by: invalidated_by
+          )
+          @condition.broadcast
+        end
       end
 
       def values
-        @states.values
+        @mutex.synchronize { @states.values }
       end
 
       def to_h
-        @states.dup
+        @mutex.synchronize { @states.dup }
+      end
+
+      def restore!(states)
+        @mutex.synchronize do
+          @states = states.transform_keys(&:to_sym)
+          @condition.broadcast
+        end
       end
 
       private

data/lib/igniter/runtime/collection_result.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+module Igniter
+  module Runtime
+    class CollectionResult
+      Item = Struct.new(:key, :status, :result, :error, keyword_init: true) do
+        def succeeded?
+          status == :succeeded
+        end
+
+        def failed?
+          status == :failed
+        end
+
+        def to_h
+          {
+            key: key,
+            status: status,
+            result: serialize_result(result),
+            error: serialize_error(error)
+          }.compact
+        end
+
+        private
+
+        def serialize_result(value)
+          case value
+          when Runtime::Result
+            value.to_h
+          else
+            value
+          end
+        end
+
+        def serialize_error(value)
+          return nil unless value
+
+          {
+            type: value.class.name,
+            message: value.message,
+            context: value.respond_to?(:context) ? value.context : {}
+          }
+        end
+      end
+
+      attr_reader :items, :mode
+
+      def initialize(items:, mode:)
+        @items = items.freeze
+        @mode = mode.to_sym
+      end
+
+      def [](key)
+        items.fetch(key)
+      end
+
+      def keys
+        items.keys
+      end
+
+      def successes
+        items.select { |_key, item| item.succeeded? }
+      end
+
+      def failures
+        items.select { |_key, item| item.failed? }
+      end
+
+      def items_summary
+        items.transform_values do |item|
+          {
+            status: item.status,
+            error: item.error&.message
+          }.compact
+        end
+      end
+
+      def failed_items
+        failures.transform_values do |item|
+          {
+            type: item.error.class.name,
+            message: item.error.message,
+            context: item.error.respond_to?(:context) ? item.error.context : {}
+          }
+        end
+      end
+
+      def to_h
+        items.transform_values(&:to_h)
+      end
+
+      def summary
+        {
+          mode: mode,
+          total: items.size,
+          succeeded: successes.size,
+          failed: failures.size,
+          status: failures.empty? ? :succeeded : :partial_failure
+        }
+      end
+
+      def as_json(*)
+        {
+          mode: mode,
+          summary: summary,
+          items: to_h
+        }
+      end
+    end
+  end
+end

data/lib/igniter/runtime/deferred_result.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require "securerandom"
+
+module Igniter
+  module Runtime
+    class DeferredResult
+      attr_reader :token, :payload, :source_node, :waiting_on
+
+      def initialize(token:, payload: {}, source_node: nil, waiting_on: nil)
+        @token = token
+        @payload = payload.freeze
+        @source_node = source_node&.to_sym
+        @waiting_on = waiting_on&.to_sym
+      end
+
+      def self.build(token: nil, payload: {}, source_node: nil, waiting_on: nil)
+        new(
+          token: token || SecureRandom.uuid,
+          payload: payload,
+          source_node: source_node,
+          waiting_on: waiting_on
+        )
+      end
+
+      def to_h
+        {
+          token: token,
+          payload: payload,
+          source_node: source_node,
+          waiting_on: waiting_on
+        }.compact
+      end
+
+      def as_json(*)
+        to_h
+      end
+    end
+  end
+end

data/lib/igniter/runtime/execution.rb

@@ -3,11 +3,14 @@
 module Igniter
   module Runtime
     class Execution
-      attr_reader :compiled_graph, :contract_instance, :inputs, :cache, :events, :audit
+      attr_reader :compiled_graph, :contract_instance, :inputs, :cache, :events, :audit, :runner_strategy, :max_workers, :store
 
-      def initialize(compiled_graph:, contract_instance:, inputs:)
+      def initialize(compiled_graph:, contract_instance:, inputs:, runner: :inline, max_workers: nil, store: nil)
         @compiled_graph = compiled_graph
         @contract_instance = contract_instance
+        @runner_strategy = runner
+        @max_workers = max_workers
+        @store = store
         @input_validator = InputValidator.new(compiled_graph)
         @inputs = @input_validator.normalize_initial_inputs(inputs)
         @cache = Cache.new
@@ -15,16 +18,16 @@ module Igniter
         @audit = Extensions::Auditing::Timeline.new(self)
         @events.subscribe(@audit)
         @resolver = Resolver.new(self)
+        @planner = Planner.new(self)
+        @runner = RunnerFactory.build(@runner_strategy, self, resolver: @resolver, max_workers: @max_workers, store: @store)
         @invalidator = Invalidator.new(self)
       end
 
       def resolve_output(name)
         output = compiled_graph.fetch_output(name)
-        with_execution_lifecycle([output.source]) do
-          state = @resolver.resolve(output.source)
-          raise state.error if state.failed?
-
-          state.value
+        with_execution_lifecycle([output.source_root]) do
+          run_targets([output.source_root])
+          resolve_exported_output(output)
         end
       end
 
@@ -33,10 +36,11 @@ module Igniter
       end
 
       def resolve_all
-        output_sources = compiled_graph.outputs.map(&:source)
+        output_sources = @planner.targets_for_outputs
 
         with_execution_lifecycle(output_sources) do
-          compiled_graph.outputs.each { |output_node| resolve(output_node.source) }
+          run_targets(output_sources)
+          compiled_graph.outputs.each { |output_node| resolve_output_value(output_node) }
           self
         end
       end
@@ -55,9 +59,28 @@ module Igniter
         self
       end
 
+      def resume(node_name, value:)
+        node = compiled_graph.fetch_node(node_name)
+        current = cache.fetch(node.name)
+        raise ResolutionError, "Node '#{node_name}' is not pending" unless current&.pending?
+
+        cache.write(NodeState.new(node: node, status: :succeeded, value: value))
+        @events.emit(:node_resumed, node: node, status: :succeeded, payload: { resumed: true })
+        @invalidator.invalidate_from(node.name)
+        persist_runtime_state!
+        self
+      end
+
+      def resume_by_token(token, value:)
+        node_name = pending_node_name_for_token(token)
+        raise ResolutionError, "No pending node found for token '#{token}'" unless node_name
+
+        resume(node_name, value: value)
+      end
+
       def success?
         resolve_all
-        !cache.values.any?(&:failed?)
+        !failed? && !pending?
       end
 
       def failed?
@@ -65,6 +88,11 @@ module Igniter
         cache.values.any?(&:failed?)
       end
 
+      def pending?
+        resolve_all
+        cache.values.any?(&:pending?)
+      end
+
       def states
         Extensions::Introspection::RuntimeFormatter.states(self)
       end
@@ -77,13 +105,25 @@ module Igniter
         Diagnostics::Report.new(self)
       end
 
+      def plan(output_names = nil)
+        @planner.plan(output_names)
+      end
+
+      def explain_plan(output_names = nil)
+        Extensions::Introspection::PlanFormatter.to_text(self, output_names)
+      end
+
       def to_h
         {
           graph: compiled_graph.name,
           execution_id: events.execution_id,
           inputs: inputs.dup,
-          success: !cache.values.any?(&:failed?),
+          runner: runner_strategy,
+          max_workers: max_workers,
+          success: success?,
           failed: cache.values.any?(&:failed?),
+          pending: cache.values.any?(&:pending?),
+          plan: plan,
           states: states,
           event_count: events.events.size
         }
@@ -95,6 +135,28 @@ module Igniter
         )
       end
 
+      def snapshot(include_resolution: true)
+        resolve_pending_safe if include_resolution
+
+        {
+          graph: compiled_graph.name,
+          execution_id: events.execution_id,
+          runner: runner_strategy,
+          max_workers: max_workers,
+          inputs: inputs.dup,
+          states: serialize_states,
+          events: events.events.map(&:as_json)
+        }
+      end
+
+      def restore!(snapshot)
+        @inputs.replace(symbolize_keys(value_from(snapshot, :inputs) || {}))
+        cache.restore!(deserialize_states(value_from(snapshot, :states) || {}))
+        events.restore!(events: value_from(snapshot, :events) || [], execution_id: value_from(snapshot, :execution_id))
+        audit.restore!(events.events)
+        self
+      end
+
       private
 
       def with_execution_lifecycle(node_names)
@@ -102,6 +164,7 @@ module Igniter
           @events.emit(:execution_started, payload: { graph: compiled_graph.name, targets: node_names.map(&:to_sym) })
           begin
             result = yield
+            persist_runtime_state!
             @events.emit(:execution_finished, payload: { graph: compiled_graph.name, targets: node_names.map(&:to_sym) })
             result
           rescue StandardError => e
@@ -114,6 +177,7 @@ module Igniter
                 error: e.message
               }
             )
+            persist_runtime_state!
             raise
           end
         else
@@ -137,6 +201,192 @@ module Igniter
       def fetch_input!(name)
         @input_validator.fetch_value!(name, @inputs)
       end
+
+      private
+
+      def resolve_exported_output(output)
+        state = @resolver.resolve(output.source_root)
+        raise state.error if state.failed?
+        return state.value if state.pending?
+
+        return state.value unless output.composition_output?
+
+        state.value.public_send(output.child_output_name)
+      end
+
+      def run_targets(node_names)
+        @runner.run(node_names)
+      end
+
+      def persist_runtime_state!
+        return unless @runner.respond_to?(:persist!)
+
+        @runner.persist!
+      end
+
+      def pending_node_name_for_token(token)
+        source_match = cache.values.find do |state|
+          state.pending? &&
+            state.value.is_a?(Runtime::DeferredResult) &&
+            state.value.token == token &&
+            state.value.source_node == state.node.name
+        end
+        return source_match.node.name if source_match
+
+        cache.values.find do |state|
+          state.pending? &&
+            state.value.is_a?(Runtime::DeferredResult) &&
+            state.value.token == token
+        end&.node&.name
+      end
+
+      def resolve_pending_safe
+        resolve_all
+      rescue Igniter::Error
+        nil
+      end
+
+      def serialize_states
+        cache.to_h.each_with_object({}) do |(node_name, state), memo|
+          memo[node_name] = {
+            status: state.status,
+            version: state.version,
+            resolved_at: state.resolved_at&.iso8601,
+            invalidated_by: state.invalidated_by,
+            value: serialize_state_value(state.value),
+            error: serialize_state_error(state.error)
+          }
+        end
+      end
+
+      def deserialize_states(snapshot_states)
+        snapshot_states.each_with_object({}) do |(node_name, state_data), memo|
+          node = compiled_graph.fetch_node(node_name)
+          memo[node.name] = NodeState.new(
+            node: node,
+            status: (state_data[:status] || state_data["status"]).to_sym,
+            value: deserialize_state_value(node, state_data[:value] || state_data["value"]),
+            error: deserialize_state_error(state_data[:error] || state_data["error"]),
+            version: state_data[:version] || state_data["version"],
+            resolved_at: deserialize_time(state_data[:resolved_at] || state_data["resolved_at"]),
+            invalidated_by: (state_data[:invalidated_by] || state_data["invalidated_by"])&.to_sym
+          )
+        end
+      end
+
+      def serialize_state_value(value)
+        case value
+        when Runtime::DeferredResult
+          { type: :deferred, data: value.as_json }
+        when Runtime::Result
+          {
+            type: :result_snapshot,
+            snapshot: value.execution.snapshot(include_resolution: false)
+          }
+        when Runtime::CollectionResult
+          {
+            type: :collection_result,
+            mode: value.mode,
+            items: value.items.transform_values do |item|
+              {
+                key: item.key,
+                status: item.status,
+                result: serialize_state_value(item.result),
+                error: serialize_state_error(item.error)
+              }
+            end
+          }
+        else
+          value
+        end
+      end
+
+      def deserialize_state_value(node, value)
+        if value.is_a?(Hash) && (value[:type] || value["type"])&.to_sym == :deferred
+          data = value[:data] || value["data"] || {}
+          return Runtime::DeferredResult.build(
+            token: data[:token] || data["token"],
+            payload: data[:payload] || data["payload"] || {},
+            source_node: data[:source_node] || data["source_node"],
+            waiting_on: data[:waiting_on] || data["waiting_on"]
+          )
+        end
+
+        if value.is_a?(Hash) && (value[:type] || value["type"])&.to_sym == :result_snapshot
+          snapshot = value[:snapshot] || value["snapshot"] || {}
+          if node.kind == :composition
+            child_contract = node.contract_class.restore(snapshot)
+            return child_contract.result
+          end
+
+          if node.kind == :branch
+            snapshot_graph = snapshot[:graph] || snapshot["graph"]
+            contract_class = node.possible_contracts.find { |candidate| candidate.compiled_graph.name == snapshot_graph }
+            return value unless contract_class
+
+            child_contract = contract_class.restore(snapshot)
+            return child_contract.result
+          end
+
+          if node.kind == :collection
+            child_contract = node.contract_class.restore(snapshot)
+            return child_contract.result
+          end
+        end
+
+        if value.is_a?(Hash) && (value[:type] || value["type"])&.to_sym == :collection_result
+          items = (value[:items] || value["items"] || {}).each_with_object({}) do |(key, item), memo|
+            memo[key.is_a?(String) && key.match?(/\A\d+\z/) ? key.to_i : key] = Runtime::CollectionResult::Item.new(
+              key: item[:key] || item["key"] || key,
+              status: (item[:status] || item["status"]).to_sym,
+              result: deserialize_state_value(node, item[:result] || item["result"]),
+              error: deserialize_state_error(item[:error] || item["error"])
+            )
+          end
+          return Runtime::CollectionResult.new(
+            items: items,
+            mode: (value[:mode] || value["mode"] || :collect).to_sym
+          )
+        end
+
+        value
+      end
+
+      def serialize_state_error(error)
+        return nil unless error
+
+        {
+          type: error.class.name,
+          message: error.message,
+          context: error.respond_to?(:context) ? error.context : {}
+        }
+      end
+
+      def deserialize_state_error(error_data)
+        return nil unless error_data
+
+        ResolutionError.new(
+          error_data[:message] || error_data["message"],
+          context: error_data[:context] || error_data["context"] || {}
+        )
+      end
+
+      def deserialize_time(value)
+        case value
+        when Time
+          value
+        when String
+          Time.iso8601(value)
+        else
+          value || Time.now.utc
+        end
+      end
+
+      def value_from(data, key)
+        data[key] || data[key.to_s]
+      end
+
+      alias_method :resolve_output_value, :resolve_exported_output
     end
   end
 end