igniter 0.2.0 → 0.3.0

data/lib/igniter/runtime/execution.rb

@@ -3,11 +3,14 @@
 module Igniter
   module Runtime
     class Execution
-      attr_reader :compiled_graph, :contract_instance, :inputs, :cache, :events, :audit
+      attr_reader :compiled_graph, :contract_instance, :inputs, :cache, :events, :audit, :runner_strategy, :max_workers, :store
 
-      def initialize(compiled_graph:, contract_instance:, inputs:)
+      def initialize(compiled_graph:, contract_instance:, inputs:, runner: :inline, max_workers: nil, store: nil)
         @compiled_graph = compiled_graph
         @contract_instance = contract_instance
+        @runner_strategy = runner
+        @max_workers = max_workers
+        @store = store
         @input_validator = InputValidator.new(compiled_graph)
         @inputs = @input_validator.normalize_initial_inputs(inputs)
         @cache = Cache.new
@@ -15,16 +18,16 @@ module Igniter
         @audit = Extensions::Auditing::Timeline.new(self)
         @events.subscribe(@audit)
         @resolver = Resolver.new(self)
+        @planner = Planner.new(self)
+        @runner = RunnerFactory.build(@runner_strategy, self, resolver: @resolver, max_workers: @max_workers, store: @store)
         @invalidator = Invalidator.new(self)
       end
 
       def resolve_output(name)
         output = compiled_graph.fetch_output(name)
-        with_execution_lifecycle([output.source]) do
-          state = @resolver.resolve(output.source)
-          raise state.error if state.failed?
-
-          state.value
+        with_execution_lifecycle([output.source_root]) do
+          run_targets([output.source_root])
+          resolve_exported_output(output)
         end
       end
 
@@ -33,10 +36,11 @@ module Igniter
       end
 
       def resolve_all
-        output_sources = compiled_graph.outputs.map(&:source)
+        output_sources = @planner.targets_for_outputs
 
         with_execution_lifecycle(output_sources) do
-          compiled_graph.outputs.each { |output_node| resolve(output_node.source) }
+          run_targets(output_sources)
+          compiled_graph.outputs.each { |output_node| resolve_output_value(output_node) }
           self
         end
       end
@@ -55,9 +59,28 @@ module Igniter
         self
       end
 
+      def resume(node_name, value:)
+        node = compiled_graph.fetch_node(node_name)
+        current = cache.fetch(node.name)
+        raise ResolutionError, "Node '#{node_name}' is not pending" unless current&.pending?
+
+        cache.write(NodeState.new(node: node, status: :succeeded, value: value))
+        @events.emit(:node_resumed, node: node, status: :succeeded, payload: { resumed: true })
+        @invalidator.invalidate_from(node.name)
+        persist_runtime_state!
+        self
+      end
+
+      def resume_by_token(token, value:)
+        node_name = pending_node_name_for_token(token)
+        raise ResolutionError, "No pending node found for token '#{token}'" unless node_name
+
+        resume(node_name, value: value)
+      end
+
       def success?
         resolve_all
-        !cache.values.any?(&:failed?)
+        !failed? && !pending?
       end
 
       def failed?
@@ -65,6 +88,11 @@ module Igniter
         cache.values.any?(&:failed?)
       end
 
+      def pending?
+        resolve_all
+        cache.values.any?(&:pending?)
+      end
+
       def states
         Extensions::Introspection::RuntimeFormatter.states(self)
       end
@@ -77,13 +105,25 @@ module Igniter
         Diagnostics::Report.new(self)
       end
 
+      def plan(output_names = nil)
+        @planner.plan(output_names)
+      end
+
+      def explain_plan(output_names = nil)
+        Extensions::Introspection::PlanFormatter.to_text(self, output_names)
+      end
+
       def to_h
         {
           graph: compiled_graph.name,
           execution_id: events.execution_id,
           inputs: inputs.dup,
-          success: !cache.values.any?(&:failed?),
+          runner: runner_strategy,
+          max_workers: max_workers,
+          success: success?,
           failed: cache.values.any?(&:failed?),
+          pending: cache.values.any?(&:pending?),
+          plan: plan,
           states: states,
           event_count: events.events.size
         }
@@ -95,6 +135,28 @@ module Igniter
         )
       end
 
+      def snapshot(include_resolution: true)
+        resolve_pending_safe if include_resolution
+
+        {
+          graph: compiled_graph.name,
+          execution_id: events.execution_id,
+          runner: runner_strategy,
+          max_workers: max_workers,
+          inputs: inputs.dup,
+          states: serialize_states,
+          events: events.events.map(&:as_json)
+        }
+      end
+
+      def restore!(snapshot)
+        @inputs.replace(symbolize_keys(value_from(snapshot, :inputs) || {}))
+        cache.restore!(deserialize_states(value_from(snapshot, :states) || {}))
+        events.restore!(events: value_from(snapshot, :events) || [], execution_id: value_from(snapshot, :execution_id))
+        audit.restore!(events.events)
+        self
+      end
+
       private
 
       def with_execution_lifecycle(node_names)
@@ -102,6 +164,7 @@ module Igniter
           @events.emit(:execution_started, payload: { graph: compiled_graph.name, targets: node_names.map(&:to_sym) })
           begin
             result = yield
+            persist_runtime_state!
             @events.emit(:execution_finished, payload: { graph: compiled_graph.name, targets: node_names.map(&:to_sym) })
             result
           rescue StandardError => e
@@ -114,6 +177,7 @@ module Igniter
                 error: e.message
               }
             )
+            persist_runtime_state!
             raise
           end
         else
@@ -137,6 +201,192 @@ module Igniter
       def fetch_input!(name)
         @input_validator.fetch_value!(name, @inputs)
       end
+
+      private
+
+      def resolve_exported_output(output)
+        state = @resolver.resolve(output.source_root)
+        raise state.error if state.failed?
+        return state.value if state.pending?
+
+        return state.value unless output.composition_output?
+
+        state.value.public_send(output.child_output_name)
+      end
+
+      def run_targets(node_names)
+        @runner.run(node_names)
+      end
+
+      def persist_runtime_state!
+        return unless @runner.respond_to?(:persist!)
+
+        @runner.persist!
+      end
+
+      def pending_node_name_for_token(token)
+        source_match = cache.values.find do |state|
+          state.pending? &&
+            state.value.is_a?(Runtime::DeferredResult) &&
+            state.value.token == token &&
+            state.value.source_node == state.node.name
+        end
+        return source_match.node.name if source_match
+
+        cache.values.find do |state|
+          state.pending? &&
+            state.value.is_a?(Runtime::DeferredResult) &&
+            state.value.token == token
+        end&.node&.name
+      end
+
+      def resolve_pending_safe
+        resolve_all
+      rescue Igniter::Error
+        nil
+      end
+
+      def serialize_states
+        cache.to_h.each_with_object({}) do |(node_name, state), memo|
+          memo[node_name] = {
+            status: state.status,
+            version: state.version,
+            resolved_at: state.resolved_at&.iso8601,
+            invalidated_by: state.invalidated_by,
+            value: serialize_state_value(state.value),
+            error: serialize_state_error(state.error)
+          }
+        end
+      end
+
+      def deserialize_states(snapshot_states)
+        snapshot_states.each_with_object({}) do |(node_name, state_data), memo|
+          node = compiled_graph.fetch_node(node_name)
+          memo[node.name] = NodeState.new(
+            node: node,
+            status: (state_data[:status] || state_data["status"]).to_sym,
+            value: deserialize_state_value(node, state_data[:value] || state_data["value"]),
+            error: deserialize_state_error(state_data[:error] || state_data["error"]),
+            version: state_data[:version] || state_data["version"],
+            resolved_at: deserialize_time(state_data[:resolved_at] || state_data["resolved_at"]),
+            invalidated_by: (state_data[:invalidated_by] || state_data["invalidated_by"])&.to_sym
+          )
+        end
+      end
+
+      def serialize_state_value(value)
+        case value
+        when Runtime::DeferredResult
+          { type: :deferred, data: value.as_json }
+        when Runtime::Result
+          {
+            type: :result_snapshot,
+            snapshot: value.execution.snapshot(include_resolution: false)
+          }
+        when Runtime::CollectionResult
+          {
+            type: :collection_result,
+            mode: value.mode,
+            items: value.items.transform_values do |item|
+              {
+                key: item.key,
+                status: item.status,
+                result: serialize_state_value(item.result),
+                error: serialize_state_error(item.error)
+              }
+            end
+          }
+        else
+          value
+        end
+      end
+
+      def deserialize_state_value(node, value)
+        if value.is_a?(Hash) && (value[:type] || value["type"])&.to_sym == :deferred
+          data = value[:data] || value["data"] || {}
+          return Runtime::DeferredResult.build(
+            token: data[:token] || data["token"],
+            payload: data[:payload] || data["payload"] || {},
+            source_node: data[:source_node] || data["source_node"],
+            waiting_on: data[:waiting_on] || data["waiting_on"]
+          )
+        end
+
+        if value.is_a?(Hash) && (value[:type] || value["type"])&.to_sym == :result_snapshot
+          snapshot = value[:snapshot] || value["snapshot"] || {}
+          if node.kind == :composition
+            child_contract = node.contract_class.restore(snapshot)
+            return child_contract.result
+          end
+
+          if node.kind == :branch
+            snapshot_graph = snapshot[:graph] || snapshot["graph"]
+            contract_class = node.possible_contracts.find { |candidate| candidate.compiled_graph.name == snapshot_graph }
+            return value unless contract_class
+
+            child_contract = contract_class.restore(snapshot)
+            return child_contract.result
+          end
+
+          if node.kind == :collection
+            child_contract = node.contract_class.restore(snapshot)
+            return child_contract.result
+          end
+        end
+
+        if value.is_a?(Hash) && (value[:type] || value["type"])&.to_sym == :collection_result
+          items = (value[:items] || value["items"] || {}).each_with_object({}) do |(key, item), memo|
+            memo[key.is_a?(String) && key.match?(/\A\d+\z/) ? key.to_i : key] = Runtime::CollectionResult::Item.new(
+              key: item[:key] || item["key"] || key,
+              status: (item[:status] || item["status"]).to_sym,
+              result: deserialize_state_value(node, item[:result] || item["result"]),
+              error: deserialize_state_error(item[:error] || item["error"])
+            )
+          end
+          return Runtime::CollectionResult.new(
+            items: items,
+            mode: (value[:mode] || value["mode"] || :collect).to_sym
+          )
+        end
+
+        value
+      end
+
+      def serialize_state_error(error)
+        return nil unless error
+
+        {
+          type: error.class.name,
+          message: error.message,
+          context: error.respond_to?(:context) ? error.context : {}
+        }
+      end
+
+      def deserialize_state_error(error_data)
+        return nil unless error_data
+
+        ResolutionError.new(
+          error_data[:message] || error_data["message"],
+          context: error_data[:context] || error_data["context"] || {}
+        )
+      end
+
+      def deserialize_time(value)
+        case value
+        when Time
+          value
+        when String
+          Time.iso8601(value)
+        else
+          value || Time.now.utc
+        end
+      end
+
+      def value_from(data, key)
+        data[key] || data[key.to_s]
+      end
+
+      alias_method :resolve_output_value, :resolve_exported_output
     end
   end
 end

data/lib/igniter/runtime/input_validator.rb

@@ -3,17 +3,6 @@
 module Igniter
   module Runtime
     class InputValidator
-      SUPPORTED_TYPES = {
-        integer: Integer,
-        float: Float,
-        numeric: Numeric,
-        string: String,
-        boolean: :boolean,
-        array: Array,
-        hash: Hash,
-        symbol: Symbol
-      }.freeze
-
       def initialize(compiled_graph)
         @compiled_graph = compiled_graph
       end
@@ -104,26 +93,15 @@ module Igniter
         return if value.nil?
         return unless input_node.type
 
-        unless supported_type?(input_node.type)
+        unless TypeSystem.supported?(input_node.type)
           raise input_error(input_node, "Unsupported input type '#{input_node.type}' for '#{input_node.name}'")
         end
 
-        return if type_match?(input_node.type, value)
+        return if TypeSystem.match?(input_node.type, value)
 
         raise input_error(input_node, "Input '#{input_node.name}' must be of type #{input_node.type}, got #{value.class}")
       end
 
-      def supported_type?(type)
-        SUPPORTED_TYPES.key?(type.to_sym)
-      end
-
-      def type_match?(type, value)
-        matcher = SUPPORTED_TYPES.fetch(type.to_sym)
-        return value == true || value == false if matcher == :boolean
-
-        value.is_a?(matcher)
-      end
-
       def symbolize_keys(hash)
         hash.each_with_object({}) { |(key, value), memo| memo[key.to_sym] = value }
       end

data/lib/igniter/runtime/invalidator.rb

@@ -37,7 +37,7 @@ module Igniter
 
       def emit_output_invalidations_for(source_name, cause_name)
         @execution.compiled_graph.outputs.each do |output_node|
-          next unless output_node.source == source_name.to_sym
+          next unless output_node.source_root == source_name.to_sym
 
           @execution.events.emit(
             :node_invalidated,

data/lib/igniter/runtime/job_worker.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Igniter
+  module Runtime
+    class JobWorker
+      def initialize(contract_class, store: Igniter.execution_store)
+        @contract_class = contract_class
+        @store = store
+      end
+
+      def resume(execution_id:, token:, value:)
+        contract = @contract_class.restore_from_store(execution_id, store: @store)
+        contract.execution.resume_by_token(token, value: value)
+        contract
+      end
+    end
+  end
+end

data/lib/igniter/runtime/node_state.rb

@@ -19,6 +19,14 @@ module Igniter
         status == :stale
       end
 
+      def pending?
+        status == :pending
+      end
+
+      def running?
+        status == :running
+      end
+
       def succeeded?
         status == :succeeded
       end
@@ -26,6 +34,18 @@ module Igniter
       def failed?
         status == :failed
       end
+
+      def to_h
+        {
+          node_name: node.name,
+          status: status,
+          version: version,
+          resolved_at: resolved_at,
+          invalidated_by: invalidated_by,
+          value: value,
+          error: error
+        }
+      end
     end
   end
 end

data/lib/igniter/runtime/planner.rb

@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+
+module Igniter
+  module Runtime
+    class Planner
+      def initialize(execution)
+        @execution = execution
+      end
+
+      def targets_for_outputs(output_names = nil)
+        selected_outputs = if output_names
+          Array(output_names).map { |name| @execution.compiled_graph.fetch_output(name) }
+        else
+          @execution.compiled_graph.outputs
+        end
+
+        selected_outputs
+          .map(&:source_root)
+          .uniq
+      end
+
+      def plan(output_names = nil)
+        targets = targets_for_outputs(output_names)
+        nodes = relevant_nodes_for(targets)
+
+        node_entries = nodes.each_with_object({}) do |node, memo|
+          memo[node.name] = plan_entry(node)
+        end
+
+        {
+          targets: targets,
+          ready: node_entries.values.select { |entry| entry[:ready] }.map { |entry| entry[:name] },
+          blocked: node_entries.values.select { |entry| entry[:blocked] }.map { |entry| entry[:name] },
+          nodes: node_entries
+        }
+      end
+
+      private
+
+      def relevant_nodes_for(targets)
+        seen = {}
+        ordered = []
+
+        targets.each do |target_name|
+          visit(@execution.compiled_graph.fetch_node(target_name), seen, ordered)
+        end
+
+        ordered
+      end
+
+      def visit(node, seen, ordered)
+        return if seen[node.name]
+
+        seen[node.name] = true
+        node.dependencies.each do |dependency_name|
+          dependency = dependency_node_for(dependency_name)
+          visit(dependency, seen, ordered)
+        end
+        ordered << node
+      end
+
+      def dependency_node_for(dependency_name)
+        dependency = @execution.compiled_graph.fetch_dependency(dependency_name)
+        return dependency if dependency.kind != :output
+
+        @execution.compiled_graph.fetch_node(dependency.source_root)
+      end
+
+      def plan_entry(node)
+        state = @execution.cache.fetch(node.name)
+        dependency_entries = node.dependencies.map { |dependency_name| dependency_entry(dependency_name) }
+        blocked_dependencies = dependency_entries.reject { |entry| entry[:satisfied] }.map { |entry| entry[:name] }
+        ready = resolution_required?(state) && blocked_dependencies.empty?
+
+        {
+          id: node.id,
+          name: node.name,
+          path: node.path,
+          kind: node.kind,
+          status: state&.status || :pending,
+          ready: ready,
+          blocked: !ready && resolution_required?(state),
+          dependencies: dependency_entries,
+          waiting_on: blocked_dependencies
+        }
+      end
+
+      def dependency_entry(dependency_name)
+        dependency = @execution.compiled_graph.fetch_dependency(dependency_name)
+        source_node = dependency.kind == :output ? @execution.compiled_graph.fetch_node(dependency.source_root) : dependency
+        state = @execution.cache.fetch(source_node.name)
+
+        {
+          name: dependency_name.to_sym,
+          source: source_node.name,
+          kind: dependency.kind,
+          status: state&.status || inferred_status(source_node),
+          satisfied: dependency_satisfied?(source_node, state)
+        }
+      end
+
+      def dependency_satisfied?(node, state)
+        case node.kind
+        when :input
+          input_available?(node)
+        else
+          state&.succeeded?
+        end
+      end
+
+      def inferred_status(node)
+        return :ready if node.kind == :input && input_available?(node)
+
+        :pending
+      end
+
+      def input_available?(node)
+        @execution.inputs.key?(node.name) || node.default?
+      end
+
+      def resolution_required?(state)
+        state.nil? || state.stale? || state.pending? || state.running?
+      end
+    end
+  end
+end