roast-ai 0.4.8 → 0.4.10

data/lib/roast/dsl/config_manager.rb

@@ -0,0 +1,96 @@
+# typed: true
+# frozen_string_literal: true
+
+module Roast
+  module DSL
+    class ConfigManager
+      class ConfigManagerError < Roast::Error; end
+      class ConfigManagerNotPreparedError < ConfigManagerError; end
+      class ConfigManagerAlreadyPreparedError < ConfigManagerError; end
+
+      #: (Cog::Registry, Array[^() -> void]) -> void
+      def initialize(cog_registry, config_procs)
+        @cog_registry = cog_registry
+        @config_procs = config_procs
+        @config_context = ConfigContext.new #: ConfigContext
+        @general_configs = {} #: Hash[singleton(Cog), Cog::Config]
+        @name_scoped_configs = {} #: Hash[singleton(Cog), Hash[Symbol, Cog::Config]]
+      end
+
+      #: () -> void
+      def prepare!
+        raise ConfigManagerAlreadyPreparedError if preparing? || prepared?
+
+        @preparing = true
+        bind_registered_cogs
+        @config_procs.each { |cp| @config_context.instance_eval(&cp) }
+        @prepared = true
+      end
+
+      #: () -> bool
+      def preparing?
+        @preparing ||= false
+      end
+
+      #: () -> bool
+      def prepared?
+        @prepared ||= false
+      end
+
+      #: (singleton(Cog), ?Symbol?) -> Cog::Config
+      def config_for(cog_class, name = nil)
+        raise ConfigManagerNotPreparedError unless prepared?
+
+        # All cogs will always have a config; empty by default if the cog was never explicitly configured
+        config = fetch_general_config(cog_class)
+        name_scoped_config = fetch_name_scoped_config(cog_class, name) unless name.nil?
+        config = config.merge(name_scoped_config) if name_scoped_config
+        config
+      end
+
+      private
+
+      #: (singleton(Cog)) -> Cog::Config
+      def fetch_general_config(cog_class)
+        @general_configs[cog_class] ||= cog_class.config_class.new
+      end
+
+      #: (singleton(Cog), Symbol) -> Cog::Config
+      def fetch_name_scoped_config(cog_class, name)
+        name_scoped_configs_for_cog = @name_scoped_configs[cog_class] ||= {}
+        name_scoped_configs_for_cog[name] ||= cog_class.config_class.new
+      end
+
+      #: () -> void
+      def bind_registered_cogs
+        @cog_registry.cogs.each { |cog_method_name, cog_class| bind_cog(cog_method_name, cog_class) }
+      end
+
+      #: (Symbol, singleton(Cog)) -> void
+      def bind_cog(cog_method_name, cog_class)
+        on_config_method = method(:on_config)
+        cog_method = proc do |cog_name = nil, &cog_config_proc|
+          on_config_method.call(cog_class, cog_name, &cog_config_proc)
+        end
+        @config_context.instance_eval do
+          define_singleton_method(cog_method_name, cog_method)
+        end
+      end
+
+      #: (singleton(Cog), Symbol) { () -> void } -> void
+      def on_config(cog_class, cog_name, &cog_config_proc)
+        # Called when the cog method is invoked in the workflow's 'config' block.
+        # This allows configuration parameters to be set for the cog generally or for a specific named instance
+        config_object = if cog_name.nil?
+          fetch_general_config(cog_class)
+        else
+          fetch_name_scoped_config(cog_class, cog_name)
+        end
+        # NOTE: Sorbet expects the proc passed to instance_exec to be declared as taking an argument
+        # but our cog_config_proc does not get an argument
+        config_object.instance_exec(&T.unsafe(cog_config_proc)) if cog_config_proc
+        config_object
+      end
+    end
+  end
+end

data/lib/roast/dsl/execution_context.rb

@@ -0,0 +1,9 @@
+# typed: true
+# frozen_string_literal: true
+
+module Roast
+  module DSL
+    # Context in which the `execute` blocks of a workflow definition are evaluated
+    class ExecutionContext; end
+  end
+end

data/lib/roast/dsl/execution_manager.rb

@@ -0,0 +1,137 @@
+# typed: true
+# frozen_string_literal: true
+
+module Roast
+  module DSL
+    # Context in which the `execute` block of a workflow is evaluated
+    class ExecutionManager
+      class ExecutionManagerError < Roast::Error; end
+      class ExecutionManagerNotPreparedError < ExecutionManagerError; end
+      class ExecutionManagerAlreadyPreparedError < ExecutionManagerError; end
+      class ExecutionManagerCurrentlyRunningError < ExecutionManagerError; end
+      class ExecutionScopeDoesNotExistError < ExecutionManagerError; end
+      class ExecutionScopeNotSpecifiedError < ExecutionManagerError; end
+
+      #: (Cog::Registry, ConfigManager, Hash[Symbol?, Array[^() -> void]], ?Symbol?) -> void
+      def initialize(cog_registry, config_manager, all_execution_procs, scope = nil)
+        @cog_registry = cog_registry
+        @config_manager = config_manager
+        @all_execution_procs = all_execution_procs
+        @scope = scope
+        @cogs = Cog::Store.new #: Cog::Store
+        @cog_stack = Cog::Stack.new #: Cog::Stack
+        @execution_context = ExecutionContext.new #: ExecutionContext
+        @cog_input_manager = CogInputManager.new(@cog_registry, @cogs) #: CogInputManager
+      end
+
+      #: () -> void
+      def prepare!
+        raise ExecutionManagerAlreadyPreparedError if preparing? || prepared?
+
+        @preparing = true
+        bind_registered_cogs
+        my_execution_procs.each { |ep| @execution_context.instance_eval(&ep) }
+        @prepared = true
+      end
+
+      def run!
+        raise ExecutionManagerNotPreparedError unless prepared?
+        raise ExecutionManagerCurrentlyRunningError if running?
+
+        @running = true
+        @cog_stack.map do |name, cog|
+          cog.run!(
+            @config_manager.config_for(cog.class, name.to_sym),
+            cog_input_manager,
+          )
+        end
+        @running = false
+      end
+
+      #: () -> bool
+      def preparing?
+        @preparing ||= false
+      end
+
+      #: () -> bool
+      def prepared?
+        @prepared ||= false
+      end
+
+      #: () -> bool
+      def running?
+        @running ||= false
+      end
+
+      #: () -> CogInputContext
+      def cog_input_manager
+        raise ExecutionManagerNotPreparedError unless prepared?
+
+        @cog_input_manager.context
+      end
+
+      private
+
+      #: () -> Array[^() -> void]
+      def my_execution_procs
+        raise ExecutionScopeDoesNotExistError unless @all_execution_procs.key?(@scope)
+
+        @all_execution_procs[@scope] || []
+      end
+
+      #: (Symbol, Cog) -> void
+      def add_cog_instance(name, cog)
+        @cogs.insert(name, cog)
+        @cog_stack.push([name, cog])
+      end
+
+      # TODO: add typing for output
+      #: (Symbol) -> untyped
+      def output(name)
+        @cogs[name].output
+      end
+
+      #: () -> void
+      def bind_registered_cogs
+        @cog_registry.cogs.each { |cog_method_name, cog_class| bind_cog(cog_method_name, cog_class) }
+      end
+
+      #: (Symbol, singleton(Cog)) -> void
+      def bind_cog(cog_method_name, cog_class)
+        on_execute_method = method(:on_execute)
+        cog_method = proc do |cog_name = Random.uuid, &cog_input_proc|
+          on_execute_method.call(cog_class, cog_name, &cog_input_proc)
+        end
+        @execution_context.instance_eval do
+          define_singleton_method(cog_method_name, cog_method)
+        end
+      end
+
+      #: (singleton(Cog), Symbol) { (Cog::Input) -> untyped } -> void
+      def on_execute(cog_class, cog_name, &cog_input_proc)
+        # Called when the cog method is invoked in the workflow's 'execute' block.
+        # This creates the cog instance and prepares it for execution.
+        cog_instance = if cog_class == Cogs::Execute
+          create_special_execute_cog(cog_name, cog_input_proc)
+        else
+          cog_class.new(cog_name, cog_input_proc)
+        end
+        add_cog_instance(cog_name, cog_instance)
+      end
+
+      #: (Symbol, ^(Cogs::Execute::Input) -> untyped) -> Cogs::Execute
+      def create_special_execute_cog(cog_name, cog_input_proc)
+        trigger = proc do |input|
+          raise ExecutionScopeNotSpecifiedError unless input.scope.present?
+
+          em = ExecutionManager.new(@cog_registry, @config_manager, @all_execution_procs, input.scope)
+          em.prepare!
+          em.run!
+
+          # TODO: collect the outputs of the cogs in the execution manager that just ran and do something with them
+        end
+        Cogs::Execute.new(cog_name, cog_input_proc, trigger)
+      end
+    end
+  end
+end

data/lib/roast/dsl/workflow.rb

@@ -0,0 +1,113 @@
+# typed: true
+# frozen_string_literal: true
+
+module Roast
+  module DSL
+    class Workflow
+      class WorkflowError < Roast::Error; end
+      class WorkflowNotPreparedError < WorkflowError; end
+      class WorkflowAlreadyPreparedError < WorkflowError; end
+      class WorkflowAlreadyStartedError < WorkflowError; end
+      class InvalidCogReference < WorkflowError; end
+
+      class << self
+        #: (String) -> void
+        def from_file(workflow_path)
+          workflow = new(workflow_path)
+          workflow.prepare!
+          workflow.start!
+        end
+      end
+
+      #: (String) -> void
+      def initialize(workflow_path)
+        @workflow_path = Pathname.new(workflow_path)
+        @workflow_definition = File.read(workflow_path)
+        @cog_registry = Cog::Registry.new #: Cog::Registry
+        @config_procs = [] #: Array[^() -> void]
+        @execution_procs = { nil: [] } #: Hash[Symbol?, Array[^() -> void]]
+        @config_manager = nil #: ConfigManager?
+        @execution_manager = nil #: ExecutionManager?
+      end
+
+      #: () -> void
+      def prepare!
+        raise WorkflowAlreadyPreparedError if preparing? || prepared?
+
+        @preparing = true
+        extract_dsl_procs!
+        @config_manager = ConfigManager.new(@cog_registry, @config_procs)
+        @config_manager.prepare!
+        @execution_manager = ExecutionManager.new(@cog_registry, @config_manager, @execution_procs)
+        @execution_manager.prepare!
+
+        @prepared = true
+      end
+
+      #: () -> void
+      def start!
+        raise WorkflowNotPreparedError unless @config_manager.present? && @execution_manager.present?
+        raise WorkflowAlreadyStartedError if started? || completed?
+
+        @started = true
+        @execution_manager.run!
+        @completed = true
+      end
+
+      #: () -> bool
+      def preparing?
+        @preparing ||= false
+      end
+
+      #: () -> bool
+      def prepared?
+        @prepared ||= false
+      end
+
+      #: () -> bool
+      def started?
+        @started ||= false
+      end
+
+      #: () -> bool
+      def completed?
+        @completed ||= false
+      end
+
+      #: { () [self: ConfigContext] -> void } -> void
+      def config(&block)
+        @config_procs << block
+      end
+
+      #: (?Symbol?) { () [self: ExecutionContext] -> void } -> void
+      def execute(scope = nil, &block)
+        (@execution_procs[scope] ||= []) << block
+      end
+
+      def use(cogs = [], from: nil)
+        require from if from
+
+        Array.wrap(cogs).each do |cog_name|
+          path = @workflow_path.realdirpath.dirname.join("cogs/#{cog_name}")
+          require path.to_s if from.nil?
+
+          cog_class_name = cog_name.camelize
+          raise InvalidCogReference, "Expected #{cog_class_name} class, not found in #{path}" unless Object.const_defined?(cog_class_name)
+
+          cog_class = cog_class_name.constantize # rubocop:disable Sorbet/ConstantsFromStrings
+          @cog_registry.use(cog_class)
+        end
+      end
+
+      private
+
+      # Evaluate the top-level workflow definition
+      # This collects the procs passed to `config` and `execute` calls in the workflow definition,
+      # but does not evaluate any of them individually yet.
+      #: () -> void
+      def extract_dsl_procs!
+        instance_eval(@workflow_definition)
+      end
+    end
+  end
+end

data/lib/roast/error.rb

@@ -0,0 +1,7 @@
+# typed: true
+# frozen_string_literal: true
+
+module Roast
+  # Base class for all internal Roast errors.
+  class Error < StandardError; end
+end

data/lib/roast/errors.rb

@@ -4,12 +4,12 @@
 module Roast
   module Errors
     # Custom error for API resource not found (404) responses
-    class ResourceNotFoundError < StandardError; end
+    class ResourceNotFoundError < Roast::Error; end
 
     # Custom error for when API authentication fails
-    class AuthenticationError < StandardError; end
+    class AuthenticationError < Roast::Error; end
 
     # Exit the app, for instance via Ctrl-C during an InputStep
-    class ExitEarly < StandardError; end
+    class ExitEarly < Roast::Error; end
   end
 end

data/lib/roast/graph/edge.rb

@@ -0,0 +1,25 @@
+# typed: true
+# frozen_string_literal: true
+
+module Roast
+  class Graph
+    class Edge
+      #: (Node) -> void
+      attr_writer :join_node
+
+      attr_reader :from_node, :to_node
+
+      #: (Node, Node, ?proc: Proc?) -> void
+      def initialize(from_node, to_node, proc: nil)
+        @from_node = from_node
+        @to_node = to_node
+        @proc = proc # TODO: Shadowing proc builtin here
+      end
+
+      #: () -> String
+      def to_s
+        "#{@from_node} -> #{@to_node}"
+      end
+    end
+  end
+end

data/lib/roast/graph/node.rb

@@ -0,0 +1,40 @@
+# typed: true
+# frozen_string_literal: true
+
+module Roast
+  class Graph
+    class Node
+      # class InvalidExecutableError < Roast::Error; end
+
+      attr_reader :name, :executable
+
+      #: (Symbol, ?executable: Proc | Graph | nil) -> void
+      def initialize(name, executable: nil)
+        @name = name
+        @executable = executable
+      end
+
+      #: () -> T::Boolean
+      def subgraph?
+        @executable.is_a?(Graph)
+      end
+
+      #: () -> T::Boolean
+      def done?
+        @name == :DONE
+      end
+
+      #: (Hash) -> void
+      def execute(state)
+        return if @executable.nil?
+
+        executable = @executable
+        if executable.is_a?(Proc)
+          executable.call(state)
+        elsif executable.is_a?(Graph)
+          executable.execute(state)
+        end
+      end
+    end
+  end
+end

data/lib/roast/graph/quantum_edge.rb

@@ -0,0 +1,27 @@
+# typed: true
+# frozen_string_literal: true
+
+module Roast
+  class Graph
+    class QuantumEdge
+      #: (Roast::Graph::Node, Proc) -> void
+      def initialize(from_node, to_proc)
+        @from_node = from_node
+        @to_proc = to_proc
+      end
+
+      #: (Hash[untyped, untyped], Hash[Symbol, Roast::Graph::Node]) -> Array[Roast::Graph::Edge]
+      def collapse(state, nodes)
+        to_node_names = @to_proc.call(state)
+        to_node_names = to_node_names.is_a?(Array) ? to_node_names : [to_node_names]
+
+        to_node_names.map do |to_node_name|
+          to_node = nodes[to_node_name]
+          raise Error, "No node found with name #{to_node_name.inspect}" if to_node.nil?
+
+          Edge.new(@from_node, to_node)
+        end
+      end
+    end
+  end
+end

data/lib/roast/graph/threaded_exec.rb

@@ -0,0 +1,93 @@
+# typed: true
+# frozen_string_literal: true
+
+module Roast
+  class Graph
+    class StateConflictError < Error; end
+
+    class ThreadedExec
+      def initialize(nodes, og_state)
+        @nodes = nodes
+        @og_state = og_state
+      end
+
+      #: () -> Hash[untyped, untyped]
+      def async_execute
+        states = threaded_execute(@nodes, @og_state)
+        merge_states!(@og_state, states)
+        @og_state
+      end
+
+      # Returns a hash of the new states for each node.
+      #: (Array[Node], Hash[untyped, untyped]) -> Hash[Symbol, Hash[untyped, untyped]]
+      def threaded_execute(nodes, og_state)
+        states = {}
+        threads = nodes.map do |current_node|
+          states[current_node.name] = og_state.dup
+          Thread.new do
+            current_node.execute(states[current_node.name])
+          end
+        end
+
+        threads.map(&:value)
+
+        states
+      end
+
+      #: (Hash, Hash) -> void
+      def merge_states!(orig_state, new_states)
+        orig_state.merge!(merge_new_states(orig_state, new_states))
+      end
+
+      # We merge all the new states together first so we can catch any keys that were modified by multiple threads.
+      #: (Hash[untyped, untyped], Hash[Symbol, Hash[untyped, untyped]]) -> Hash[untyped, untyped]
+      def merge_new_states(orig_state, new_states)
+        # We only care about what the threads have changes from the original state.
+        changed_states = changed_states(orig_state, new_states)
+
+        return {} if changed_states.empty?
+
+        # Grab some entry to be the one we merge all the other changes into.
+        base_node_name, base_changed_state = T.must(changed_states.shift)
+        base_node_name = T.cast(base_node_name, Symbol)
+        base_changed_state = T.cast(base_changed_state, T::Hash[T.untyped, T.untyped])
+
+        changed_states.each do |node_name, changed_state|
+          base_changed_state.merge!(changed_state) do |key, old_value, new_value|
+            raise_state_conflict(key, old_value, new_value, [base_node_name, node_name])
+          end
+        end
+
+        base_changed_state
+      end
+
+      #: (Hash, Hash[Symbol, Hash]) -> Hash[Symbol, Hash]
+      def changed_states(orig_state, new_states)
+        new_states.transform_values do |new_state|
+          changed_state_entries(orig_state, new_state)
+        end
+      end
+
+      # Filter new states content to only include keys that were modified by the new states.
+      #: (Hash, Hash) -> Hash
+      def changed_state_entries(orig_state, new_state)
+        new_state.reject do |key, value|
+          orig_state[key] == value
+        end
+      end
+
+      #: (Symbol, untyped, untyped, Array[Symbol]) -> void
+      def raise_state_conflict(key, old_value, new_value, conflicting_nodes)
+        raise StateConflictError, <<~CONFLICT.chomp
+          Parallel nodes modified the same state key.
+          Conflicting nodes: #{conflicting_nodes.join(", ")}
+          Key: :#{key}
+          Old value:
+          #{old_value.inspect}
+          New value:
+          #{new_value.inspect}
+        CONFLICT
+      end
+    end
+  end
+end