smart_proxy_dynflow 0.7.0 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d19598abd3ab23ee44430f2e5c2207c6ac11d9c7919769ed5885408ee2e39f1c
-  data.tar.gz: 39d351c883a1f3c5d9402462961591d7bd4bd70e4ae0ca04d8c7095a710c70d2
+  metadata.gz: 8852c64e45de97691310f1c9fd17bef228a9ea87db2085377cee1cd404546752
+  data.tar.gz: 54afa19849d245f44f8d5e03c69d82e30015ff95978b266cad0db3919eb82195
 SHA512:
-  metadata.gz: a557af273f69dda522cd9236d4f99cce270fd66d71c3b09150df6ba37eecbb0ddd2663e670b81c5ba806dd1af6927740c66f70b02551a42468d8e39f296f9e0f
-  data.tar.gz: 37a46da28658e4b26eceb19a86bdd2849cc387caf004213166bfd16580ec603a3f035fe797a7892423fad9e97129fcfeb95c82a789522d3088bf2a0dc3a3eefd
+  metadata.gz: 4a1799f87c74aa10807ea642d217686f208fe2de5444b4a87c540a8685e419d4c88b272e03faa83a9756fa7ef4e996f3cd8195f730ae43e97800e4666fecae59
+  data.tar.gz: ab1dd31d7555502d566803ba4e87965e2431bb6d32c67cc6e0e39f7ac1c6c5369e22b1b6c7f6e3abb8cfdcccf6b4432029ba10b277171931f6a5fa05ed792d8d

data/lib/smart_proxy_dynflow/action/batch_runner.rb

@@ -2,13 +2,13 @@ require 'smart_proxy_dynflow/action/runner'
 
 module Proxy::Dynflow::Action
   class BatchRunner < ::Proxy::Dynflow::Action::Runner
-    def plan(launcher, input)
-      plan_self :targets => launcher.runner_input(input), :operation => launcher.operation
+    def plan(launcher, input, runner_id)
+      plan_self :targets => launcher.runner_input(input), :operation => launcher.operation, :runner_id => runner_id
     end
 
     def initiate_runner
       launcher = Proxy::Dynflow::TaskLauncherRegistry.fetch(input[:operation])
-      launcher.runner_class.new(input[:targets], suspended_action: suspended_action)
+      launcher.runner_class.new(input[:targets], suspended_action: suspended_action, id: input[:runner_id])
     end
   end
 end

data/lib/smart_proxy_dynflow/action/external_polling.rb

@@ -11,7 +11,6 @@ module Proxy::Dynflow::Action
       end
     end
 
-    def poll
-    end
+    def poll; end
   end
 end

data/lib/smart_proxy_dynflow/action/output_collector.rb

@@ -2,6 +2,7 @@ module Proxy::Dynflow::Action
   class OutputCollector < ::Proxy::Dynflow::Action::Runner
     def init_run
       output[:result] = []
+      output[:runner_id] = input[:runner_id]
       suspend
     end
   end

data/lib/smart_proxy_dynflow/action/runner.rb

@@ -1,8 +1,10 @@
 require 'smart_proxy_dynflow/action/shareable'
+require 'smart_proxy_dynflow/action/external_polling'
 module Proxy::Dynflow
   module Action
     class Runner < Shareable
       include ::Dynflow::Action::Cancellable
+      include ::Proxy::Dynflow::Action::WithExternalPolling
 
       def run(event = nil)
         case event
@@ -14,6 +16,9 @@ module Proxy::Dynflow
           process_external_event(event)
         when ::Dynflow::Action::Cancellable::Cancel
           kill_run
+        when ::Proxy::Dynflow::Action::WithExternalPolling::Poll
+          poll
+          suspend
         else
           raise "Unexpected event #{event.inspect}"
         end
@@ -69,6 +74,10 @@ module Proxy::Dynflow
         end
       end
 
+      def poll
+        runner_dispatcher.refresh_output(output[:runner_id])
+      end
+
       def failed_run?
         output[:exit_status] != 0
       end

data/lib/smart_proxy_dynflow/core.rb

@@ -1,3 +1,5 @@
+require 'fileutils'
+
 module Proxy::Dynflow
   class Core
     attr_accessor :world, :accepted_cert_serial
@@ -29,6 +31,7 @@ module Proxy::Dynflow
         Log.instance.warn "Could not open DB for dynflow at '#{db_file}', " \
                           "will keep data in memory. Restart will drop all dynflow data."
       else
+        FileUtils.mkdir_p(File.dirname(db_file))
         db_conn_string += "/#{db_file}"
       end
 

data/lib/smart_proxy_dynflow/helpers.rb

@@ -36,9 +36,8 @@ module Proxy
       def task_status(task_id)
         ep = world.persistence.load_execution_plan(task_id)
         actions = ep.actions.map do |action|
-          hash = action.to_hash
-          hash[:output][:result] = action.output_result if action.is_a?(Proxy::Dynflow::Action::Runner)
-          hash
+          refresh_output(ep, action)
+          expand_output(action)
         end
         ep.to_hash.merge(:actions => actions)
       rescue KeyError => _e
@@ -58,6 +57,18 @@ module Proxy
                     params['step_id'].to_i,
                     ::Proxy::Dynflow::Runner::ExternalEvent.new(params))
       end
+
+      def refresh_output(execution_plan, action)
+        if action.is_a?(Proxy::Dynflow::Action::WithExternalPolling) && %i[running suspended].include?(action.run_step&.state)
+          world.event(execution_plan.id, action.run_step_id, Proxy::Dynflow::Action::WithExternalPolling::Poll)
+        end
+      end
+
+      def expand_output(action)
+        hash = action.to_hash
+        hash[:output][:result] = action.output_result if action.is_a?(Proxy::Dynflow::Action::Runner)
+        hash
+      end
     end
   end
 end

data/lib/smart_proxy_dynflow/io_buffer.rb

@@ -0,0 +1,115 @@
+module Proxy
+  module Dynflow
+    # A buffer around an IO object providing buffering and convenience methods
+    # for non-blocking reads and writes.
+    #
+    # @note Using a single IOBuffer with a single IO for both reads and writes might not be a good idea. If you need to use a single IO for both reads and writes, wrap it in two separate IOBuffers.
+    #
+    # @attr_accessor [IO] io The IO which the buffer wraps
+    # @attr_reader [String] buffer The buffer where the data read from the underlying IO is buffered
+    class IOBuffer
+      attr_accessor :io
+      attr_reader :buffer
+
+      # @param [IO] io The IO object to be buffered
+      def initialize(io)
+        @buffer = ''
+        @io = io
+      end
+
+      # Sets a callback to be executed each time data is read from the
+      # underlying IO.
+      #
+      # @note Note that if the callback is provided, the buffer will store the return value of the callback instead of the raw data.
+      #
+      # @yieldparam [String] data read from the underlying IO
+      # @yieldreturn [String] data to be buffered
+      # @return [void]
+      def on_data(&block)
+        @callback = block
+      end
+
+      # Exposes the underlying IO so that the buffer itself can be used in IO.select calls.
+      #
+      # @return [IO] the underlying IO
+      def to_io
+        @io
+      end
+
+      # Exposes the contents of the buffer as a String
+      #
+      # @return [String] the buffered data
+      def to_s
+        @buffer
+      end
+
+      # Checks whether the buffer is empty
+      #
+      # @return [true, false] whether the buffer is empty
+      def empty?
+        @buffer.empty?
+      end
+
+      # Checks whether the underlying IO is empty
+      #
+      # @return [true, false] whether the underlying IO is empty
+      def closed?
+        @io.closed?
+      end
+
+      # Closes the underlying IO. Does nothing if the IO is already closed.
+      #
+      # @return [void]
+      def close
+        @io.close unless @io.closed?
+      end
+
+      # Reads all the data that is currently waiting in the IO and stores it. If
+      # EOFError is encountered during the read, the underlying IO is closed.
+      #
+      # @return [void]
+      def read_available!
+        data = ''
+        loop { data += @io.read_nonblock(4096) }
+      rescue IO::WaitReadable # rubocop:disable Lint/HandleExceptions
+      rescue EOFError
+        close
+      ensure
+        @buffer += with_callback(data) unless data.empty?
+      end
+
+      # Writes all the data into the IO that can be written without blocking. It
+      # is a no-op if there are no data to be written. If an EOFError is
+      # encountered during the write, the underlying IO is closed.
+      #
+      # @return [void]
+      def write_available!
+        until @buffer.empty?
+          n = @io.write_nonblock(@buffer)
+          @buffer = @buffer[n..-1]
+        end
+      rescue IO::WaitWritable # rubocop:disable Lint/HandleExceptions
+      rescue EOFError
+        close
+      end
+
+      # Adds data to the buffer. If the buffer is used for writing, then this
+      # should be the preferred method of queueing the data to be written.
+      #
+      # @return [void]
+      def add_data(data)
+        @buffer += data
+      end
+
+      private
+
+      def with_callback(data)
+        if @callback
+          @callback.call(data)
+        else
+          data
+        end
+      end
+    end
+  end
+end

data/lib/smart_proxy_dynflow/plugin.rb

@@ -11,7 +11,7 @@ module Proxy::Dynflow
     settings_file "dynflow.yml"
     requires :foreman_proxy, ">= 1.16.0"
     default_settings :console_auth => true,
-                     :execution_plan_cleaner_age => 60 * 60 * 24
+                     :execution_plan_cleaner_age => 60 * 30
     plugin :dynflow, Proxy::Dynflow::VERSION
 
     capability(proc { self.available_operations })

data/lib/smart_proxy_dynflow/process_manager.rb

@@ -0,0 +1,166 @@
+require 'smart_proxy_dynflow/io_buffer'
+
+module Proxy
+  module Dynflow
+    # An abstraction for managing local processes.
+    #
+    # It can be used to:
+    # - spawn a local process
+    # - track its lifecycle
+    # - communicate with it through its standard input, output and error
+    # - step through the execution one event at a time or start the child process and wait until it finishes
+    #
+    # @example Run date command and collect its output
+    #   pm = ProcessManager.new('date')
+    #   pm.run!
+    #   pm.status #=> 0
+    #   pm.stdout.to_s.chomp #=> "Thu Feb  3 04:27:42 PM CET 2022"
+    #
+    # @example Run a shell loop, outputting all the lines it generates
+    #   pm = ProcessManager.new(['/bin/sh', '-c', 'for i in 1 2 3; do echo $i; sleep 1; done'])
+    #   pm.on_stdout { |data| puts data; '' }
+    #   pm.run!
+    #   #=> 1
+    #   #=> 2
+    #   #=> 3
+    #
+    # @example Run bc (calculator) interactively and count down from 10 to 0
+    #   pm = ProcessManager.new('bc')
+    #   pm.on_stdout do |data|
+    #     if data.match?(/^\d+/)
+    #       n = data.to_i
+    #       if n.zero?
+    #         pm.stdin.to_io.close
+    #       else
+    #         pm.stdin.add_data("#{n} - 1\n")
+    #       end
+    #     end
+    #     data
+    #   end
+    #   pm.stdin.add_data("10\n")
+    #   pm.run!
+    #   pm.stdout.to_s.lines #=. ["10\n", "9\n", "8\n", "7\n", "6\n", "5\n", "4\n", "3\n", "2\n", "1\n", "0\n"]
+    #
+    # @attr_reader [Proxy::Dynflow::IOBuffer] stdin IOBuffer buffering writes to child process' standard input
+    # @attr_reader [Proxy::Dynflow::IOBuffer] stdout IOBuffer buffering reads from child process' standard output
+    # @attr_reader [Proxy::Dynflow::IOBuffer] stderr IOBuffer buffering reads from child process' standard error
+    # @attr_reader [nil, Integer] pid Process id of the child process, nil if the process was not started yet, -1 if the process could not be started
+    # @attr_reader [nil, Integer] status Exit status of the child process. nil if the child process has not finished yet, 255 if the process could not be started
+    class ProcessManager
+      attr_reader :stdin, :stdout, :stderr, :pid, :status
+
+      # @param [String, [String], [Hash, String]] command A command to run in one of the forms accepted by Kernel.spawn
+      def initialize(command)
+        @command = command
+        @stdin  = IOBuffer.new(nil)
+        @stdout = IOBuffer.new(nil)
+        @stderr = IOBuffer.new(nil)
+      end
+
+      # Starts the process manager and runs it until it finishes
+      #
+      # @return [ProcessManager] the process manager itself to allow method chaining
+      def run!
+        start! unless started?
+        process until done?
+        self
+      end
+
+      # Starts the child process. It creates 3 pipes for communicating with the
+      # child process and the forks it. The process manager is considered done
+      # if the child process cannot be started.
+      #
+      # @return [void]
+      def start!
+        in_read,  in_write  = IO.pipe
+        out_read, out_write = IO.pipe
+        err_read, err_write = IO.pipe
+
+        @pid = spawn(*@command, :in => in_read, :out => out_write, :err => err_write)
+        [in_read, out_write, err_write].each(&:close)
+
+        @stdin.io  = in_write
+        @stdout.io = out_read
+        @stderr.io = err_read
+      rescue Errno::ENOENT => e
+        [in_read, in_write, out_read, out_write, err_read, err_write].each(&:close)
+        @pid = -1
+        @status = 255
+        @stderr.add_data(e.message)
+      end
+
+      # Determines whether the process manager already forked off its child process
+      #
+      # @return [true, false] whether the process manager already forked off its child process
+      def started?
+        !pid.nil?
+      end
+
+      # Determines whether the child process of the process manager already finished
+      #
+      # @return [true, false] whether the child process of the process manager already finished
+      def done?
+        started? && !status.nil?
+      end
+
+      # Runs a single iteration of the manager's processing loop. It waits until either:
+      # - data is available in pipes connected to the child process' standard output or error
+      # - there is pending data to be written and the pipe connected to the child process' standard input is writable
+      # - a timeout is reached
+      #
+      # After the wait, all pending data is read and written.
+      #
+      # If all the pipes connected to the child process are closed, it marks the
+      # execution as complete and performs cleanup.
+      #
+      # @param timeout [nil, Numeric] controls how long this call should wait for data to become available. Waits indefinitely if nil.
+      # @return [void]
+      def process(timeout: nil)
+        raise 'Cannot process until the manager is started' unless started?
+        writers = [@stdin].reject { |buf| buf.empty? || buf.closed? }
+        readers = [@stdout, @stderr].reject(&:closed?)
+
+        if readers.empty? && writers.empty?
+          finish
+          return
+        end
+
+        ready_readers, ready_writers = IO.select(readers, writers, nil, timeout)
+        (ready_readers || []).each(&:read_available!)
+        (ready_writers || []).each(&:write_available!)
+      end
+
+      # Sets block to be executed each time data is read from child process' standard output
+      #
+      # @return [void]
+      def on_stdout(&block)
+        @stdout.on_data(&block)
+      end
+
+      # Sets block to be executed each time data is read from child process' standard error
+      #
+      # @return [void]
+      def on_stderr(&block)
+        @stderr.on_data(&block)
+      end
+
+      # Makes the process manager close all the pipes it may have opened to communicate with the child process
+      #
+      # @return [void]
+      def close
+        [@stdin, @stdout, @stderr].each(&:close)
+      end
+
+      private
+
+      # Makes the process manager finish its run, closing opened FDs and reaping the child process
+      #
+      # @return [void]
+      def finish
+        close
+        _pid, status = Process.wait2(@pid)
+        @status = status.exitstatus
+      end
+    end
+  end
+end

data/lib/smart_proxy_dynflow/runner/base.rb

@@ -6,9 +6,9 @@ module Proxy::Dynflow
       attr_reader :id
       attr_writer :logger
 
-      def initialize(*_args, suspended_action: nil)
+      def initialize(*_args, suspended_action: nil, id: nil)
         @suspended_action = suspended_action
-        @id = SecureRandom.uuid
+        @id = id || SecureRandom.uuid
         initialize_continuous_outputs
       end
 
@@ -93,6 +93,12 @@ module Proxy::Dynflow
       def initialize_continuous_outputs
         @continuous_output = ::Proxy::Dynflow::ContinuousOutput.new
       end
+
+      def run_refresh_output
+        logger.debug('refreshing runner on demand')
+        refresh
+        generate_updates
+      end
     end
   end
 end

data/lib/smart_proxy_dynflow/runner/dispatcher.rb

@@ -43,6 +43,11 @@ module Proxy::Dynflow
           plan_next_refresh
         end
 
+        def refresh_output
+          @logger.debug("refresh output #{@runner.id}")
+          dispatch_updates(@runner.run_refresh_output)
+        end
+
         def dispatch_updates(updates)
           updates.each { |receiver, update| (receiver || @suspended_action) << update }
 
@@ -157,6 +162,12 @@ module Proxy::Dynflow
         end
       end
 
+      def refresh_output(runner_id)
+        synchronize do
+          @runner_actors[runner_id]&.tell([:refresh_output])
+        end
+      end
+
       def handle_command_exception(*args)
         synchronize { _handle_command_exception(*args) }
       end

data/lib/smart_proxy_dynflow/runner/parent.rb

@@ -3,10 +3,10 @@ module Proxy::Dynflow
     class Parent < Base
       # targets = { identifier => { :execution_plan_id => "...", :run_step_id => id,
       #                           :input => { ... } }
-      def initialize(targets = {}, suspended_action: nil)
+      def initialize(targets = {}, suspended_action: nil, id: nil)
         @targets = targets
         @exit_statuses = {}
-        super suspended_action: suspended_action
+        super suspended_action: suspended_action, id: id
       end
 
       def generate_updates

data/lib/smart_proxy_dynflow/runner/process_manager_command.rb

@@ -0,0 +1,36 @@
+require 'smart_proxy_dynflow/process_manager'
+
+module Proxy::Dynflow
+  module Runner
+    module ProcessManagerCommand
+      def initialize_command(*command)
+        @process_manager = ProcessManager.new(command)
+        set_process_manager_callbacks(@process_manager)
+        @process_manager.start!
+        if @process_manager.done? && @process_manager.status == 255
+          publish_exception("Error running command '#{command.join(' ')}'", @process_manager.stderr.to_s)
+        end
+      end
+
+      def set_process_manager_callbacks(pm)
+        pm.on_stdout do |data|
+          publish_data(data, 'stdout')
+          ''
+        end
+        pm.on_stderr do |data|
+          publish_data(data, 'stderr')
+          ''
+        end
+      end
+
+      def refresh
+        @process_manager.process(timeout: 0.1)
+        publish_exit_status(@process_manager.status) if @process_manager.done?
+      end
+
+      def close
+        @process_manager&.close
+      end
+    end
+  end
+end

data/lib/smart_proxy_dynflow/task_launcher/group.rb

@@ -3,6 +3,11 @@ require 'smart_proxy_dynflow/runner'
 module Proxy::Dynflow
   module TaskLauncher
     class AbstractGroup < Batch
+      def initialize(*args)
+        super
+        @runner_id = SecureRandom.uuid
+      end
+
       def self.runner_class
         raise NotImplementedError
       end
@@ -13,7 +18,7 @@ module Proxy::Dynflow
 
       def launch_children(parent, input_hash)
         super(parent, input_hash)
-        trigger(parent, Action::BatchRunner, self, input_hash)
+        trigger(parent, Action::BatchRunner, self, input_hash, @runner_id)
       end
 
       def operation
@@ -36,7 +41,8 @@ module Proxy::Dynflow
       end
 
       def transform_input(input)
-        wipe_callback(input)
+        tmp = wipe_callback(input)
+        input.merge('action_input' => tmp['action_input'].merge(:runner_id => @runner_id))
       end
 
       def wipe_callback(input)

data/lib/smart_proxy_dynflow/version.rb

@@ -1,5 +1,5 @@
 module Proxy
   module Dynflow
-    VERSION = '0.7.0'.freeze
+    VERSION = '0.8.0'.freeze
   end
 end

data/settings.d/dynflow.yml.example

@@ -1,10 +1,10 @@
 ---
 :enabled: true
-:database: /var/lib/foreman-proxy/dynflow/dynflow.sqlite
+:database:
 
 # Require a valid cert to access Dynflow console
 # :console_auth: true
 
 # Maximum age of execution plans to keep before having them cleaned
-# by the execution plan cleaner (in seconds), defaults to 24 hours
-# :execution_plan_cleaner_age: 86400
+# by the execution plan cleaner (in seconds), defaults to 30 minutes
+# :execution_plan_cleaner_age: 1800

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: smart_proxy_dynflow
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.8.0
 platform: ruby
 authors:
 - Ivan Nečas
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-02-17 00:00:00.000000000 Z
+date: 1980-01-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dynflow
@@ -146,8 +146,6 @@ files:
 - Gemfile
 - LICENSE
 - bundler.d/dynflow.rb
-- bundler.d/inspect_anything.rb
-- bundler.d/x.local.rb
 - lib/smart_proxy_dynflow.rb
 - lib/smart_proxy_dynflow/action.rb
 - lib/smart_proxy_dynflow/action/batch.rb
@@ -164,11 +162,12 @@ files:
 - lib/smart_proxy_dynflow/core.rb
 - lib/smart_proxy_dynflow/helpers.rb
 - lib/smart_proxy_dynflow/http_config.ru
+- lib/smart_proxy_dynflow/io_buffer.rb
 - lib/smart_proxy_dynflow/log.rb
 - lib/smart_proxy_dynflow/middleware/keep_current_request_id.rb
 - lib/smart_proxy_dynflow/otp_manager.rb
 - lib/smart_proxy_dynflow/plugin.rb
-- lib/smart_proxy_dynflow/process_manager.rbs
+- lib/smart_proxy_dynflow/process_manager.rb
 - lib/smart_proxy_dynflow/proxy_adapter.rb
 - lib/smart_proxy_dynflow/runner.rb
 - lib/smart_proxy_dynflow/runner/base.rb
@@ -176,6 +175,7 @@ files:
 - lib/smart_proxy_dynflow/runner/command_runner.rb
 - lib/smart_proxy_dynflow/runner/dispatcher.rb
 - lib/smart_proxy_dynflow/runner/parent.rb
+- lib/smart_proxy_dynflow/runner/process_manager_command.rb
 - lib/smart_proxy_dynflow/runner/update.rb
 - lib/smart_proxy_dynflow/settings.rb
 - lib/smart_proxy_dynflow/settings_loader.rb
@@ -208,7 +208,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.2.26
 signing_key:
 specification_version: 4
 summary: Dynflow runtime for Foreman smart proxy

data/bundler.d/inspect_anything.rb

@@ -1 +0,0 @@
-# gem 'inspect_anything'

data/bundler.d/x.local.rb

@@ -1 +0,0 @@
-# gem 'dynflow', :path => '../dynflow'

data/lib/smart_proxy_dynflow/process_manager.rbs

@@ -1,46 +0,0 @@
-# TypeProf 0.21.2
-
-# Classes
-module Proxy
-  module Dynflow
-    class IOBuffer
-      @callback: nil
-
-      attr_accessor io: nil
-      attr_reader buffer: String
-      def initialize: (nil io) -> void
-      def on_data: -> nil
-      def to_io: -> IO
-      def to_s: -> String
-      def empty?: -> bool
-      def closed?: -> untyped
-      def close: -> nil
-      def read_available!: -> nil
-      def write_available!: -> nil
-      def add_data: (untyped data) -> String
-
-      private
-      def with_callback: (String? data) -> String?
-    end
-
-    class ProcessManager
-      @command: untyped
-
-      attr_reader stdin: IOBuffer
-      attr_reader stdout: IOBuffer
-      attr_reader stderr: IOBuffer
-      attr_reader pid: Integer
-      attr_reader status: Integer?
-      def initialize: (untyped command) -> void
-      def run!: -> ProcessManager
-      def start!: -> String
-      def started?: -> bool
-      def done?: -> bool
-      def close: -> [IOBuffer, IOBuffer, IOBuffer]
-      def process: (?timeout: nil) -> Array[untyped]?
-      def finish: -> Integer?
-      def on_stdout: -> nil
-      def on_stderr: -> nil
-    end
-  end
-end