qb 0.3.25 → 0.4.0

data/lib/qb/ipc/stdio/server.rb

@@ -0,0 +1,229 @@
+# encoding: UTF-8
+# frozen_string_literal: true
+
+# Requirements
+# =======================================================================
+
+# Stdlib
+# -----------------------------------------------------------------------
+
+# Used to generate a random ID to use in socket file names
+require 'securerandom'
+
+# Need {FileUtils.rm_rf}
+require 'fileutils'
+
+# Deps
+# -----------------------------------------------------------------------
+
+require 'nrser'
+
+# Project / Package
+# -----------------------------------------------------------------------
+
+
+
+# Declarations
+# ========================================================================
+
+module QB::IPC
+  module STDIO; end
+end
+
+
+# Definitions
+# =======================================================================
+
+# Server functionality to make the master QB process' STDIO streams available
+# to external processes, specifically Ansible modules.
+# 
+# Ansible's handling of STDIO in modules is really not suitable for our use
+# case - we want to see what modules and other external process commands
+# are doing in real time, much like invoking them in a Bash script.
+# 
+# This thing is **far** from perfect, but it's been incredibly helpful for a
+# simple solution.
+# 
+# Basically, {OutService} instances are created for `STDOUT` and `STDERR`,
+# which each create a {UNIXServer} on a local socket file and spawn a {Thread}
+# to listen to it. The socket's path is then made available to the Ansible
+# child process via ENV vars, and that process in turn carries those ENV vars
+# to it's module child processes, who can then use an instance of the
+# corresponding {QB::IPC::STDIO::Client} class to connect to those sockets and
+# write output that is passed through to the master QB process' output streams.
+# 
+# The protocol is simply text line-based, and modules - or any other process -
+# written in other languages can easily connect and write as well.
+# 
+# @note
+#   This feature only works for `localhost`. I have no idea what it will do
+#   in other cases. It doesn't seem like it should break anything, but remotely
+#   executing modules definitely won't be able to connect to the sockets on
+#   the host.
+# 
+# @todo
+#   There is also a {InService} for `STDIN`, but it's is pretty experimental /
+#   broken at this point. That would be nice to fix in the future so that
+#   programs that make use of user interaction work seamlessly through QB.
+#   
+#   This will probably require using pseudo-TTY streams or whatever mess.
+# 
+class QB::IPC::STDIO::Server
+  
+  # Sub-Tree Requirements
+  # ========================================================================
+  
+  require_relative './server/in_service'
+  require_relative './server/out_service'
+  require_relative './server/log_service'
+  
+  
+  # Mixins
+  # ========================================================================
+  
+  # Add {.logger} and {#logger} methods
+  include NRSER::Log::Mixin
+  
+  
+  # Class Methods
+  # ========================================================================
+  
+  # Clean up resources for an instance. Broken out because I was trying to
+  # make it run as a finalizer to remove the directory in all cases, but that
+  # does not seem to be triggering. Whatever man...
+  # 
+  # @param [Fixnum] object_id:
+  #   The instance's `#object_id`, just for logging purposes.
+  # 
+  # @param [Array<Service>]
+  #   The instance's services, which we will {Service#close!}.
+  # 
+  # @param [Pathname] socket_dir:
+  #   The tmpdir created for the sockets, which we will remove.
+  # 
+  # @return [nil]
+  # 
+  def self.clean_up_for object_id:, services:, socket_dir:
+    logger.debug "Cleaning up...",
+      object_id: object_id,
+      socket_dir: socket_dir
+    
+    services.each do |service|
+      logger.catch.warn(
+        "Unable to close service",
+        service: service,
+      ) { service.close! }
+    end
+        
+    FileUtils.rm_rf( socket_dir ) if socket_dir.exist?
+    
+    logger.debug "Clean!",
+      object_id: object_id,
+      socket_dir: socket_dir
+    
+    nil
+  end # .finalize
+  
+  
+  # Make a {Proc} to use for finalization.
+  # 
+  # Needs to be done outside instance scope to doesn't close over the
+  # instance.
+  # 
+  # @param **kwds
+  #   Passed to {.clean_up_for}.
+  # 
+  # @return [Proc<() => nil>]
+  #   @todo Document return value.
+  # 
+  def self.finalizer_for **kwds
+    -> {
+      logger.debug "Finalizing...", **kwds
+      clean_up_for **kwds
+      logger.debug "Finalized", **kwds
+    }
+  end # .finalizer_for
+  
+  
+  # Attributes
+  # ========================================================================
+  
+  # Where the UNIX socket files get put.
+  # 
+  # @return [Pathname]
+  #     
+  attr_reader :socket_dir
+  
+  
+  # Construction
+  # ========================================================================
+  
+  # Instantiate a new `QB::IPC::STDIO::Server`.
+  # 
+  def initialize
+    @socket_dir = Dir.mktmpdir( 'qb_ipc_stdio' ).to_pn
+    
+    @in_service   = QB::IPC::STDIO::Server::InService.new \
+                      name: :in,
+                      socket_dir: socket_dir,
+                      src: $stdin
+                      
+    @out_service  = QB::IPC::STDIO::Server::OutService.new \
+                      name: :out,
+                      socket_dir: socket_dir,
+                      dest: $stdout
+                      
+    @err_service  = QB::IPC::STDIO::Server::OutService.new \
+                      name: :err,
+                      socket_dir: socket_dir,
+                      dest: $stderr
+    
+    @log_service  = QB::IPC::STDIO::Server::LogService.new \
+                      name: :log,
+                      socket_dir: socket_dir
+                      
+    ObjectSpace.define_finalizer \
+      self,
+      self.class.finalizer_for(
+        object_id: object_id,
+        services: services,
+        socket_dir: socket_dir
+      )
+  end # #initialize
+  
+  
+  # Instance Methods
+  # ========================================================================
+  
+  # @return [Array<(InService, OutService, OutService)>]
+  #   Array of in, out and err services.
+  # 
+  def services
+    [ @in_service, @out_service, @err_service, @log_service ]
+  end # #services
+  
+  
+  # Start all the {#services} by calling {Service#open!} on them.
+  # 
+  # @return [self]
+  # 
+  def start!
+    services.each &:open!
+    self
+  end # #open!
+  
+  
+  # Stop all {#services} by calling {Service#close!} on them and clean up
+  # the resources.
+  # 
+  # @return [self]]
+  # 
+  def stop!
+    self.class.clean_up_for \
+      object_id: object_id,
+      services: services,
+      socket_dir: socket_dir
+    self
+  end
+  
+end # class QB::IPC::STDIO::Server

data/lib/qb/ipc/stdio/server/in_service.rb

@@ -0,0 +1,18 @@
+require_relative './service'
+
+# QB STDIO Service to proxy interactive user input from the main process
+# to modules.
+class QB::IPC::STDIO::Server::InService < QB::IPC::STDIO::Server::Service
+  def initialize name:, socket_dir:, src:
+    super name: name, socket_dir: socket_dir
+    @src = src
+  end
+  
+  def work_in_thread
+    while (line = @src.gets) do
+      @socket.puts line
+    end
+    
+    close!
+  end
+end # InService

data/lib/qb/ipc/stdio/server/log_service.rb

@@ -0,0 +1,168 @@
+require 'nrser/core_ext/hash'
+require_relative './service'
+
+# QB STDIO Service to receive log lines in JSON format and forward them
+# on to the logger.
+# 
+class QB::IPC::STDIO::Server::LogService < QB::IPC::STDIO::Server::Service
+  
+  class Log < SemanticLogger::Log
+    
+    
+    # Construction
+    # ========================================================================
+    
+    def initialize **kwds
+      super *kwds.values_at( :name, :level, :level_index )
+      
+      if kwds.key? :timestamp
+        self.time = Time.parse kwds[:timestamp]
+      end
+      
+      self.tags = kwds[:tags] || []
+      self.named_tags = kwds[:named_tags] || {}
+      
+      @pid = kwds[:pid] || '???'
+      @thread = kwds[:thread] || '???'
+      
+      exception = if kwds.key? :exception
+        klass = kwds[:exception]["name"].safe_constantize
+        
+        if klass
+          # HACK  Good God...
+          #       
+          #       What we're doing is constructing an instance of the
+          #       exception class so that SemLog is happy with it... so we
+          #       take the class name, load that constant, then we *don't*
+          #       create an instance, because that could require args, and
+          #       all we need is something that holds the message and
+          #       backtrace, so we add the message as the response from
+          #       dynamically-created `#to_s` and `#message` methods added
+          #       *to that instance only*. Then we set the backtrace using
+          #       the regular instance API.
+          #       
+          #       ...and it kinda seems to work. But I suspect it will fuck
+          #       me/us/someone at some point if left like this...
+          #       
+          
+          message = kwds[:exception]["message"] || '(none)'
+          
+          error = klass.allocate
+          
+          metaclass = class << error; self; end
+          
+          [:to_s, :message].each do |name|
+            metaclass.send( :define_method, name ){ message }
+          end
+          
+          if kwds[:exception]["stack_trace"]
+            error.set_backtrace kwds[:exception]["stack_trace"]
+          end
+          
+          error
+        end
+      end
+      
+      assign exception: exception, **kwds.slice(
+        :message,
+        :payload,
+        :min_duration,
+        :metric,
+        :metric_amount,
+        :duration,
+        # :backtrace,
+        # :log_exception,
+        # :on_exception_level,
+        :dimension,
+      )
+    end
+        
+    # Instance Methods
+    # ========================================================================
+    
+    def process_info thread_name_length = 30
+      "IPC:#{ @pid }:#{"%.#{ thread_name_length }s" % @thread}"
+    end
+    
+    
+  end # class Log
+  
+  
+  def initialize name:, socket_dir:
+    super name: name, socket_dir: socket_dir
+    @loggers = {}
+  end
+  
+  def work_in_thread
+    while (line = @socket.gets) do
+      logger.trace "received line",
+        line: line
+      
+      load_log_in_thread line
+    end
+  end
+  
+  
+  protected
+  # ========================================================================
+    
+    # Get a {NRSER::Log::Logger} for a log name, creating them on demand
+    # and caching after that.
+    # 
+    # @param [String] name
+    #   Name from the log message.
+    # 
+    # @return [NRSER::Log::Logger]
+    # 
+    def logger_for name
+      @loggers[name] ||= NRSER::Log[name]
+    end
+    
+    
+    # Log a {Log} in it's logger if it should log.
+    # 
+    # @protected
+    # 
+    # @param [Log] log
+    #   Log instance to dispatch
+    # 
+    # @return [void]
+    # 
+    def write_log log
+      logger = logger_for log.name
+      # logger.level = :trace
+      logger.log( log ) if logger.should_log?( log )
+    end
+    
+    
+    # Try to load the line into a {SemanticLogger::Log} instance.
+    # 
+    def load_log_in_thread line
+      # logger.with_level :trace do
+        decoded = logger.catch.warn(
+          "Unable to decode log message",
+          line: line,
+        ) { ActiveSupport::JSON.decode line }
+        
+        logger.trace "Decoded log message", decoded
+        
+        return nil unless decoded
+      
+        logger.catch.warn(
+          "Unable to process log message",
+          line: line,
+          decoded: decoded,
+        ) do
+          log = Log.new **decoded.to_options
+          
+          logger.trace "Constructed {Log}",
+            log: log
+          
+          write_log log
+        end # logger.catch.warn
+      # end # logger.with_level :trace
+    end
+  
+  public # end protected ***************************************************
+  
+end # LogService

data/lib/qb/ipc/stdio/server/out_service.rb

@@ -0,0 +1,20 @@
+require_relative './service'
+  
+# QB STDIO Service to proxy output from modules back to the main user
+# process.
+class QB::IPC::STDIO::Server::OutService < QB::IPC::STDIO::Server::Service
+  def initialize name:, socket_dir:, dest:
+    super name: name, socket_dir: socket_dir
+    @dest = dest
+  end
+  
+  def work_in_thread
+    while (line = @socket.gets) do
+      logger.trace "received line",
+        line: line,
+        dest: @dest
+      
+      @dest.puts line
+    end
+  end
+end # InService

data/lib/qb/ipc/stdio/server/service.rb

@@ -0,0 +1,229 @@
+# Requirements
+# =====================================================================
+
+# Stdlib
+# ----------------------------------------------------------------------------
+
+require 'thread'
+require 'socket'
+require 'fileutils'
+
+# Deps
+# ----------------------------------------------------------------------------
+
+require 'nrser'
+
+# Project
+# ----------------------------------------------------------------------------
+
+# Need {QB::IPC::STDIO.path_env_var_name}
+require 'qb/ipc/stdio'
+
+
+# Definitions
+# =====================================================================
+  
+# STDIO as a service exposed on a UNIX socket so that modules can stream
+# their output to it, which is in turn printed to the console `qb` is running
+# in.
+class QB::IPC::STDIO::Server::Service
+  
+  # Mixins
+  # ========================================================================
+  
+  # Add {.logger} and {#logger}
+  include NRSER::Log::Mixin
+  
+  
+  # Attributes
+  # ========================================================================
+  
+  # The service's name, like `:in`, `:out`, `:err`.
+  # 
+  # @return [Synbol]
+  #     
+  attr_reader :name
+  
+  
+  # Absolute path to socket file.
+  # 
+  # @return [Pathname]
+  #     
+  attr_reader :path
+  
+  
+  # TODO document `thread` attribute.
+  # 
+  # @return [attr_type]
+  #     
+  attr_reader :thread
+  
+  
+  # TODO document `env_var_name` attribute.
+  # 
+  # @return [String]
+  #     
+  attr_reader :env_var_name
+  
+  
+  # The UNIX socket server.
+  # 
+  # @return [UNIXServer?]
+  #     
+  attr_reader :server
+  
+  
+  # The socket we accept from the server.
+  # 
+  # @return [UNIXSocket]
+  #     
+  attr_reader :socket
+  
+  
+  # Construction
+  # ========================================================================
+  
+  # Construct an IO service.
+  # 
+  # @param [Symbol] name
+  #   What this service is for... `:in`, `:out`, `:err`...
+  #   
+  #   Used as the thread name.
+  # 
+  def initialize name:, socket_dir:
+    @name = name
+    @thread = nil
+    @server = nil
+    @socket = nil
+    @env_var_name = QB::IPC::STDIO.path_env_var_name name
+    
+    @path = socket_dir.join "#{ name }.sock"
+    
+    self.logger = create_logger
+    
+    logger.debug "Initialized"
+  end
+  
+  
+  protected
+  # ========================================================================
+    
+    # Initialize the {#logger}.
+    # 
+    # @protected
+    # @return [nil]
+    # 
+    def create_logger
+      logger = NRSER::Log[ self ]
+      
+      # HACK
+      # 
+      # Tracing the IO is *really* noisy and spaghettis up the log output
+      # due to the threaded nature of the this beast... which is what you
+      # *want* if you're debugging main/module process IO, since it shows
+      # you what's happening synchronously, but that's pretty much all you
+      # can debug when it's being output.
+      # 
+      # The `debug`-level output is
+      # 
+      # For that reason, I quickly threw
+      #  
+      if ENV['QB_TRACE_STDIO'].truthy?
+        logger.level = :trace
+      elsif ENV['QB_DEBUG_STDIO'].truthy?
+        logger.level = :debug
+      else
+        logger.level = :info
+      end
+      
+      logger
+    end
+    
+  public # end private *****************************************************
+  
+  
+  # Instance Methods
+  # ========================================================================
+  
+  # @return [String]
+  #   a short string describing the instance. Used to set the name for
+  #   instance loggers.
+  def to_s
+    "#<#{ self.class.name } name=#{ name.inspect } path=#{ path.to_s }>"
+  end # #to_s
+  
+  
+  def open!
+    logger.debug "Opening..."
+    
+    # make sure env var is not already set (basically just prevents you from
+    # accidentally opening two instances with the same name)
+    if ENV.key? env_var_name
+      raise "env already contains key #{ env_var_name }" \
+            "with value #{ ENV[env_var_name] }"
+    end
+    
+    @thread = Thread.new do
+      Thread.current.name = name
+      logger.trace "thread started."
+      
+      @server = UNIXServer.new path.to_s
+      
+      while true do
+        @socket = server.accept
+        work_in_thread
+      end
+    end
+    
+    # set the env key so children can find the socket path
+    ENV[env_var_name] = path.to_s
+    logger.debug "Set env var",
+      env_var_name => ENV[env_var_name]
+    
+    logger.debug "Service open."
+  end # open
+  
+  
+  # We're done here, clean up!
+  # 
+  # @todo
+  #   Not sure how correct this is... fucking threading. *Seems* to work...
+  # 
+  # @return [nil]
+  # 
+  def close!
+    logger.debug "Closing...",
+      socket: socket,
+      server: server,
+      path_exists: path.exist?,
+      thread: thread,
+      env_var: {
+        env_var_name => ENV[env_var_name],
+      }
+    
+    # Remove the path from the ENV so if we do anything after this the
+    # old one isn't hanging around
+    ENV.delete env_var_name
+    
+    # Kill the thread first so that it can't try to do anything else
+    thread.kill if thread && thread.alive?
+    
+    socket.close unless socket.nil?
+    @socket = nil
+    server.close unless server.nil?
+    @server = nil
+    FileUtils.rm( path ) if path.exist?
+    
+    logger.debug "Closed.",
+      socket: socket,
+      server: server,
+      path_exists: path.exist?,
+      thread: thread,
+      env_var: {
+        env_var_name => ENV[env_var_name],
+      }
+    
+    nil
+  end # #close!
+  
+end # QB::IPC::STDIO::Server::Service