andyw8-seeing_is_believing 4.2.0

data/lib/seeing_is_believing/evaluate_by_moving_files.rb

@@ -0,0 +1,195 @@
+# Not sure what the best way to evaluate these is
+# This approach will move the old file out of the way,
+# write the program in its place, invoke it, and move it back.
+#
+# Another option is to replace __FILE__ macros ourselves
+# and then write to a temp file but evaluate in the context
+# of the expected directory. Some issues could arise with this,
+# though: if you required the file again, it wouldn't already
+# be in the loaded features (might be able to just add it)
+# if you  did something like File.read(__FILE__) it would
+# read the wrong file... of course, since we rewrite the file,
+# its body will be incorrect, anyway.
+
+require 'rbconfig'
+require 'socket'
+require "childprocess"
+
+require 'seeing_is_believing/result'
+require 'seeing_is_believing/debugger'
+require 'seeing_is_believing/swap_files'
+require 'seeing_is_believing/event_stream/consumer'
+require 'seeing_is_believing/event_stream/events'
+
+# Forking locks up for some reason when we run SiB inside of SiB, so use `spawn`
+ChildProcess.posix_spawn = true
+
+# ChildProcess works on the M1 ("Apple Silicon"),
+# but it emits a bunch of logs that wind up back in the editors.
+# I opened an issue https://github.com/enkessler/childprocess/issues/176
+#
+# Nothing happened for 3 months, and I eventually switched it over to use `Kernel.spawn`
+# But that only worked on my local machine, and failed in CI on Oses I don't have access to
+# https://github.com/JoshCheek/seeing_is_believing/commit/f32929637625aece8ccb020c58470a5b4f659fbe
+#
+# Then Ryan patched it for me during a Seattle.rb meetup
+# https://github.com/enkessler/childprocess/pull/177
+#
+# But their `arch` detection code gets back different values between Ruby 2.7.2 and 2.7.3:
+# https://github.com/enkessler/childprocess/issues/179
+#
+# My gem has been broken on the M1 for... prob the entire life of the M1
+# * https://github.com/JoshCheek/seeing_is_believing/issues/160
+# * https://github.com/JoshCheek/seeing_is_believing/issues/161
+#
+# Sooooo... in order to unbreak it, I'm going to try to detect this situation
+# and guerilla patch Childprocess so that it doesn't depend on `RbConfig::CONFIG['host_cpu']`
+# we're essentially blowing away this method:
+# https://github.com/enkessler/childprocess/blob/v4.1.0/lib/childprocess.rb#L131-L150
+if RbConfig::CONFIG['host'] =~ /arm/ && RbConfig::CONFIG['host'] =~ /darwin/
+  def ChildProcess.arch
+    'arm64'
+  end
+end
+
+
+class SeeingIsBelieving
+  class EvaluateByMovingFiles
+    def self.call(*args)
+      new(*args).call
+    end
+
+    attr_accessor :user_program, :rewritten_program, :provided_input, :require_flags, :load_path_flags, :encoding, :timeout_seconds, :debugger, :event_handler, :max_line_captures
+
+    attr_accessor :file_directory, :file_path, :local_cwd, :relative_filename, :backup_path
+
+    def initialize(file_path, user_program, rewritten_program, options={})
+      options = options.dup
+      self.user_program      = user_program
+      self.rewritten_program = rewritten_program
+      self.encoding          = options.delete(:encoding)           || "u"
+      self.timeout_seconds   = options.delete(:timeout_seconds)    || 0 # 0 is the new infinity
+      self.provided_input    = options.delete(:provided_input)     || String.new
+      self.event_handler     = options.delete(:event_handler)      || raise(ArgumentError, "must provide an :event_handler")
+      self.load_path_flags   = (options.delete(:load_path_dirs)    || []).flat_map { |dir| ['-I', dir] }
+      self.require_flags     = (options.delete(:require_files)     || ['seeing_is_believing/the_matrix']).map { |filename| ['-r', filename] }.flatten
+      self.max_line_captures = (options.delete(:max_line_captures) || Float::INFINITY) # (optimization: child stops producing results at this number, even though it might make more sense for the consumer to stop emitting them)
+      self.local_cwd         = options.delete(:local_cwd)          || false
+      self.file_path         = file_path
+      self.file_directory    = File.dirname file_path
+      file_name              = File.basename file_path
+      self.relative_filename = local_cwd ? file_name : file_path
+      self.backup_path       = File.join file_directory, "seeing_is_believing_backup.#{file_name}"
+
+      options.any? && raise(ArgumentError, "Unknown options: #{options.inspect}")
+    end
+
+    def call
+      SwapFiles.call file_path, backup_path, user_program, rewritten_program do |swap_files|
+        evaluate_file swap_files
+      end
+    end
+
+    private
+
+    def evaluate_file(swap_files)
+      event_server = TCPServer.new(0) # dynamically allocates an available port
+
+      # setup streams
+      stdout, child_stdout = IO.pipe("utf-8")
+      stderr, child_stderr = IO.pipe("utf-8")
+
+      # setup environment variables
+      env = ENV.to_hash.merge 'SIB_VARIABLES.MARSHAL.B64' =>
+                                [Marshal.dump(
+                                  event_stream_port: event_server.addr[1],
+                                  max_line_captures: max_line_captures,
+                                  num_lines:         user_program.lines.count,
+                                  filename:          relative_filename,
+                                )].pack('m0')
+
+      child = ChildProcess.build(*popen_args)
+      child.cwd    = file_directory if local_cwd
+      child.leader = true
+      child.duplex = true
+      child.environment.merge!(env)
+      child.io.stdout = child_stdout
+      child.io.stderr = child_stderr
+
+      child.start
+
+      # close child streams b/c they won't emit EOF if parent still has an open reference
+      close_streams(child_stdout, child_stderr)
+      child.io.stdin.binmode
+      child.io.stdin.sync = true
+
+      # Start receiving events from the child
+      eventstream = event_server.accept
+
+      # send stdin (char at a time b/c input could come from a stream)
+      Thread.new do
+        begin
+          provided_input.each_char { |char| child.io.stdin.write char }
+        rescue
+          # don't explode if child closes IO
+        ensure
+          child.io.stdin.close
+        end
+      end
+
+      # set up the event consumer
+      consumer = EventStream::Consumer.new(events: eventstream, stdout: stdout, stderr: stderr)
+      consumer_thread = Thread.new do
+        consumer.each do |e|
+          swap_files.show_user_program if e.is_a? SeeingIsBelieving::EventStream::Events::FileLoaded
+          event_handler.call e
+        end
+      end
+
+      # wait for completion
+      if timeout_seconds == 0
+        child.wait
+      else
+        child.poll_for_exit(timeout_seconds)
+      end
+      consumer.process_exitstatus(child.exit_code)
+      consumer_thread.join
+    rescue ChildProcess::TimeoutError
+      consumer.process_timeout(timeout_seconds)
+      child.stop
+      consumer_thread.join
+    ensure
+      # On Windows, we need to call stop if there is an error since it interrupted
+      # the previos waiting/polling. If we don't call stop, in that situation, it will
+      # leave orphan processes. On Unix, we need to always call stop or it may leave orphans
+      begin
+        if ChildProcess.unix?
+          child.stop
+        elsif $!
+          child.stop
+          consumer.process_exitstatus(child.exit_code)
+        end
+        child.alive? && child.stop
+      rescue ChildProcess::Error
+        # On AppVeyor, I keep getting errors
+        #   The handle is invalid: https://ci.appveyor.com/project/JoshCheek/seeing-is-believing/build/22
+        #   Access is denied:      https://ci.appveyor.com/project/JoshCheek/seeing-is-believing/build/24
+      end
+      close_streams(stdout, stderr, eventstream, event_server)
+    end
+
+    def popen_args
+      [RbConfig.ruby,
+         '-W0',                                     # no warnings (b/c I hijack STDOUT/STDERR)
+         *(encoding ? ["-K#{encoding}"] : []),      # allow the encoding to be set
+         '-I', File.realpath('..', __dir__),        # add lib to the load path
+         *load_path_flags,                          # users can inject dirs to be added to the load path
+         *require_flags,                            # users can inject files to be required
+         relative_filename]
+    end
+
+    def close_streams(*streams)
+      streams.each { |io| io.close unless io.closed? }
+    end
+  end
+end

data/lib/seeing_is_believing/event_stream/consumer.rb

@@ -0,0 +1,221 @@
+# encoding: utf-8
+
+require 'seeing_is_believing/event_stream/events'
+require 'seeing_is_believing/error'
+require 'thread'
+
+# Polyfill String#scrub on Ruby 2.0.0
+require 'seeing_is_believing/compatibility'
+using SeeingIsBelieving::Compatibility
+
+class SeeingIsBelieving
+  module EventStream
+    class Consumer
+      # Contemplated doing FinishCriteria in binary, but the cost of doing it with an array
+      # like this is negligible and it has the nice advantage that the elements in the array
+      # are named # so if I ever look at it, I don't have to tranlsate a number to figure out
+      # the names https://gist.github.com/JoshCheek/10deb07277b6c85efc7b5e65c007785d
+      class FinishCriteria
+        EventThreadFinished  = Module.new
+        StdoutThreadFinished = Module.new
+        StderrThreadFinished = Module.new
+        ProcessExited        = Module.new
+
+        def initialize
+          @unmet_criteria = [
+            EventThreadFinished,
+            StdoutThreadFinished,
+            StderrThreadFinished,
+            ProcessExited,
+          ]
+        end
+
+        # finish criteria are satisfied,
+        # we can stop processing events
+        def satisfied?
+          @unmet_criteria.empty?
+        end
+
+        def event_thread_finished!
+          @unmet_criteria.delete EventThreadFinished
+        end
+
+        def stdout_thread_finished!
+          @unmet_criteria.delete StdoutThreadFinished
+        end
+
+        def stderr_thread_finished!
+          @unmet_criteria.delete StderrThreadFinished
+        end
+
+        def received_exitstatus!
+          @unmet_criteria.delete ProcessExited
+        end
+
+        def received_timeout!
+          @unmet_criteria.delete ProcessExited
+        end
+      end
+
+      # https://github.com/JoshCheek/seeing_is_believing/issues/46
+      def self.fix_encoding(str)
+        begin
+          str.encode! Encoding::UTF_8
+        rescue EncodingError
+          str = str.force_encoding(Encoding::UTF_8)
+        end
+        str.scrub('�')
+      end
+
+      def initialize(streams)
+        @finished            = false
+        self.finish_criteria = FinishCriteria.new
+        self.queue           = Queue.new
+        event_stream         = streams.fetch :events
+        stdout_stream        = streams.fetch :stdout
+        stderr_stream        = streams.fetch :stderr
+        self.threads         = [
+          Thread.new do
+            begin
+              stdout_stream.each_line { |line| queue << Events::Stdout.new(value: line) }
+              queue << Events::StdoutClosed.new(side: :producer)
+            rescue IOError
+              queue << Events::StdoutClosed.new(side: :consumer)
+            ensure
+              queue << lambda { finish_criteria.stdout_thread_finished! }
+            end
+          end,
+
+          Thread.new do
+            begin
+              stderr_stream.each_line { |line| queue << Events::Stderr.new(value: line) }
+              queue << Events::StderrClosed.new(side: :producer)
+            rescue IOError
+              queue << Events::StderrClosed.new(side: :consumer)
+            ensure
+              queue << lambda { finish_criteria.stderr_thread_finished! }
+            end
+          end,
+
+          Thread.new do
+            begin
+              event_stream.each_line { |line| queue << line }
+              queue << Events::EventStreamClosed.new(side: :producer)
+            rescue IOError
+              queue << Events::EventStreamClosed.new(side: :consumer)
+            ensure
+              queue << lambda { finish_criteria.event_thread_finished! }
+            end
+          end,
+        ]
+      end
+
+      def call(n=1)
+        return next_event if n == 1
+        Array.new(n) { next_event }
+      end
+
+      def each
+        return to_enum :each unless block_given?
+        yield call 1 until @finished
+      end
+
+      # NOTE: Note it's probably a bad plan to call these methods
+      # from within the same thread as the consumer, because if it
+      # blocks, who will remove items from the queue?
+      def process_exitstatus(status)
+        status ||= 1 # see #100
+        queue << lambda {
+          queue << Events::Exitstatus.new(value: status)
+          finish_criteria.received_exitstatus!
+        }
+      end
+      def process_timeout(seconds)
+        queue << lambda {
+          queue << Events::Timeout.new(seconds: seconds)
+          finish_criteria.received_timeout!
+        }
+      end
+
+      def join
+        threads.each(&:join)
+      end
+
+      private
+
+      attr_accessor :queue, :finish_criteria, :threads
+
+      def next_event
+        raise NoMoreEvents if @finished
+        case element = queue.shift
+        when String
+          event_for element
+        when Proc
+          element.call
+          finish_criteria.satisfied? &&
+            queue << Events::Finished.new
+          next_event
+        when Events::Finished
+          @finished = true
+          element
+        when Event
+          element
+        else
+          raise SeeingIsBelieving::UnknownEvent, "WAT IS THIS?: #{element.inspect}"
+        end
+      end
+
+      def shift_token(line)
+        event_name = line[/[^ ]+/]
+        line.sub!(/^\s*[^ ]+\s*/, '')
+        event_name
+      end
+
+      # For a consideration of many different ways of passing the message, see 5633064
+      def shift_string(line)
+        str = Marshal.load shift_token(line).unpack('m0').first
+        Consumer.fix_encoding(str)
+      end
+
+      def event_for(original_line)
+        line       = original_line.chomp
+        event_name = shift_token(line).intern
+        case event_name
+        when :result
+          line_number = shift_token(line).to_i
+          type        = shift_token(line).intern
+          inspected   = shift_string(line)
+          Events::LineResult.new(type: type, line_number: line_number, inspected: inspected)
+        when :maxed_result
+          line_number = shift_token(line).to_i
+          type        = shift_token(line).intern
+          Events::ResultsTruncated.new(type: type, line_number: line_number)
+        when :exception
+          Events::Exception.new \
+            line_number: shift_token(line).to_i,
+            class_name:  shift_string(line),
+            message:     shift_string(line),
+            backtrace:   shift_token(line).to_i.times.map { shift_string line }
+        when :max_line_captures
+          token = shift_token(line)
+          value = token =~ /infinity/i ? Float::INFINITY : token.to_i
+          Events::MaxLineCaptures.new(value: value)
+        when :num_lines
+          Events::NumLines.new(value: shift_token(line).to_i)
+        when :sib_version
+          Events::SiBVersion.new(value: shift_string(line))
+        when :ruby_version
+          Events::RubyVersion.new(value: shift_string(line))
+        when :filename
+          Events::Filename.new(value: shift_string(line))
+        when :file_loaded
+          Events::FileLoaded.new
+        when :exec
+          Events::Exec.new(args: shift_string(line))
+        else
+          raise UnknownEvent, original_line.inspect
+        end
+      end
+    end
+  end
+end

data/lib/seeing_is_believing/event_stream/events.rb

@@ -0,0 +1,193 @@
+require 'seeing_is_believing/hash_struct'
+
+class SeeingIsBelieving
+  module EventStream
+    Event = HashStruct.anon do # one superclass to rule them all!
+      def self.event_name
+        raise NotImplementedError, "Subclass should have defined this!"
+      end
+
+      def event_name
+        self.class.event_name
+      end
+
+      def as_json
+        [event_name, to_h]
+      end
+    end
+
+    module Events
+      # A line was printed to stdout.
+      class Stdout < Event
+        def self.event_name
+          :stdout
+        end
+        attributes :value
+      end
+
+      # A line was printed to stderr.
+      class Stderr < Event
+        def self.event_name
+          :stderr
+        end
+        attributes :value
+      end
+
+      # The program will not record more results than this for a line.
+      # Note that if this is hit, it will emit an unrecorded_result.
+      class MaxLineCaptures < Event
+        def self.event_name
+          :max_line_captures
+        end
+        def as_json
+          value, is_infinity = if self.value == Float::INFINITY
+                                 [-1, true]
+                               else
+                                 [self.value, false]
+                               end
+          [event_name, {value: value, is_infinity: is_infinity}]
+        end
+        attribute :value
+      end
+
+      # Name of the file being evaluated.
+      class Filename < Event
+        def self.event_name
+          :filename
+        end
+        attributes :value
+      end
+
+      # For knowing when you can perform certain tasks (eg moving the file back)
+      class FileLoaded < Event
+        def self.event_name
+          :file_loaded
+        end
+      end
+
+      # Number of lines in the program.
+      class NumLines < Event
+        def self.event_name
+          :num_lines
+        end
+        attributes :value
+      end
+
+      # Version of SeeingIsBelieving used to evaluate the code.
+      # Equivalent to `SeeingIsBelieving::VERSION`, and `seeing_is_believing --version`
+      class SiBVersion < Event
+        def self.event_name
+          :sib_version
+        end
+        attributes :value
+      end
+
+      # Version of Ruby being used to evaluate the code.
+      # Equivalent to `RUBY_VERSION`
+      class RubyVersion < Event
+        def self.event_name
+          :ruby_version
+        end
+        attributes :value
+      end
+
+      # The process' exitstatus.
+      class Exitstatus < Event
+        def self.event_name
+          :exitstatus
+        end
+        attributes :value
+      end
+
+      # The process timed out
+      # note that you will probably not receive an exitstatus
+      # if this occurs. Though it's hypothetically possible...
+      # this is all asynchronous.
+      class Timeout < Event
+        def self.event_name
+          :timeout
+        end
+        attributes :seconds
+      end
+
+      # Emitted when the process invokes exec.
+      # Note that this could be a child process,
+      # so it does not necessarily mean there won't be any more line results
+      class Exec < Event
+        def self.event_name
+          :exec
+        end
+        attributes :args
+      end
+
+      # A line was executed, and its result recorded.
+      # Currently, type will either be :inspect, or :pp
+      # :pp is used by AnnotateMarkedLines to facilitate xmpfilter style.
+      # If you're consuming the event stream, it's safe to assume type will always be :inspect
+      # If you're using the library, it's whatever you've recorded it as (if you haven't changed this, it's :inspect)
+      class LineResult < Event
+        def self.event_name
+          :line_result
+        end
+        attributes :type, :line_number, :inspected
+      end
+
+      # There were more results than we are emitting for this line / type of recording
+      # See LineResult for explanation of types
+      # This would occur because the line was executed more times than the max.
+      class ResultsTruncated < Event
+        def self.event_name
+          :results_truncated
+        end
+        attributes :type, :line_number
+      end
+
+      # The program raised an exception and did not catch it.
+      # Note that currently `ExitStatus` exceptions are not emitted.
+      # That could change at some point as it seems like the stream consumer
+      # should decide whether they care about that rather than the producer.
+      class Exception < Event
+        def self.event_name
+          :exception
+        end
+        attributes :line_number, :class_name, :message, :backtrace
+      end
+
+      # The process's stdout stream was closed, there will be no more Stdout events.
+      # "side" will either be :producer or :consumer
+      class StdoutClosed < Event
+        def self.event_name
+          :stdout_closed
+        end
+        attributes :side
+      end
+
+      # The process's stderr stream was closed, there will be no more Stderr events.
+      # "side" will either be :producer or :consumer
+      class StderrClosed < Event
+        def self.event_name
+          :stderr_closed
+        end
+        attributes :side
+      end
+
+      # The process's event stream was closed, there will be no more events that come via the stream.
+      # Currently, that's all events except Stdout, StdoutClosed, Stderr, StdoutClosed, ExitStatus, and Finished
+      class EventStreamClosed < Event
+        def self.event_name
+          :event_stream_closed
+        end
+        attributes :side
+      end
+
+      # All streams are closed and the exit status is known.
+      # There will be no more events.
+      class Finished < Event
+        def self.event_name
+          :finished
+        end
+        attributes []
+      end
+    end
+  end
+end

data/lib/seeing_is_believing/event_stream/handlers/debug.rb

@@ -0,0 +1,61 @@
+class SeeingIsBelieving
+  module EventStream
+    module Handlers
+      # Even though the debugger can be disabled, which would push the decision of
+      # whether to report or not into the debugger where it belongs, you should still
+      # avoid using this class if you don't need it since it is expensive and there
+      # could be tens of millions of events, eg https://github.com/JoshCheek/seeing_is_believing/issues/12
+      class Debug
+        def initialize(debugger, handler)
+          @debugger   = debugger
+          @handler    = handler
+          @seen       = ""
+          @line_width = 150 # debugger is basically for me, so giving it a nice wide width
+          @name_width = 20
+          @attr_width = @line_width - @name_width
+        end
+
+        def call(event)
+          observe event
+          finish if event.kind_of? Events::Finished
+          @handler.call event
+        end
+
+        private
+
+        attr_reader :debugger, :handler
+
+        def finish
+          @debugger.context("EVENTS") { @seen }
+        end
+
+        def observe(event)
+          name  = event.class.name.split("::").last
+          lines = event.to_h
+                       .map { |attribute, value|
+                          case attribute
+                          when :side      then "#{attribute}: #{value}"
+                          when :value     then value.to_s.chomp
+                          when :backtrace then indented = value.map { |v| "- #{v}" }
+                                               ["backtrace:", *indented]
+                          else "#{attribute}: #{value.inspect}"
+                          end
+                       }
+                       .flatten
+          joined = lines.join ", "
+          if joined.size < @attr_width
+            @seen << sprintf("%-#{@name_width}s%s\n", name, joined)
+          elsif lines.size == 1
+            @seen << sprintf("%-#{@name_width}s%s...\n", name, lines.first[0...@attr_width-3])
+          else
+            @seen << "#{name}\n"
+            lines.each { |line|
+              line = line[0...@line_width-5] << "..." if @line_width < line.length + 2
+              @seen << sprintf("| %s\n", line)
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/seeing_is_believing/event_stream/handlers/record_exit_events.rb

@@ -0,0 +1,26 @@
+require 'seeing_is_believing/event_stream/events'
+
+class SeeingIsBelieving
+  module EventStream
+    module Handlers
+      class RecordExitEvents
+        attr_reader :exitstatus
+        attr_reader :timeout_seconds
+
+        def initialize(next_handler)
+          @next_handler = next_handler
+        end
+
+        def call(event)
+          case event
+          when Events::Exitstatus
+            @exitstatus = event.value
+          when Events::Timeout
+            @timeout_seconds = event.seconds
+          end
+          @next_handler.call(event)
+        end
+      end
+    end
+  end
+end