ractor-shim 0.0.1 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5132b882154ee4c0af587fb5e4b2f10030ef9834237abbbae5417e3fb51b62a8
-  data.tar.gz: 0ed22147461e8c6d3feae42d18fd06247ed6d8df800dc6fb2211e56ba9f807dc
+  metadata.gz: 8bb91e5a7525967249c155d6193ebacf3b261d6db8c93c06e904aabc11810c79
+  data.tar.gz: 81882745a9a7393a06a1916c19b5324272ff5001a9883c61ea4d7d2fadb032c2
 SHA512:
-  metadata.gz: bbabf61d3ee523150b793e177a2c60d14cad3ed94848a2290bd9d02f86175f251a641a24cb1841eb01a749e14b7dcbbb94fdcac1c8f23cc1b927a3ac3d533b4e
-  data.tar.gz: 9264645ee001fbb8f301163a28f88e1e3431639312ad4c249060a75d80f45d69afaae5b7504a3b3c6175eac7325d204a31e1a78a264c7d50a47052be4b7a18b7
+  metadata.gz: 80866fae0411eb23c5e1900430763c7d1ce552f0704c911e9c65a7a7b518959f3ee85d2ce0836f4ea33291c37e0061ffa8266086ad2f78d1fc5509991729690a
+  data.tar.gz: 42fae31b67236b12e74f84b4d4942e34fe7a4f73fa9280fa971733e30c16edef9a60a8a29312aca99fcac9374050a328d86c0148121c1fffe0b9a5dfd0249477

data/README.md

@@ -12,7 +12,7 @@ require 'ractor/shim'
 
 ## Development
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `ractor-shim.gemspec`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
 
 ## Contributing
 

data/Rakefile

@@ -1,4 +1,8 @@
 # frozen_string_literal: true
 
 require "bundler/gem_tasks"
-task default: %i[]
+task default: [:test]
+
+task :test do
+  exec RbConfig.ruby, "-Ilib", "test/run_tests.rb"
+end

data/lib/ractor/shim.rb

@@ -1,3 +1,385 @@
 # frozen_string_literal: true
 
-require_relative "shim/version"
+builtin_ractor = !!defined?(Ractor)
+class Ractor
+end
+Ractor.define_singleton_method(:builtin?) { builtin_ractor }
+Ractor.define_singleton_method(:shim?) { !builtin_ractor }
+
+if Ractor.shim?
+  class Ractor
+    class Error < RuntimeError
+    end
+
+    class RemoteError < Error
+      attr_reader :ractor
+      def initialize(ractor)
+        @ractor = ractor
+      end
+    end
+
+    class ClosedError < StopIteration
+    end
+
+    @count = 1
+    COUNT_MUTEX = Mutex.new
+    CHANGE_COUNT = -> delta {
+      COUNT_MUTEX.synchronize { @count += delta }
+    }
+
+    @id = 0
+    ID_MUTEX = Mutex.new
+    GET_ID = -> {
+      ID_MUTEX.synchronize { @id += 1 }
+    }
+
+    SELECT_MUTEX = Mutex.new
+    SELECT_CV = ConditionVariable.new
+
+    def self.main
+      MAIN_RACTOR
+    end
+
+    def self.main?
+      current == main
+    end
+
+    def self.current
+      Thread.current.thread_variable_get(:current_ractor) or raise "Could not find current Ractor"
+    end
+
+    def self.count
+      COUNT_MUTEX.synchronize { @count }
+    end
+
+    def self.receive
+      Ractor.current.__send__(:receive)
+    end
+
+    # def self.select_simple_spinning(*ractors)
+    #   raise ArgumentError, "specify at least one ractor or `yield_value`" if ractors.empty?
+    #   while true
+    #     ractors.each do |ractor_or_port|
+    #       if Ractor === ractor_or_port
+    #         queue = ractor_or_port.out_queue
+    #       elsif Ractor::Port === ractor_or_port
+    #         queue = ractor_or_port.queue
+    #       else
+    #         raise ArgumentError, "Unexpected argument for Ractor.select: #{ractor_or_port}"
+    #       end
+    #
+    #       begin
+    #         value = queue.pop(true)
+    #         return [ractor_or_port, value]
+    #       rescue ThreadError
+    #         Thread.pass
+    #       end
+    #     end
+    #   end
+    # end
+
+    def self.select(*ractors)
+      raise ArgumentError, "specify at least one ractor or `yield_value`" if ractors.empty?
+
+      SELECT_MUTEX.synchronize do
+        while true
+          ractors.each do |ractor_or_port|
+            if Ractor === ractor_or_port
+              queue = ractor_or_port.out_queue
+            elsif Ractor::Port === ractor_or_port
+              queue = ractor_or_port.queue
+            else
+              raise ArgumentError, "Unexpected argument for Ractor.select: #{ractor_or_port}"
+            end
+
+            begin
+              value = queue.pop(true)
+              return [ractor_or_port, value]
+            rescue ThreadError
+              # keep looping
+            end
+          end
+
+          # Wait until an item is added to a relevant Queue
+          SELECT_CV.wait(SELECT_MUTEX)
+        end
+      end
+    end
+
+    def self.make_shareable(object, copy: false)
+      # no copy, just return the object
+      object
+    end
+
+    def self.shareable?(object)
+      true
+    end
+
+    def self.store_if_absent(var, &block)
+      Ractor.current.__send__(:store_if_absent, var, &block)
+    end
+
+    def self._require(feature)
+      Kernel.require(feature)
+    end
+
+    attr_reader :name
+
+    attr_reader :in_queue, :out_queue
+
+    def initialize(*args, name: nil, &block)
+      raise ArgumentError, "must be called with a block" unless block
+      initialize_common(name, block)
+
+      @thread = Thread.new {
+        Thread.current.thread_variable_set(:current_ractor, self)
+        CHANGE_COUNT.call(1)
+        begin
+          result = self.instance_exec(*args, &block)
+          SELECT_MUTEX.synchronize {
+            unless @out_queue.closed?
+              @out_queue << result
+              SELECT_CV.broadcast
+            end
+          }
+          result
+        rescue Exception => e
+          @exception = e
+          nil
+        ensure
+          @termination_mutex.synchronize {
+            CHANGE_COUNT.call(-1)
+            @status = :terminated
+
+            monitor_message = @exception ? :aborted : :exited
+            @monitors.each { |monitor|
+              monitor << monitor_message
+            }
+            @monitors.clear
+          }
+        end
+      }
+    end
+
+    private def initialize_main
+      initialize_common(nil, nil)
+      @thread = nil
+    end
+
+    private def initialize_common(name, block)
+      @name = name.nil? ? nil : (String.try_convert(name) or raise TypeError)
+      @id = GET_ID.call
+      @status = :running
+      @from = block ? block.source_location.join(":") : nil
+      @in_queue = Queue.new
+      @out_queue = Queue.new
+      @storage = {}
+      @exception = nil
+      @monitors = []
+      @termination_mutex = Mutex.new
+    end
+
+    def [](var)
+      raise "Cannot get ractor local storage for non-current ractor" unless Ractor.current == self
+      @storage[var]
+    end
+
+    def []=(var, value)
+      raise "Cannot set ractor local storage for non-current ractor" unless Ractor.current == self
+      @storage[var] = value
+    end
+
+    private def store_if_absent(var, &block)
+      if value = @storage[var]
+        value
+      else
+        value = block.call
+        @storage[var] = value
+        value
+      end
+    end
+
+    def send(message, move: false)
+      raise Ractor::ClosedError, "The port was already closed" if @status == :terminated || @in_queue.closed?
+      @in_queue << message
+      self
+    end
+    alias_method :<<, :send
+
+    private def receive
+      @in_queue.pop
+    end
+
+    # def take
+    #   @out_queue.pop
+    # end
+
+    def close_incoming
+      @in_queue.close
+      self
+    end
+
+    def close_outgoing
+      SELECT_MUTEX.synchronize {
+        @out_queue.close
+        SELECT_CV.broadcast
+      }
+      self
+    end
+
+    def monitor(port)
+      @termination_mutex.synchronize {
+        if @status == :terminated
+          port << (@exception ? :aborted : :exited)
+          false
+        else
+          @monitors << port
+        end
+      }
+    end
+
+    def unmonitor(port)
+      @termination_mutex.synchronize {
+        @monitors.delete port
+      }
+    end
+
+    def join
+      value
+      self
+    end
+
+    def value
+      @thread.join
+
+      if exc = @exception
+        remote_error = RemoteError.new(self)
+        raise remote_error, cause: exc
+      end
+
+      @thread.value
+    end
+
+    def inspect
+      ["#<Ractor:##{@id}", @name, @from, "#{@status}>"].compact.join(' ')
+    end
+
+    MAIN_RACTOR = Ractor.allocate
+    MAIN_RACTOR.__send__(:initialize_main)
+    Thread.main.thread_variable_set(:current_ractor, MAIN_RACTOR)
+  end
+end
+
+if Ractor.builtin?
+  class Ractor
+    unless method_defined?(:join)
+      alias_method :join, :take
+    end
+
+    unless method_defined?(:value)
+      alias_method :value, :take
+    end
+
+    unless respond_to?(:main?)
+      def self.main?
+        self == Ractor.main
+      end
+    end
+  end
+end
+
+# common
+class Ractor
+  unless method_defined?(:close)
+    def close
+      close_incoming
+      close_outgoing
+    end
+  end
+end
+
+# Ractor.{shareable_proc,shareable_lambda}
+
+class << Ractor
+  if Ractor.builtin?
+    unless method_defined?(:shareable_proc)
+      def shareable_proc(&b)
+        Ractor.make_shareable(b)
+      end
+    end
+
+    unless method_defined?(:shareable_lambda)
+      def shareable_lambda(&b)
+        Ractor.make_shareable(b)
+      end
+    end
+  else
+    unless method_defined?(:shareable_proc)
+      alias_method :shareable_proc, :proc
+      public :shareable_proc
+    end
+
+    unless method_defined?(:shareable_lambda)
+      alias_method :shareable_lambda, :lambda
+      public :shareable_lambda
+    end
+  end
+end
+
+# Ractor::Port
+
+if Ractor.builtin?
+  class Ractor::Port
+    QUIT = Object.new.freeze
+
+    def initialize
+      @pipe = Ractor.new do
+        while true
+          msg = Ractor.receive
+          break if QUIT.equal?(msg)
+          Ractor.yield msg
+        end
+      end
+    end
+
+    def <<(message)
+      @pipe.send(message)
+    end
+
+    def receive
+      @pipe.take
+    end
+
+    def close
+      @pipe.send(QUIT)
+    end
+  end unless defined?(Ractor::Port)
+else
+  class Ractor::Port
+    attr_reader :queue
+
+    def initialize
+      @queue = Queue.new
+    end
+
+    def <<(message)
+      Ractor::SELECT_MUTEX.synchronize {
+        @queue << message
+        Ractor::SELECT_CV.broadcast
+      }
+      self
+    end
+
+    def receive
+      @queue.pop
+    end
+
+    def close
+      Ractor::SELECT_MUTEX.synchronize {
+        @queue.close
+        Ractor::SELECT_CV.broadcast
+      }
+      self
+    end
+  end
+end

data/test/run_tests.rb

@@ -0,0 +1,235 @@
+# frozen_string_literal: true
+
+require 'ractor/shim'
+
+BUILTIN_SKIPS = [
+  "send shareable and unshareable objects", # leaks Ractors on 3.3
+  "Ractor::IsolationError cases", # Ractor.make_shareable too permissive on 3.4
+  "Ractor.select from two ractors.", # Ractor.select with ports doesn't work on 3.4
+  "SystemExit from a Ractor is re-raised", # Ractor::ClosedError on 3.4 instead of Ractor::RemoteError
+  "SystemExit from a Thread inside a Ractor is re-raised", # Ractor::ClosedError on 3.4 instead of Ractor::RemoteError
+  "Access to global-variables are prohibited (read)", # different error message on 3.4
+  "Access to global-variables are prohibited (write)", # different error message on 3.4
+  "Ractor.make_shareable(a_proc) is not supported now.", # no error on 3.4
+  "Ractor-local storage", # more permissive on 3.4
+  "Now NoMethodError is copyable", # fails on 3.4
+  "bind_call in Ractor [Bug #20934]", # SEGV on 3.4
+  "moved objects being corrupted if embeded (Hash)", # broken on 3.4
+  "moved objects being corrupted if embeded (MatchData)", # SEGV on 3.4
+  "moved objects being corrupted if embeded (Struct)", # broken on 3.4
+  "moved objects being corrupted if embeded (Object)", # broken on 3.4
+  "moved arrays can't be used", # broken on 3.4
+  "moved strings can't be used", # broken on 3.4
+  "moved hashes can't be used", # broken on 3.4
+  "move objects inside frozen containers", # broken on 3.4
+  "moved composite types move their non-shareable parts properly", # broken on 3.4
+  "Creating classes inside of Ractors", # OOM on 3.4
+  "Ractor#join raises RemoteError when the remote Ractor aborted with an exception", # needs join semantics for exceptions
+  "Only one Ractor can call Ractor#value", # need value semantics
+  "monitor port returns `:exited` when the monitering Ractor terminated.", # Ractor#monitor
+  "monitor port returns `:exited` even if the monitoring Ractor was terminated.", # Ractor#monitor
+  "monitor returns false if the monitoring Ractor was terminated.", # Ractor#monitor
+  "monitor port returns `:aborted` when the monitering Ractor is aborted.", # Ractor#monitor
+  "monitor port returns `:aborted` even if the monitoring Ractor was aborted.", # Ractor#monitor
+]
+
+SHIM_SKIPS = [
+  "Ractor.allocate is not supported",
+  "Ractor::IsolationError cases",
+  "$DEBUG, $VERBOSE are Ractor local", # mutates global variables like $DEBUG and $VERBOSE
+  "SystemExit from a Thread inside a Ractor is re-raised", # internal error on TruffleRuby
+  "unshareable object are copied", # expected due to no copy
+  "To copy the object, now Marshal#dump is used", # expected due to no copy
+  "send shareable and unshareable objects", # expected due to no copy
+  "frozen Objects are shareable", # expected due to no copy
+  "touching moved object causes an error", # expected due to no move
+  "move example2: Array", # expected due to no move
+  "Access to global-variables are prohibited (read)", # can't check
+  "Access to global-variables are prohibited (write)", # can't check
+  "$stdin,out,err is Ractor local, but shared fds", # can't do
+  "given block Proc will be isolated, so can not access outer variables.", # can't check
+  "ivar in shareable-objects are not allowed to access from non-main Ractor", # not checked
+  "ivar in shareable-objects are not allowed to access from non-main Ractor, by @iv (get)", # not checked
+  "ivar in shareable-objects are not allowed to access from non-main Ractor, by @iv (set)", # not checked
+  "and instance variables of classes/modules are accessible if they refer shareable objects", # not checked
+  "cvar in shareable-objects are not allowed to access from non-main Ractor", # can't check
+  "also cached cvar in shareable-objects are not allowed to access from non-main Ractor", # can't check
+  "Getting non-shareable objects via constants by other Ractors is not allowed", # can't check
+  "Constant cache should care about non-shareable constants", # can't check
+  "Setting non-shareable objects into constants by other Ractors is not allowed", # can't check
+  "define_method is not allowed", # can't check
+  "ObjectSpace._id2ref can not handle unshareable objects with Ractors", # can't check
+  "Ractor.make_shareable(obj)", # expected due to no freeze
+  "Ractor.make_shareable(obj) doesn't freeze shareable objects", # expected due to no freeze
+  "Ractor.make_shareable(a_proc) is not supported now.", # can't check
+  "Ractor.shareable?(recursive_objects)", # expected, due to all are shareable
+  "Ractor.make_shareable(recursive_objects)", # expected, due to all are shareable
+  "Ractor.make_shareable(obj, copy: true) makes copied shareable object.", # expected, due to all are shareable
+  "Can not trap with not isolated Proc on non-main ractor", # can't check
+  "Ractor-local storage with Thread inheritance of current Ractor", # hard
+  "Chilled strings are not shareable", # expected, due to all are shareable
+  "moved arrays can't be used", # expected due to no move
+  "moved strings can't be used", # expected due to no move
+  "moved hashes can't be used", # expected due to no move
+  "moved composite types move their non-shareable parts properly", # expected due to no move
+  "Only one Ractor can call Ractor#value", # could be implemented
+]
+
+GLOBAL_SKIPS = [
+  "threads in a ractor will killed", # leaks Ractors
+  "TracePoint with normal Proc should be Ractor local", # line numbers differ due to test harness
+  "Can yield back values while GC is sweeping [Bug #18117]", # too slow and leaks
+  "check experimental warning", # harness disables experimental warnings
+  "failed in autolaod in Ractor", # need more isolation
+  "fork after creating Ractor", # fork, leaks
+  "Ractors should be terminated after fork", # fork, hangs
+  "st_table, which can call malloc.", # leaks
+  "Creating classes inside of Ractors", # leaks
+]
+
+skips = GLOBAL_SKIPS
+skips += BUILTIN_SKIPS if Ractor.builtin? && RUBY_VERSION < "3.5"
+skips += SHIM_SKIPS if Ractor.shim?
+if Ractor.builtin? && RUBY_VERSION < "3.4"
+  # fails on 3.3
+  skips += [
+    "unshareable frozen objects should still be frozen in new ractor after move",
+    "ivar in shareable-objects are not allowed to access from non-main Ractor",
+    "ivar in shareable-objects are not allowed to access from non-main Ractor, by @iv (get)",
+    "ivar in shareable-objects are not allowed to access from non-main Ractor, by @iv (set)",
+    900,
+    "moved objects have their shape properly set to original object's shape",
+    "Ractor-local storage with Thread inheritance of current Ractor",
+    "require in Ractor",
+    "autolaod in Ractor",
+    "moved objects being corrupted if embeded (String)",
+    "move object with generic ivar",
+    "move object with many generic ivars", # SEGV
+    "move object with complex generic ivars",
+  ]
+end
+if Ractor.builtin? && RUBY_VERSION < "3.3"
+  # fails on 3.2
+  skips += [
+    "check moved object", # SEGV
+  ]
+end
+if Ractor.builtin? && RUBY_VERSION < "3.1"
+  # fails on 3.0
+  skips += [
+    "and instance variables of classes/modules are accessible if they refer shareable objects",
+    "define_method is not allowed",
+    "check method cache invalidation", # SEGV
+  ]
+end
+if Ractor.shim? && RUBY_ENGINE == "ruby" && RUBY_VERSION < "3.0"
+  # fails on 2.7
+  skips += [
+    "Ractor.count",
+    "fstring pool 2", # spurious, probably a fstring table bug
+    "check method cache invalidation", # syntax
+  ]
+end
+if Ractor.shim? && RUBY_ENGINE == "jruby"
+  skips += [
+    "Ractor.count",
+    "ObjectSpace.each_object can not handle unshareable objects with Ractors",
+    "fstring pool 2", # spurious, probably a fstring table bug
+  ]
+end
+if $DEBUG
+  # fails probably Due to Thread.abort_on_exception being true with $DEBUG true
+  skips += [
+    "an exception in a Ractor non-main thread will not be re-raised at Ractor#receive",
+  ]
+end
+SKIPS = skips
+
+def new_empty_binding
+  # we need a shareable self for shareable procs/lambdas, but also preserve the default definee of Object
+  Object.class_eval { binding }
+end
+
+def skip(reason = nil)
+  throw :skip, :skip
+end
+
+TEST_LINES = [""] + File.readlines("#{__dir__}/test_ractor.rb").map(&:chomp)
+
+def generic_assert(expected, code, check, &error_message)
+  test_info = caller_locations(2, 1)[0]
+  line = test_info.lineno
+  test_name = TEST_LINES[line-1]
+  test_name = TEST_LINES[line-2] if test_name.start_with?("# [Bug")
+  warn "Could not find test name for test at line #{line}" unless test_name.start_with?("# ")
+  test_name = test_name[2..-1]
+
+  print "Running test from line #{line}: "
+  if SKIPS.include?(line) or SKIPS.include?(test_name)
+    puts "SKIP"
+    return
+  end
+
+  begin
+    actual = catch(:skip) do
+      eval(code, new_empty_binding, test_info.path, line).to_s
+    end
+    return if :skip == actual
+  rescue => e
+    puts "ERROR"
+    e.set_backtrace(e.backtrace.reject { |b|
+      b.include?(__FILE__)
+    }.map { |b|
+      b.sub(/(Object#)?new_empty_binding/, 'test')
+    })
+    raise e
+  end
+
+  if check.call(actual, expected)
+    puts "PASS #{actual}"
+  else
+    puts "FAIL"
+    raise RuntimeError, error_message.call(actual, expected), caller(2)
+  end
+
+  wait_ms = 10
+  waited = 0
+  while (leaked = Ractor.count - 1) > 0 and waited < wait_ms
+    sleep 0.001
+    waited += 1
+  end
+  unless leaked == 0
+    raise "Test at line #{line} leaked #{leaked} Ractors"
+  end
+
+  # cleanup global state
+  Object.send(:remove_const, :A) if Object.const_defined?(:A)
+end
+
+def assert_equal(expected, code, frozen_string_literal: nil)
+  generic_assert(expected, code, -> a, e { e == a }) { |actual, expected|
+    "Expected #{expected.inspect} but got #{actual.inspect}"
+  }
+end
+
+def assert_match(expected, code, frozen_string_literal: nil)
+  generic_assert(expected, code, -> a, e { e.match?(a) }) { |actual, expected|
+    "Expected #{expected} =~ #{actual.inspect} but it was false"
+  }
+end
+
+def assert_normal_exit(code)
+  # Skip because such tests need to run in their own process
+  return
+  # generic_assert(:unused, code, -> a, e { true }) { |actual, expected|
+  #   raise "unreachable"
+  # }
+end
+
+def yjit_enabled?
+  false
+end
+
+Warning[:experimental] = false
+
+require_relative 'test_ractor'