ractor-shim 0.0.1 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5132b882154ee4c0af587fb5e4b2f10030ef9834237abbbae5417e3fb51b62a8
-  data.tar.gz: 0ed22147461e8c6d3feae42d18fd06247ed6d8df800dc6fb2211e56ba9f807dc
+  metadata.gz: ba3ed0a7cfb270c23651d31c50286068defa1b0492459286cabc055da64ae2c8
+  data.tar.gz: 98ed41fb52ac1676960b4d6a49d49e66920c4465a779da9b1e5f12073427cee6
 SHA512:
-  metadata.gz: bbabf61d3ee523150b793e177a2c60d14cad3ed94848a2290bd9d02f86175f251a641a24cb1841eb01a749e14b7dcbbb94fdcac1c8f23cc1b927a3ac3d533b4e
-  data.tar.gz: 9264645ee001fbb8f301163a28f88e1e3431639312ad4c249060a75d80f45d69afaae5b7504a3b3c6175eac7325d204a31e1a78a264c7d50a47052be4b7a18b7
+  metadata.gz: 55647c412c0fdecac6feb0b7fcf353c86d2114979eb6aa9ee8c1c0521c3a05e35123ea9db82ba2d808f483bfe244add8b2140cad1267121d8f1cfdeb329250c9
+  data.tar.gz: 10dd59ab7425af30137bb2fa0edf416447cdd01a16aa6601d3df57a396b1833638e78197ca4e827169223ab23d9a94fe48f7ef6ff9f8e85522303a6cd8e9dc6a

data/README.md

@@ -1,18 +1,39 @@
 # ractor-shim
 
-A shim to define `Ractor` by using `Thread`, if `Ractor` is not already defined.
+A shim to define `Ractor` by using `Thread`, `Queue`, etc if `Ractor` is not already defined.
 
-This is notably useful to run programs needing `Ractor` on Ruby implementations which don't define `Ractor.
+This is notably useful to run programs relying on `Ractor` on Ruby implementations which don't define `Ractor` such as TruffleRuby and JRuby.
+
+Note that TruffleRuby and JRuby both run Ruby code in threads in parallel, so this gem enables using the Ractor API on these Rubies and run Ractors in parallel.
+
+The gem also provides the Ruby 3.5 Ractor API  (`Ractor::Port`, `Ractor#{join,value,monitor}`, etc) for CRuby 2.7 to 3.4.
+
+When `Ractor` is not already defined, this gem implements `Ractor.make_shareable(object)` by just returning `object` to avoid unnecessary overhead.
+The main reason is that the CRuby implementation of `Ractor.make_shareable` returns an immutable object, so it won't be mutated by the program anyway and there is no need to deep freeze.
+Another reason is many gems do `Ractor.make_shareable(object) if defined?(Ractor)` and so that becomes dependent on whether `Ractor` is defined before that gem loads. That problem disappears if `Ractor.make_shareable(object)` is noop.
+
+## Installation
+
+```bash
+$ gem install ractor-shim
+```
+or
+```ruby
+# In Gemfile
+gem "ractor-shim"
+```
 
 ## Usage
 
 ```ruby
 require 'ractor/shim'
+
+Ractor.new { ... }
 ```
 
 ## Development
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `ractor-shim.gemspec`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
 
 ## Contributing
 

data/Rakefile

@@ -1,4 +1,8 @@
 # frozen_string_literal: true
 
 require "bundler/gem_tasks"
-task default: %i[]
+task default: [:test]
+
+task :test do
+  exec RbConfig.ruby, "-Ilib", "test/run_tests.rb"
+end

data/lib/ractor/shim.rb

@@ -1,3 +1,424 @@
 # frozen_string_literal: true
 
-require_relative "shim/version"
+builtin_ractor = !!defined?(Ractor)
+class Ractor
+end
+Ractor.define_singleton_method(:builtin?) { builtin_ractor }
+Ractor.define_singleton_method(:shim?) { !builtin_ractor }
+
+# Ractor::Port
+
+if Ractor.builtin?
+  class Ractor::Port
+    QUIT = Object.new.freeze
+
+    def initialize
+      @pipe = Ractor.new do
+        while true
+          msg = Ractor.receive
+          break if QUIT.equal?(msg)
+          Ractor.yield msg
+        end
+      end
+    end
+
+    def send(message)
+      @pipe.send(message)
+    end
+    alias_method :<<, :send
+
+    def receive
+      @pipe.take
+    end
+
+    def close
+      @pipe.send(QUIT)
+    end
+
+    def closed?
+      @pipe.inspect.end_with?(' terminated>')
+    end
+
+    def inspect
+      super
+    end
+  end unless defined?(Ractor::Port)
+else
+  class Ractor::Port
+    attr_reader :queue
+    private :queue
+
+    def initialize
+      @queue = Queue.new
+    end
+
+    def send(message, move: false)
+      Ractor::SELECT_MUTEX.synchronize {
+        @queue << message
+        Ractor::SELECT_CV.broadcast
+      }
+      self
+    end
+    alias_method :<<, :send
+
+    def receive
+      @queue.pop
+    end
+
+    def close
+      Ractor::SELECT_MUTEX.synchronize {
+        @queue.close
+        Ractor::SELECT_CV.broadcast
+      }
+      self
+    end
+
+    def closed?
+      @queue.closed?
+    end
+
+    def inspect
+      super
+    end
+  end
+end
+
+if Ractor.shim?
+  class Ractor
+    class Error < RuntimeError
+    end
+
+    class RemoteError < Error
+      attr_reader :ractor
+      def initialize(ractor)
+        @ractor = ractor
+      end
+    end
+
+    class ClosedError < StopIteration
+    end
+
+    @count = 1
+    COUNT_MUTEX = Mutex.new
+    CHANGE_COUNT = -> delta {
+      COUNT_MUTEX.synchronize { @count += delta }
+    }
+
+    @id = 0
+    ID_MUTEX = Mutex.new
+    GET_ID = -> {
+      ID_MUTEX.synchronize { @id += 1 }
+    }
+
+    SELECT_MUTEX = Mutex.new
+    SELECT_CV = ConditionVariable.new
+
+    def self.main
+      MAIN_RACTOR
+    end
+
+    def self.main?
+      current == main
+    end
+
+    def self.current
+      Thread.current.thread_variable_get(:current_ractor) or raise "Could not find current Ractor"
+    end
+
+    def self.count
+      COUNT_MUTEX.synchronize { @count }
+    end
+
+    def self.receive
+      Ractor.current.__send__(:receive)
+    end
+
+    class << self
+      alias_method :recv, :receive
+
+      def new(...)
+        super(...)
+      end
+    end
+
+    # def self.select_simple_spinning(*ractors)
+    #   raise ArgumentError, "specify at least one ractor or `yield_value`" if ractors.empty?
+    #   while true
+    #     ractors.each do |ractor_or_port|
+    #       if Ractor === ractor_or_port
+    #         queue = ractor_or_port.out_queue
+    #       elsif Ractor::Port === ractor_or_port
+    #         queue = ractor_or_port.queue
+    #       else
+    #         raise ArgumentError, "Unexpected argument for Ractor.select: #{ractor_or_port}"
+    #       end
+    #
+    #       begin
+    #         value = queue.pop(true)
+    #         return [ractor_or_port, value]
+    #       rescue ThreadError
+    #         Thread.pass
+    #       end
+    #     end
+    #   end
+    # end
+
+    def self.select(*ractors)
+      raise ArgumentError, "specify at least one ractor or `yield_value`" if ractors.empty?
+
+      SELECT_MUTEX.synchronize do
+        while true
+          ractors.each do |ractor_or_port|
+            if Ractor === ractor_or_port
+              queue = ractor_or_port.__send__(:out_queue)
+            elsif Ractor::Port === ractor_or_port
+              queue = ractor_or_port.__send__(:queue)
+            else
+              raise ArgumentError, "Unexpected argument for Ractor.select: #{ractor_or_port}"
+            end
+
+            begin
+              value = queue.pop(true)
+              return [ractor_or_port, value]
+            rescue ThreadError
+              # keep looping
+            end
+          end
+
+          # Wait until an item is added to a relevant Queue
+          SELECT_CV.wait(SELECT_MUTEX)
+        end
+      end
+    end
+
+    def self.make_shareable(object, copy: false)
+      # no copy, just return the object
+      object
+    end
+
+    def self.shareable?(object)
+      true
+    end
+
+    def self.[](var)
+      Ractor.current[var]
+    end
+
+    def self.[]=(var, value)
+      Ractor.current[var] = value
+    end
+
+    def self.store_if_absent(var, &block)
+      Ractor.current.__send__(:store_if_absent, var, &block)
+    end
+
+    def self._require(feature)
+      Kernel.require(feature)
+    end
+
+    attr_reader :name, :default_port
+
+    attr_reader :out_queue
+    private :out_queue
+
+    def initialize(*args, name: nil, &block)
+      raise ArgumentError, "must be called with a block" unless block
+      initialize_common(name, block)
+      CHANGE_COUNT.call(1)
+
+      @thread = Thread.new {
+        Thread.current.thread_variable_set(:current_ractor, self)
+        begin
+          result = self.instance_exec(*args, &block)
+          SELECT_MUTEX.synchronize {
+            unless @out_queue.closed?
+              @out_queue << result
+              SELECT_CV.broadcast
+            end
+          }
+          result
+        rescue Exception => e
+          @exception = e
+          nil
+        ensure
+          @termination_mutex.synchronize {
+            CHANGE_COUNT.call(-1)
+            @status = :terminated
+
+            monitor_message = @exception ? :aborted : :exited
+            @monitors.each { |monitor|
+              monitor << monitor_message
+            }
+            @monitors.clear
+          }
+        end
+      }
+    end
+
+    private def initialize_main
+      initialize_common(nil, nil)
+      @thread = nil
+    end
+
+    private def initialize_common(name, block)
+      @name = name.nil? ? nil : (String.try_convert(name) or raise TypeError)
+      @id = GET_ID.call
+      @status = :running
+      @from = block ? block.source_location.join(":") : nil
+      @default_port = Ractor::Port.new
+      @out_queue = Queue.new
+      @storage = {}
+      @exception = nil
+      @monitors = []
+      @termination_mutex = Mutex.new
+    end
+
+    def [](var)
+      raise "Cannot get ractor local storage for non-current ractor" unless Ractor.current == self
+      @storage[var]
+    end
+
+    def []=(var, value)
+      raise "Cannot set ractor local storage for non-current ractor" unless Ractor.current == self
+      @storage[var] = value
+    end
+
+    private def store_if_absent(var, &block)
+      if value = @storage[var]
+        value
+      else
+        value = block.call
+        @storage[var] = value
+        value
+      end
+    end
+
+    def send(message, ...)
+      raise Ractor::ClosedError, "The port was already closed" if @status == :terminated || @default_port.closed?
+      @default_port.send(message, ...)
+      self
+    end
+    alias_method :<<, :send
+
+    private def receive
+      @default_port.receive
+    end
+
+    # def take
+    #   @out_queue.pop
+    # end
+
+    private def close_incoming
+      @default_port.close
+      self
+    end
+
+    private def close_outgoing
+      SELECT_MUTEX.synchronize {
+        @out_queue.close
+        SELECT_CV.broadcast
+      }
+      self
+    end
+
+    def close
+      close_incoming
+      close_outgoing
+    end
+
+    def monitor(port)
+      @termination_mutex.synchronize {
+        if @status == :terminated
+          port << (@exception ? :aborted : :exited)
+          false
+        else
+          @monitors << port
+        end
+      }
+    end
+
+    def unmonitor(port)
+      @termination_mutex.synchronize {
+        @monitors.delete port
+      }
+    end
+
+    def join
+      value
+      self
+    end
+
+    def value
+      @thread.join
+
+      if exc = @exception
+        remote_error = RemoteError.new(self)
+        raise remote_error, cause: exc
+      end
+
+      @thread.value
+    end
+
+    def inspect
+      ["#<Ractor:##{@id}", @name, @from, "#{@status}>"].compact.join(' ')
+    end
+    alias_method :to_s, :inspect
+
+    MAIN_RACTOR = Ractor.allocate
+    MAIN_RACTOR.__send__(:initialize_main)
+    Thread.main.thread_variable_set(:current_ractor, MAIN_RACTOR)
+  end
+end
+
+if Ractor.builtin?
+  class Ractor
+    unless method_defined?(:join)
+      alias_method :join, :take
+    end
+
+    unless method_defined?(:value)
+      alias_method :value, :take
+    end
+
+    unless respond_to?(:main?)
+      def self.main?
+        self == Ractor.main
+      end
+    end
+
+    unless method_defined?(:close)
+      def close
+        close_incoming
+        close_outgoing
+      end
+    end
+  end
+end
+
+# Ractor.{shareable_proc,shareable_lambda}
+
+class << Ractor
+  if Ractor.builtin?
+    unless method_defined?(:shareable_proc)
+      def shareable_proc(&b)
+        Ractor.make_shareable(b)
+      end
+    end
+
+    unless method_defined?(:shareable_lambda)
+      def shareable_lambda(&b)
+        Ractor.make_shareable(b)
+      end
+    end
+  else
+    unless method_defined?(:shareable_proc)
+      alias_method :shareable_proc, :proc
+      public :shareable_proc
+    end
+
+    unless method_defined?(:shareable_lambda)
+      alias_method :shareable_lambda, :lambda
+      public :shareable_lambda
+    end
+  end
+end

data/test/methods/Ractor.singleton_class.txt

@@ -0,0 +1,16 @@
+[]
+[]=
+_require
+count
+current
+main
+main?
+make_shareable
+new
+receive
+recv
+select
+shareable?
+shareable_lambda
+shareable_proc
+store_if_absent

data/test/methods/Ractor.txt

@@ -0,0 +1,13 @@
+<<
+[]
+[]=
+close
+default_port
+inspect
+join
+monitor
+name
+send
+to_s
+unmonitor
+value

data/test/methods/Ractor::Port.txt

@@ -0,0 +1,6 @@
+<<
+close
+closed?
+inspect
+receive
+send

data/test/ractor_methods_test.rb

@@ -0,0 +1,83 @@
+# Based on https://github.com/truffleruby/truffleruby/blob/master/spec/truffle/methods_spec.rb
+
+# How to regenerate files:
+#
+# - switch to MRI, the version we are compatible with
+# - run `jt -u ruby test spec/truffle/methods_spec.rb`
+
+# jt test and jt tag can be used as normal,
+# but instead of jt untag, jt purge must be used to remove tags:
+# $ jt purge spec/truffle/methods_spec.rb
+
+# socket modules are found with:
+# m1=ObjectSpace.each_object(Module).to_a; require "socket"; m2=ObjectSpace.each_object(Module).to_a; p m2-m1
+
+modules = %w[
+  Ractor Ractor.singleton_class
+  Ractor::Port Ractor::Port.singleton_class
+]
+
+def ruby(code, *flags)
+  IO.popen([RbConfig.ruby, *flags], "r+") { |pipe|
+    pipe.write code
+    pipe.close_write
+    pipe.read
+  }
+end
+
+if RUBY_ENGINE == "ruby" && RUBY_VERSION >= "3.5"
+  modules.each do |mod|
+    file = "#{__dir__}/methods/#{mod}.txt"
+    code = "puts #{mod}.public_instance_methods(false).sort"
+    methods = ruby(code)
+    methods = methods.lines.map { |line| line.chomp.to_sym }
+    contents = methods.map { |meth| "#{meth}\n" }.join
+    File.write file, contents
+  end
+end
+
+code = <<-RUBY
+require "ractor/shim"
+#{modules.inspect}.each { |m|
+  puts m
+  puts eval(m).public_instance_methods(false).sort
+  puts
+}
+RUBY
+all_methods = {}
+ruby(code, "-I#{File.dirname(__dir__)}/lib").rstrip.split("\n\n").each do |group|
+  mod, *methods = group.lines.map(&:chomp)
+  all_methods[mod] = methods.map(&:to_sym)
+end
+
+modules.each do |mod|
+  file = "#{__dir__}/methods/#{mod}.txt"
+  expected = File.readlines(file).map { |line| line.chomp.to_sym }
+  methods = all_methods[mod]
+
+  if methods != expected
+    extras = methods - expected
+    missing = expected - methods
+
+    if mod == "Ractor.singleton_class"
+      extras -= %i[builtin? shim?] # intended extras
+    end
+
+    if RUBY_ENGINE == "ruby" and RUBY_VERSION < "3.5"
+      if mod == "Ractor"
+        extras -= %i[take close_incoming close_outgoing] # expected extra on those versions
+        missing -= %i[default_port monitor unmonitor] # hard to implement on those versions
+      elsif mod == "Ractor.singleton_class"
+        extras -= %i[yield receive_if] # expected extra on those versions
+        if RUBY_VERSION < "3.4"
+          missing -= %i[[] []= store_if_absent _require] # hard to implement on those versions
+        end
+      end
+    end
+
+    raise "#{mod} methods should not include #{extras}" unless extras.empty?
+    raise "#{mod} methods should include #{missing}" unless missing.empty?
+  end
+end
+
+puts "#{__FILE__}: OK"