polyphony 0.43.8

data/lib/polyphony/extensions/thread.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+require_relative '../core/exceptions'
+
+# Thread extensions
+class ::Thread
+  attr_reader :main_fiber, :result
+
+  alias_method :orig_initialize, :initialize
+  def initialize(*args, &block)
+    @join_wait_queue = []
+    @finalization_mutex = Mutex.new
+    @args = args
+    @block = block
+    orig_initialize { execute }
+  end
+
+  def execute
+    # agent must be created in the context of the new thread, therefore it
+    # cannot be created in Thread#initialize
+    @agent = Polyphony::LibevAgent.new
+    setup
+    @ready = true
+    result = @block.(*@args)
+  rescue Polyphony::MoveOn, Polyphony::Terminate => e
+    result = e.value
+  rescue Exception => result
+  ensure
+    @ready = true
+    finalize(result)
+  end
+
+  attr_accessor :agent
+
+  def setup
+    @main_fiber = Fiber.current
+    @main_fiber.setup_main_fiber
+    setup_fiber_scheduling
+  end
+
+  def finalize(result)
+    unless Fiber.current.children.empty?
+      Fiber.current.terminate_all_children
+      Fiber.current.await_all_children
+    end
+    @finalization_mutex.synchronize do
+      @terminated = true
+      @result = result
+      signal_waiters(result)
+    end
+    @agent.finalize
+  end
+
+  def signal_waiters(result)
+    @join_wait_queue.each { |w| w.signal(result) }
+  end
+
+  alias_method :orig_join, :join
+  def join(timeout = nil)
+    watcher = Fiber.current.auto_watcher
+
+    @finalization_mutex.synchronize do
+      if @terminated
+        @result.is_a?(Exception) ? (raise @result) : (return @result)
+      else
+        @join_wait_queue << watcher
+      end
+    end
+    timeout ? move_on_after(timeout) { watcher.await } : watcher.await
+  end
+  alias_method :await, :join
+
+  alias_method :orig_raise, :raise
+  def raise(error = nil)
+    Thread.pass until @main_fiber
+    error = RuntimeError.new if error.nil?
+    error = RuntimeError.new(error) if error.is_a?(String)
+    error = error.new if error.is_a?(Class)
+
+    sleep 0.0001 until @ready
+    main_fiber&.raise(error)
+  end
+
+  alias_method :orig_kill, :kill
+  def kill
+    return if @terminated
+
+    raise Polyphony::Terminate
+  end
+
+  alias_method :orig_inspect, :inspect
+  def inspect
+    return orig_inspect if self == Thread.main
+
+    state = status || 'dead'
+    "#<Thread:#{object_id} #{location} (#{state})>"
+  end
+  alias_method :to_s, :inspect
+
+  def location
+    @block.source_location.join(':')
+  end
+
+  def <<(value)
+    main_fiber << value
+  end
+  alias_method :send, :<<
+end

data/lib/polyphony/net.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require_relative './extensions/socket'
+require_relative './extensions/openssl'
+
+module Polyphony
+  # A more elegant networking API
+  module Net
+    class << self
+      def tcp_connect(host, port, opts = {})
+        socket = ::Socket.new(:INET, :STREAM).tap do |s|
+          addr = ::Socket.sockaddr_in(port, host)
+          s.connect(addr)
+        end
+        if opts[:secure_context] || opts[:secure]
+          secure_socket(socket, opts[:secure_context], opts.merge(host: host))
+        else
+          socket
+        end
+      end
+
+      def tcp_listen(host = nil, port = nil, opts = {})
+        host ||= '0.0.0.0'
+        raise 'Port number not specified' unless port
+
+        socket = socket_from_options(host, port, opts)
+        if opts[:secure_context] || opts[:secure]
+          secure_server(socket, opts[:secure_context], opts)
+        else
+          socket
+        end
+      end
+
+      def socket_from_options(host, port, opts)
+        ::Socket.new(:INET, :STREAM).tap do |s|
+          s.reuse_addr if opts[:reuse_addr]
+          s.dont_linger if opts[:dont_linger]
+          addr = ::Socket.sockaddr_in(port, host)
+          s.bind(addr)
+          s.listen(0)
+        end
+      end
+
+      def secure_socket(socket, context, opts)
+        context ||= OpenSSL::SSL::SSLContext.new
+        setup_alpn(context, opts[:alpn_protocols]) if opts[:alpn_protocols]
+        socket = secure_socket_wrapper(socket, context)
+
+        socket.tap do |s|
+          s.hostname = opts[:host] if opts[:host]
+          s.connect
+          s.post_connection_check(opts[:host]) if opts[:host]
+        end
+      end
+
+      def secure_socket_wrapper(socket, context)
+        if context
+          OpenSSL::SSL::SSLSocket.new(socket, context)
+        else
+          OpenSSL::SSL::SSLSocket.new(socket)
+        end
+      end
+
+      def secure_server(socket, context, opts)
+        setup_alpn(context, opts[:alpn_protocols]) if opts[:alpn_protocols]
+        OpenSSL::SSL::SSLServer.new(socket, context)
+      end
+
+      def setup_alpn(context, protocols)
+        context.alpn_protocols = protocols
+        context.alpn_select_cb = lambda do |peer_protocols|
+          (protocols & peer_protocols).first
+        end
+      end
+    end
+  end
+end

data/lib/polyphony/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Polyphony
+  VERSION = '0.43.8'
+end

data/polyphony.gemspec

@@ -0,0 +1,40 @@
+require_relative './lib/polyphony/version'
+
+Gem::Specification.new do |s|
+  s.name        = 'polyphony'
+  s.version     = Polyphony::VERSION
+  s.licenses    = ['MIT']
+  s.summary     = 'Fine grained concurrency for Ruby'
+  s.author      = 'Sharon Rosner'
+  s.email       = 'ciconia@gmail.com'
+  s.files       = `git ls-files`.split
+  s.homepage    = 'https://digital-fabric.github.io/polyphony'
+  s.metadata    = {
+    "source_code_uri" => "https://github.com/digital-fabric/polyphony",
+    "documentation_uri" => "https://digital-fabric.github.io/polyphony/",
+    "homepage_uri" => "https://digital-fabric.github.io/polyphony/",
+    "changelog_uri" => "https://github.com/digital-fabric/polyphony/blob/master/CHANGELOG.md"
+  }
+  s.rdoc_options = ["--title", "polyphony", "--main", "README.md"]
+  s.extra_rdoc_files = ["README.md"]
+  s.extensions = ["ext/polyphony/extconf.rb"]
+  s.require_paths = ["lib"]
+  s.required_ruby_version = '>= 2.6'
+
+  s.add_development_dependency  'httparty',             '0.17.0'
+  s.add_development_dependency  'localhost',            '1.1.4'
+  s.add_development_dependency  'minitest',             '5.13.0'
+  s.add_development_dependency  'minitest-reporters',   '1.4.2'
+  s.add_development_dependency  'simplecov',            '0.17.1'
+  s.add_development_dependency  'rubocop',              '0.85.1'
+  s.add_development_dependency  'pg',                   '1.1.4'
+  s.add_development_dependency  'rake-compiler',        '1.0.5'
+  s.add_development_dependency  'redis',                '4.1.0'
+  s.add_development_dependency  'hiredis',              '0.6.3'
+  s.add_development_dependency  'http_parser.rb',       '~>0.6.0'
+
+  s.add_development_dependency  'jekyll',               '~>3.8.6'
+  s.add_development_dependency  'jekyll-remote-theme',  '~>0.4.1'
+  s.add_development_dependency  'jekyll-seo-tag',       '~>2.6.1'
+  s.add_development_dependency  'just-the-docs',        '~>0.3.0'
+end

data/test/coverage.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require 'coverage'
+require 'simplecov'
+
+# Since we load code using Modulation, and the stock coverage gem does not
+# calculate coverage for code loaded using `Kernel#eval` et al, we need to use
+# the TracePoint API in order to trace execution. Here we monkey-patch the two
+# main Coverage class methods, start and result to use TracePoint. Otherwise we
+# let SimpleCov do its business.
+module Coverage
+  EXCLUDE = %w{coverage eg helper run
+  }.map { |n| File.expand_path("test/#{n}.rb") }
+
+  LIB_FILES = Dir["#{File.join(FileUtils.pwd, 'lib')}/polyphony/**/*.rb"]
+
+  class << self
+    def relevant_lines_for_filename(filename)
+      @classifier ||= SimpleCov::LinesClassifier.new
+      @classifier.classify(IO.read(filename).lines)
+    end
+
+    def start
+      @result = {}
+      trace = TracePoint.new(:line) do |tp|
+        next if tp.path =~ /\(/
+      
+        absolute = File.expand_path(tp.path)
+        next unless LIB_FILES.include?(absolute)# =~ /^#{LIB_DIR}/
+        
+        @result[absolute] ||= relevant_lines_for_filename(absolute)
+        @result[absolute][tp.lineno - 1] = 1
+      end
+      trace.enable
+    end
+
+    def result
+      @result
+    end
+  end
+end
+
+class << SimpleCov::LinesClassifier
+  alias_method :orig_whitespace_line?, :whitespace_line?
+  def whitespace_line?(line)
+    # apparently TracePoint tracing does not cover lines including only keywords
+    # such as begin end etc, so here we mark those lines as whitespace, so they
+    # won't count towards the coverage score.
+    line.strip =~ /^(begin|end|ensure|else|\{|\})|(\s*rescue\s.+)$/ ||
+                  orig_whitespace_line?(line)
+  end
+end
+
+SimpleCov.start

data/test/eg.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Kernel
+  RE_CONST  = /^[A-Z]/.freeze
+  RE_ATTR   = /^@(.+)$/.freeze
+
+  def eg(hash)
+    Module.new.tap do |m|
+      s = m.singleton_class
+      hash.each do |k, v|
+        case k
+        when RE_CONST
+          m.const_set(k, v)
+        when RE_ATTR
+          m.instance_variable_set(k, v)
+        else
+          block = if v.respond_to?(:to_proc)
+                    proc { |*args| instance_exec(*args, &v) }
+                  else
+                    proc { v }
+                  end
+          s.define_method(k, &block)
+        end
+      end
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+
+require_relative './coverage' if ENV['COVERAGE']
+
+require 'polyphony'
+
+require 'fileutils'
+require_relative './eg'
+
+require 'minitest/autorun'
+require 'minitest/reporters'
+
+::Exception.__disable_sanitized_backtrace__ = true
+
+Minitest::Reporters.use! [
+  Minitest::Reporters::SpecReporter.new
+]
+
+class ::Fiber
+  attr_writer :auto_watcher
+end
+
+class MiniTest::Test
+  def setup
+    # puts "* setup #{self.name}"
+    if Fiber.current.children.size > 0
+      puts "Children left: #{Fiber.current.children.inspect}"
+      exit!
+    end
+    Fiber.current.setup_main_fiber
+    Fiber.current.instance_variable_set(:@auto_watcher, nil)
+    Thread.current.agent = Polyphony::LibevAgent.new
+    sleep 0 # apparently this helps with timer accuracy
+  end
+
+  def teardown
+    # puts "* teardown #{self.name.inspect} Fiber.current: #{Fiber.current.inspect}"
+    Fiber.current.terminate_all_children
+    Fiber.current.await_all_children
+    Fiber.current.auto_watcher = nil
+  rescue => e
+    puts e
+    puts e.backtrace.join("\n")
+    exit!
+  end
+end
+
+module Kernel
+  def capture_exception
+    yield
+  rescue Exception => e
+    e
+  end
+end

data/test/q.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'fiber'
+require_relative '../lib/polyphony_ext'
+
+queue = Polyphony::LibevQueue.new
+
+queue.push :a
+queue.push :b
+queue.push :c
+p [queue.shift_no_wait]
+queue.push :d
+p [queue.shift_no_wait]
+p [queue.shift_no_wait]
+p [queue.shift_no_wait]
+p [queue.shift_no_wait]
+
+queue.unshift :e
+p [queue.shift_no_wait]
+
+queue.push :f
+queue.push :g
+p [queue.shift_no_wait]

data/test/run.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+Dir.glob("#{__dir__}/test_*.rb").each do |path|
+  require(path)
+end

data/test/stress.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+count = ARGV[0] ? ARGV[0].to_i : 100
+
+TEST_CMD = 'ruby test/run.rb'
+
+def run_test(count)
+  puts "#{count}: running tests..."
+  system(TEST_CMD)
+  return if $?.exitstatus == 0
+
+  puts "Failure after #{count} tests"
+  exit!
+end
+
+trap('INT') { exit! }
+t0 = Time.now
+count.times { |i| run_test(i + 1) }
+elapsed = Time.now - t0
+puts format(
+  "Successfully ran %d tests in %f seconds (%f per test)",
+  count,
+  elapsed,
+  elapsed / count
+)

data/test/test_agent.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+require_relative 'helper'
+
+class AgentTest < MiniTest::Test
+  def setup
+    super
+    @prev_agent = Thread.current.agent
+    @agent = Polyphony::LibevAgent.new
+    Thread.current.agent = @agent
+  end
+
+  def teardown
+    @agent.finalize
+    Thread.current.agent = @prev_agent
+  end
+
+  def test_sleep
+    count = 0
+    t0 = Time.now
+    spin {
+      @agent.sleep 0.01
+      count += 1
+      @agent.sleep 0.01
+      count += 1
+      @agent.sleep 0.01
+      count += 1
+    }.await
+    assert Time.now - t0 >= 0.03
+    assert_equal 3, count
+  end
+
+  def test_write_read_partial
+    i, o = IO.pipe
+    buf = +''
+    f = spin { @agent.read(i, buf, 5, false) }
+    @agent.write(o, 'Hello world')
+    return_value = f.await
+    
+    assert_equal 'Hello', buf
+    assert_equal return_value, buf
+  end
+
+  def test_write_read_to_eof_limited_buffer
+    i, o = IO.pipe
+    buf = +''
+    f = spin { @agent.read(i, buf, 5, true) }
+    @agent.write(o, 'Hello')
+    snooze
+    @agent.write(o, ' world')
+    snooze
+    o.close
+    return_value = f.await
+    
+    assert_equal 'Hello', buf
+    assert_equal return_value, buf
+  end
+
+  def test_write_read_to_eof
+    i, o = IO.pipe
+    buf = +''
+    f = spin { @agent.read(i, buf, 10**6, true) }
+    @agent.write(o, 'Hello')
+    snooze
+    @agent.write(o, ' world')
+    snooze
+    o.close
+    return_value = f.await
+    
+    assert_equal 'Hello world', buf
+    assert_equal return_value, buf
+  end
+
+  def test_waitpid
+    pid = fork do
+      @agent.post_fork
+      exit(42)
+    end
+    
+    result = @agent.waitpid(pid)
+    assert_equal [pid, 42], result
+  end
+
+  def test_read_loop
+    i, o = IO.pipe
+
+    buf = []
+    spin do
+      buf << :ready
+      @agent.read_loop(i) { |d| buf << d }
+      buf << :done
+    end
+
+    # writing always causes snoozing
+    o << 'foo'
+    o << 'bar'
+    o.close
+
+    # read_loop will snooze after every read
+    6.times { snooze }
+
+    assert_equal [:ready, 'foo', 'bar', :done], buf
+  end
+
+  def test_accept_loop
+    server = TCPServer.new('127.0.0.1', 1234)
+
+    clients = []
+    server_fiber = spin do
+      @agent.accept_loop(server) { |c| clients << c }
+    end
+
+    c1 = TCPSocket.new('127.0.0.1', 1234)
+    10.times { snooze }
+
+    assert_equal 1, clients.size
+
+    c2 = TCPSocket.new('127.0.0.1', 1234)
+    10.times { snooze }
+
+    assert_equal 2, clients.size
+
+  ensure
+    c1&.close
+    c2&.close
+    server_fiber.stop
+    snooze
+    server&.close
+  end
+end