polyphony 0.67 → 0.71

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6b8553a2f84e79520ce53412ee6891873c9a9d25e0789e82b26a7153d3d7ae67
-  data.tar.gz: 6889ef71f11d79d6fbb82adc77447f9b3526a29858f2aa3c9e51ee9098add2c7
+  metadata.gz: 36566e2f3d8ca8535569ec2766183295a6b17f3af67123a6442242881b004ef8
+  data.tar.gz: d38059a39ce762cb043c68374829050b6abd4596fc900f68d24120e843efa9db
 SHA512:
-  metadata.gz: 91814f767f74c673681a2215f82a302f3aab5ced6ae7197720e474b8c1ad6846eb10001580c1f11d5b62bd0ed7563422fd71f0fb5728eb0b3fcb4ebd36d2038c
-  data.tar.gz: 74e1c7ed34b4f212ca1ffd5881da3e9f689c3934d0f3680da2396d7444b5a214a610ccc779c1ef59713116f0a8240491dc60f32b5d5e30de5092ddc41b8c9f18
+  metadata.gz: 4d85656e6ca42458d262433bed6817e6fe5d60b194fddce269fae9ab9d479a7167b3460ac01cdb3c3018d9e3bf3b61ea5560aa2f186dedb841ac81dec311ffb5
+  data.tar.gz: 6db9f437744ece1eeae09c738ac31172439472acd21e8956e75defbdf4b113b5fd61e04ff73ef9f29046b54baf44e0064350b6c04df9883c34b16efd9bad35e1

data/CHANGELOG.md

@@ -1,3 +1,23 @@
+## 0.71 2021-08-20
+
+- Fix compilation on Ruby 3.0
+
+## 0.70 2021-08-19
+
+- New implementation for `#supervise`
+- Reset backend state and runqueue on fork
+
+## 0.69 2021-08-16
+
+- Rename `#__polyphony_read_method__` to `#__parser_read_method__`
+
+## 0.68 2021-08-13
+
+- Fix missing default value in socket classes' `#readpartial`
+- Fix linking of operations in `Backend#chain` (io_uring version)
+- Rename `Fiber#attach` to `Fiber#attach_to`
+- Expose original `SSLServer#accept`
+
 ## 0.67 2021-08-06
 
 - Improve fiber monitoring
@@ -11,7 +31,7 @@
 
 ## 0.65 2021-07-29
 
-- Add `#__polyphony_read_method__` method for read method detection
+- Add `#__parser_read_method__` method for read method detection
 
 ## 0.64 2021-07-26
 

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    polyphony (0.67)
+    polyphony (0.71)
 
 GEM
   remote: https://rubygems.org/
@@ -75,4 +75,4 @@ DEPENDENCIES
   simplecov (= 0.17.1)
 
 BUNDLED WITH
-   2.2.3
+   2.2.26

data/TODO.md

@@ -26,31 +26,8 @@
 
 - Add support for `close` to io_uring backend
 
-- Graceful shutdown again:
-  - What happens to children when doing a graceful shutdown?
-  - What are the implications of passing graceful shutdown flag to children?
-  - What about errors while doing a graceful shutdown?
-  - What about graceful restarts?
-  - Some interesting discussions:
-    - https://trio.discourse.group/search?q=graceful%20shutdown
-    - https://github.com/python-trio/trio/issues/147
-    - https://github.com/python-trio/trio/issues/143
-    - https://trio.discourse.group/t/graceful-shutdown/93/2
-    - https://250bpm.com/blog:146/
-    - https://www.rodrigoaraujo.me/posts/golang-pattern-graceful-shutdown-of-concurrent-events/
-    - https://github.com/tj/go-gracefully
-  - `Fiber#finalize_children` should pass graceful shutdown flag to children
-  - A good use case is an HTTP server that on graceful shutdown:
-    - stops listening
-    - waits for all ongoing requests to finish, optionally with a timeout
-
 ## Roadmap for Polyphony 1.0
 
-- check integration with rb-inotify
-
-- Improve `#supervise`. It does not work as advertised, and seems to exhibit an
-  inconsistent behaviour (see supervisor example).
-
 - Add test that mimics the original design for Monocrono:
   - 256 fibers each waiting for a message
   - When message received do some blocking work using a `ThreadPool`

data/bin/pdbg

@@ -8,23 +8,105 @@ UNIX_SOCKET_PATH = '/tmp/pdbg.sock'
 
 cmd = ARGV.join(' ')
 injected_lib_path = File.expand_path('../lib/polyphony/debugger/server_inject.rb', __dir__)
-p cmd
 pid = fork { exec("env POLYPHONY_DEBUG_SOCKET_PATH=#{UNIX_SOCKET_PATH} ruby #{cmd}") }
 puts "Started debugged process (#{pid})"
 
-sleep 3
-socket = UNIXSocket.new(UNIX_SOCKET_PATH)
+socket = nil
+while !socket
+  socket = UNIXSocket.new(UNIX_SOCKET_PATH) rescue nil
+end
+
+def parse_command(cmd)
+  case cmd
+  when /^(step|s)$/
+    { cmd: :step }
+  when /^(state|st)$/
+    { cmd: :state }
+  when /^(help|h)$/
+    { cmd: :help }
+  when /^(list|l)$/
+    { cmd: :list }
+  else
+    nil
+  end
+end
+
+def display_info(info)
+  info = eval(info)
+  case (info && info[:kind])
+  when :listing
+    print_listing(info)
+  when :state
+    print_state(info)
+  else
+    p info
+  end
+rescue SyntaxError
+  puts "Failed to eval:"
+  p info
+end
+
+FILE_LINES_CACHE = {}
+
+def self.get_snippet(path, lineno)
+  lines = FILE_LINES_CACHE[path] ||= IO.read(path).lines
+  start_idx = lineno - 5
+  stop_idx = lineno + 3
+  stop_idx = lines.size - 1 if stop_idx >= lines.size
+  start_idx = 0 if start_idx < 0
+  (start_idx..stop_idx).map { |idx| [idx + 1, lines[idx]]}
+end
+
+def print_snippet(info, snippet, cur_line)
+  places = FILE_LINES_CACHE[info[:path]].size.to_s.size
+  snippet.each do |(lineno, line)|
+    is_cur = lineno == cur_line
+    formatted = format("%s% #{places}d %s", is_cur ? '=> ' : '   ', lineno, line)
+    puts formatted
+  end
+end
+
+def print_listing(info)
+  snippet = get_snippet(info[:path], info[:lineno])
+  puts "Fiber: #{info[:fiber]} Location: #{info[:path]}:#{info[:lineno]}"
+  puts
+  print_snippet(info, snippet, info[:lineno])
+  puts
+end
+
+def print_help
+  puts
+  puts "Here's some help..."
+  puts
+end
+
+def print_state(info)
+  p info
+end
+
+def get_user_cmd
+  while true
+    STDOUT << "(pdbg) "
+    cmd = parse_command(STDIN.gets)
+    next unless cmd
+    
+    if cmd[:cmd] == :help
+      print_help
+    else
+      return cmd if cmd
+    end
+  end
+end
+
 socket.puts 'pdbg'
 response = socket.gets
 if response.chomp == 'pdbg'
   puts 'Connected to process'
 end
 loop do
-  status = socket.gets
-  puts status
+  info = socket.gets.chomp
+  display_info(info)
 
-  STDOUT << "> "
-  cmd = STDIN.gets
-  puts '-' * 40
-  socket.puts cmd
+  cmd = get_user_cmd
+  socket.puts cmd.inspect
 end

data/ext/polyphony/backend_common.c

@@ -31,6 +31,24 @@ inline void backend_base_mark(struct Backend_base *base) {
   runqueue_mark(&base->parked_runqueue);
 }
 
+void backend_base_reset(struct Backend_base *base) {
+  runqueue_finalize(&base->runqueue);
+  runqueue_finalize(&base->parked_runqueue);
+  
+  runqueue_initialize(&base->runqueue);
+  runqueue_initialize(&base->parked_runqueue);
+
+  base->currently_polling = 0;
+  base->op_count = 0;
+  base->switch_count = 0;
+  base->poll_count = 0;
+  base->pending_count = 0;
+  base->idle_gc_period = 0;
+  base->idle_gc_last_time = 0;
+  base->idle_proc = Qnil;
+  base->trace_proc = Qnil;  
+}
+
 const unsigned int ANTI_STARVE_SWITCH_COUNT_THRESHOLD = 64;
 
 inline void conditional_nonblocking_poll(VALUE backend, struct Backend_base *base, VALUE current, VALUE next) {

data/ext/polyphony/backend_common.h

@@ -32,6 +32,7 @@ struct Backend_base {
 void backend_base_initialize(struct Backend_base *base);
 void backend_base_finalize(struct Backend_base *base);
 void backend_base_mark(struct Backend_base *base);
+void backend_base_reset(struct Backend_base *base);
 VALUE backend_base_switch_fiber(VALUE backend, struct Backend_base *base);
 void backend_base_schedule_fiber(VALUE thread, VALUE backend, struct Backend_base *base, VALUE fiber, VALUE value, int prioritize);
 void backend_base_park_fiber(struct Backend_base *base, VALUE fiber);

data/ext/polyphony/backend_io_uring.c

@@ -106,9 +106,7 @@ VALUE Backend_post_fork(VALUE self) {
   io_uring_queue_exit(&backend->ring);
   io_uring_queue_init(backend->prepared_limit, &backend->ring, 0);
   context_store_free(&backend->store);
-  backend->base.currently_polling = 0;
-  backend->base.pending_count = 0;
-  backend->pending_sqes = 0;
+  backend_base_reset(&backend->base);
 
   return self;
 }
@@ -1248,7 +1246,7 @@ VALUE Backend_chain(int argc,VALUE *argv, VALUE self) {
     }
     
     io_uring_sqe_set_data(last_sqe, ctx);
-    unsigned int flags = (i == argc - 1) ? IOSQE_ASYNC : IOSQE_ASYNC & IOSQE_IO_LINK;
+    unsigned int flags = (i == (argc - 1)) ? IOSQE_ASYNC : IOSQE_ASYNC | IOSQE_IO_LINK;
     io_uring_sqe_set_flags(last_sqe, flags);
     sqe_count++;
   }
@@ -1528,10 +1526,6 @@ void Init_Backend() {
   rb_define_method(cBackend, "waitpid", Backend_waitpid, 1);
   rb_define_method(cBackend, "write", Backend_write_m, -1);
 
-  #ifdef POLYPHONY_UNSET_NONBLOCK
-  ID_ivar_is_nonblocking = rb_intern("@is_nonblocking");
-  #endif
-
   SYM_io_uring = ID2SYM(rb_intern("io_uring"));
   SYM_send = ID2SYM(rb_intern("send"));
   SYM_splice = ID2SYM(rb_intern("splice"));

data/ext/polyphony/backend_libev.c

@@ -157,6 +157,8 @@ VALUE Backend_post_fork(VALUE self) {
   ev_loop_destroy(backend->ev_loop);
   backend->ev_loop = EV_DEFAULT;
 
+  backend_base_reset(&backend->base);
+
   return self;
 }
 

data/lib/polyphony.rb

@@ -124,6 +124,6 @@ Polyphony.install_at_exit_handler
 
 if (debug_socket_path = ENV['POLYPHONY_DEBUG_SOCKET_PATH'])
   puts "Starting debug server on #{debug_socket_path}"
-  require 'polyphony/debugger/server'
-  Polyphony::DebugServer.start(debug_socket_path)
+  require 'polyphony/debugger'
+  Polyphony.start_debug_server(debug_socket_path)
 end

data/lib/polyphony/debugger.rb

@@ -0,0 +1,225 @@
+# frozen_string_literal: true
+
+require 'polyphony/extensions/debug'
+
+module Polyphony
+  TP_EVENTS = [
+    :line,
+    :call,
+    :return,
+    :b_call,
+    :b_return
+  ]    
+
+  def self.start_debug_server(socket_path)
+    server = DebugServer.new(socket_path)
+    controller = DebugController.new(server)      
+    trace = TracePoint.new(*TP_EVENTS) { |tp| controller.handle_tp(trace, tp) }
+    trace.enable
+
+    at_exit do
+      Kernel.trace "program terminated"
+      trace.disable
+      server.stop
+    end
+  end
+
+  class DebugController
+    def initialize(server)
+      @server = server
+      @server.wait_for_client
+      @state = { fibers: {} }
+      @control_fiber = Fiber.new { |f| control_loop(f) }
+      @control_fiber.transfer Fiber.current
+    end
+
+    def control_loop(source_fiber)
+      @peer = source_fiber
+      cmd = { cmd: :initial }
+      loop do
+        cmd = send(:"cmd_#{cmd[:cmd]}", cmd)
+      end
+    end
+
+    POLYPHONY_LIB_DIR = File.expand_path('..', __dir__)
+
+    def get_next_trace_event
+      @peer.transfer.tap { |e| update_state(e) }
+    end
+
+    def update_state(event)
+      trace update_state: event
+      @state[:fiber] = event[:fiber]
+      @state[:path] = event[:path]
+      @state[:lineno] = event[:lineno]
+      update_fiber_state(event)
+    end
+
+    def update_fiber_state(event)
+      fiber_state = @state[:fibers][event[:fiber]] ||= { stack: [] }
+      case event[:kind]
+      when :call, :c_call, :b_call
+        fiber_state[:stack] << event
+      when :return, :c_return, :b_return
+        fiber_state[:stack].pop
+      end
+      fiber_state[:binding] = event[:binding]
+      fiber_state[:path] = event[:path]
+      fiber_state[:lineno] = event[:lineno]
+    end
+
+    def state_presentation(state)
+      {
+        fiber:  fiber_id(state[:fiber]),
+        path:   state[:path],
+        lineno: state[:lineno]
+      }
+    end
+
+    def fiber_id(fiber)
+      {
+        object_id: fiber.object_id,
+        tag: fiber.tag
+      }
+    end
+
+    def fiber_representation(fiber)
+      {
+        object_id: fiber.object_id,
+        tag: fiber.tag,
+        parent: fiber.parent && fiber_id(fiber.parent),
+        children: fiber.children.map { |c| fiber_id(c) }
+      }
+    end
+
+    def get_next_command(info)
+      @server.get_command(info)
+    end
+
+    def cmd_initial(cmd)
+      get_next_command(nil)
+    end
+
+    def info_listing(state)
+      {
+        kind: :listing,
+        fiber: fiber_id(state[:fiber]),
+        path: state[:path],
+        lineno: state[:lineno]
+      }
+    end
+
+    def info_state(state)
+      info_listing(state).merge(
+        kind: :state,
+        fibers: info_fiber_states(state[:fibers])
+      )
+    end
+
+    def info_fiber_states(fiber_states)
+      fiber_states.inject({}) do |h, (f, s)|
+        h[fiber_id(f)] = {
+          stack: s[:stack].map { |e| { path: e[:path], lineno: e[:lineno] } }
+        }
+        h
+      end
+    end
+    
+    def cmd_step(cmd)
+      tp = nil
+      fiber = nil
+      while true
+        event = get_next_trace_event
+        @peer = event[:fiber]
+        if event[:kind] == :line && event[:path] !~ /#{POLYPHONY_LIB_DIR}/
+          return get_next_command(info_listing(@state))
+        end
+      end
+    rescue => e
+      trace "Uncaught error: #{e.inspect}"
+      @trace&.disable
+    end
+
+    def cmd_help(cmd)
+      get_next_command(kind: :help)
+    end
+
+    def cmd_list(cmd)
+      get_next_command(info_listing(@state))
+    end
+
+    def cmd_state(cmd)
+      get_next_command(info_state(@state))
+    end
+
+    def handle_tp(trace, tp)
+      return if Thread.current == @server.thread
+      return if Fiber.current == @control_fiber
+
+      kind = tp.event
+      event = {
+        fiber: Fiber.current,
+        kind: kind,
+        path: tp.path,
+        lineno: tp.lineno,
+        binding: tp.binding
+      }
+      case kind
+      when :call, :c_call, :b_call
+        event[:method_id] = tp.method_id
+        event[:parameters] = tp.parameters
+      when :return, :c_return, :b_return
+        event[:method_id] = tp.method_id
+        event[:return_value] = tp.return_value
+      end
+      @control_fiber.transfer(event)
+    end
+  end
+
+  class DebugServer
+    attr_reader :thread
+
+    def initialize(socket_path)
+      @socket_path = socket_path
+      @fiber = Fiber.current
+      start_server_thread
+    end
+
+    def start_server_thread
+      @thread = Thread.new do
+        puts("Listening on #{@socket_path}")
+        FileUtils.rm(@socket_path) if File.exists?(@socket_path)
+        socket = UNIXServer.new(@socket_path)
+        loop do
+          @client = socket.accept
+        end
+      end
+    end
+
+    def stop
+      @thread.kill
+    end
+
+    def handle_client(client)
+      @client = client
+    end
+
+    def wait_for_client
+      sleep 0.1 until @client
+      msg = @client.gets
+      @client.puts msg
+    end
+
+    def get_command(info)
+      @client&.orig_write "#{info.inspect}\n"
+      cmd = @client&.orig_gets&.chomp
+      eval(cmd)
+    rescue SystemCallError
+      nil
+    rescue => e
+      trace "Error in interact_with_client: #{e.inspect}"
+      e.backtrace[0..3].each { |l| trace l }
+      @client = nil
+    end
+  end
+end