polyphony 0.69 → 0.73

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1074fc28d6110cf489e8dce4fcc9c9a2430b557b57a23d865d4ac35c2bb14847
-  data.tar.gz: 4d0de60084fdc5eec62c25306e098a19a84e7d859547397eff70e673be2fab9b
+  metadata.gz: cfcb66ef2171cc3c4216bc674c38468ef4bd23333031ec77d61d74bc17b32c33
+  data.tar.gz: 215cf149bd26e9d3d8c64b5c6f63eeccad86c5b6dc9ed5289bb6a1537e51ab56
 SHA512:
-  metadata.gz: f2aa641bceb29837bedc2b8120109b05dac20afa32923b6e20bca9a3d9d6b54886ed1410915f24497a839866ff8a660573a847868747ff1c91aa4e0ba87b45c2
-  data.tar.gz: b49b24d6ea81f8c1a95402c9024102e0f999a079566af7667ad0ae5aab9d5fef4b934ae9711e447e7a0364d533ffe9071f41ce8309be425913606f3dad1e0860
+  metadata.gz: 9157512378e6edb0a362ea3db4fc35ecae3b6d2b54c025184a39ddf5e0e85528ac97aac1f3b829b461aa85e58a1655f35c7ad4d2aec0e8dda97c75d130625b34
+  data.tar.gz: f90758ee02b5dd28f488372d260b73413e4d01afa14756b8bfc787cd335c42ef43fa1b5907060cb33065727e804296bc6a25fee91d40b94e390e21c6b98d903e

data/.github/FUNDING.yml

@@ -0,0 +1 @@
+github: ciconia

data/.github/workflows/test.yml

@@ -16,7 +16,7 @@ jobs:
     runs-on: ${{matrix.os}}
     steps:
     - uses: actions/checkout@v1
-    - uses: actions/setup-ruby@v1
+    - uses: ruby/setup-ruby@v1
       with:
         ruby-version: ${{matrix.ruby}}
     - name: Install dependencies
@@ -24,7 +24,7 @@ jobs:
         gem install bundler
         bundle install
     - name: Show Linux kernel version
-      run: uname -r
+      run: uname -a
     - name: Compile C-extension
       run: POLYPHONY_USE_LIBEV=1 bundle exec rake compile
     - name: Run tests

data/.gitignore

@@ -58,4 +58,6 @@ lib/*.so
 _site
 .sass-cache
 
-log
+log
+
+.ruby-version

data/CHANGELOG.md

@@ -1,3 +1,32 @@
+## 0.73 2021-12-16
+
+- Refactor core extensions into separate files for each class
+- Improve compatibility with Ruby 3.1
+- Improve accuracy of `timer_loop`
+
+## 0.72 2021-11-22
+
+- Add support for supervising added child fibers
+- Do not use io_uring under LinuxKit (#66)
+- Fix error message in `Fiber#restart`
+- refactor `runqueue_ring_buffer_mark`
+- Fix setting up test server on random port
+- Enhance `Fiber#supervise`
+  - Add support for specifying `restart: true` (same as `restart: :always`)
+  - Add support for supervising child fibers (when fibers are not specified)
+- Add `Fiber#attach_all_children_to`
+- Improve `SSLServer#accept` when `servername_cb` is set
+- Fix `#supervise` in Ruby 3.0
+
+## 0.71 2021-08-20
+
+- Fix compilation on Ruby 3.0
+
+## 0.70 2021-08-19
+
+- New implementation for `#supervise`
+- Reset backend state and runqueue on fork
+
 ## 0.69 2021-08-16
 
 - Rename `#__polyphony_read_method__` to `#__parser_read_method__`
@@ -18,7 +47,7 @@
 ## 0.66 2021-08-01
 
 - Fix all splicing APIs on non-linux OSes (#63)
-- Add GC marking of buffers when cancelling read/write ops in io_uring backend 
+- Add GC marking of buffers when cancelling read/write ops in io_uring backend
 
 ## 0.65 2021-07-29
 
@@ -82,7 +111,7 @@
 ## 0.55.0 2021-06-17
 
 - Finish io_uring implementation of Backend#chain
-- Reimplement io_uring op_context acquire/release algorithm (using ref count) 
+- Reimplement io_uring op_context acquire/release algorithm (using ref count)
 - Fix #gets on sockets
 - Redesign event anti-starvation mechanism
 
@@ -177,7 +206,7 @@
 
 ## 0.47.0 2020-11-10
 
-- Implement `#spin_scope` used for creating blocking fiber scopes 
+- Implement `#spin_scope` used for creating blocking fiber scopes
 - Reimplement `move_on_after`, `cancel_after`, `Timeout.timeout` using
   `Backend#timeout` (avoids creating canceller fiber for most common use case)
 - Implement `Backend#timeout` API
@@ -272,7 +301,7 @@
 - Reimplement Event using `Agent#wait_event`
 - Improve Queue shift queue performance
 - Introduce `Agent#wait_event` API for waiting on asynchronous events
-- Minimize `fcntl` syscalls in IO operations 
+- Minimize `fcntl` syscalls in IO operations
 
 ## 0.43.7 2020-07-20
 

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    polyphony (0.69)
+    polyphony (0.73)
 
 GEM
   remote: https://rubygems.org/
@@ -75,4 +75,4 @@ DEPENDENCIES
   simplecov (= 0.17.1)
 
 BUNDLED WITH
-   2.2.3
+   2.3.0.dev

data/TODO.md

@@ -26,31 +26,8 @@
 
 - Add support for `close` to io_uring backend
 
-- Graceful shutdown again:
-  - What happens to children when doing a graceful shutdown?
-  - What are the implications of passing graceful shutdown flag to children?
-  - What about errors while doing a graceful shutdown?
-  - What about graceful restarts?
-  - Some interesting discussions:
-    - https://trio.discourse.group/search?q=graceful%20shutdown
-    - https://github.com/python-trio/trio/issues/147
-    - https://github.com/python-trio/trio/issues/143
-    - https://trio.discourse.group/t/graceful-shutdown/93/2
-    - https://250bpm.com/blog:146/
-    - https://www.rodrigoaraujo.me/posts/golang-pattern-graceful-shutdown-of-concurrent-events/
-    - https://github.com/tj/go-gracefully
-  - `Fiber#finalize_children` should pass graceful shutdown flag to children
-  - A good use case is an HTTP server that on graceful shutdown:
-    - stops listening
-    - waits for all ongoing requests to finish, optionally with a timeout
-
 ## Roadmap for Polyphony 1.0
 
-- check integration with rb-inotify
-
-- Improve `#supervise`. It does not work as advertised, and seems to exhibit an
-  inconsistent behaviour (see supervisor example).
-
 - Add test that mimics the original design for Monocrono:
   - 256 fibers each waiting for a message
   - When message received do some blocking work using a `ThreadPool`
@@ -130,7 +107,7 @@
 
       Hitting the breakpoint will show the current location in the source code
       (with few lines before and after), and present a prompt for commands.
-    
+
     - commands:
       - `step` / `up` / `skip` / `continue` etc. - step into, step out, step over, run
       - `switch` - switch fiber

data/bin/pdbg

@@ -89,7 +89,7 @@ def get_user_cmd
     STDOUT << "(pdbg) "
     cmd = parse_command(STDIN.gets)
     next unless cmd
-    
+
     if cmd[:cmd] == :help
       print_help
     else

data/docs/_user-guide/all-about-timers.md

@@ -44,7 +44,7 @@ The `#sleep` forever call can be used for example in the main fiber when we do
 all our work in other fibers, since once the main fiber terminates the program
 exits.
 
-## Doing Work Later 
+## Doing Work Later
 
 While `#sleep` allows you to block execution of the current fiber, sometimes you
 want to perform some work later, while not blocking the current fiber. This is done simply by spinning off another fiber:

data/docs/api-reference/exception.md

@@ -3,15 +3,19 @@ layout: page
 title: ::Exception
 parent: API Reference
 permalink: /api-reference/exception/
+source_url: https://github.com/digital-fabric/polyphony/blob/master/lib/polyphony/extensions/core.rb
+ruby_docs_url: https://rubyapi.org/3.0/o/exception
 ---
 # ::Exception
 
-[Ruby core Exception documentation](https://ruby-doc.org/core-2.7.0/Exception.html)
+[Ruby core Exception documentation](https://rubyapi.org/3.0/o/exception)
 
 The core `Exception` class is enhanced to provide a better backtrace that takes
 into account the fiber hierarchy. In addition, a `source_fiber` attribute allows
 tracking the fiber from which an uncaught exception was propagated.
 
+
+
 ## Class Methods
 
 ## Instance methods

data/docs/api-reference/fiber.md

@@ -205,7 +205,7 @@ f.raise
 Pops the first message from the fiber's mailbox. If no message is available,
 `#receive` will block until a message is pushed to the mailbox. The received
 message can be any kind of object. This method is complemented by
-`Fiber#<<`/`Fiber#send`. 
+`Fiber#<<`/`Fiber#send`.
 
 ```ruby
 spin { Fiber.current.parent << 'hello from child' }
@@ -328,7 +328,7 @@ Returns the fiber's current state, which can be any of the following:
 Supervises all child fibers, optionally restarting any fiber that terminates.
 
 The given `opts` argument controls the behaviour of the supervision. The
-following options are currently supported: 
+following options are currently supported:
 
 - `:restart`: restart options
   - `nil` - Child fibers are not restarted (default behaviour).

data/docs/faq.md

@@ -99,7 +99,7 @@ In conclusion:
 
 ## If callbacks suck, why not use promises?
 
-Promises have gained a lot of traction during the last few years as an  
+Promises have gained a lot of traction during the last few years as an
 alternative to callbacks, above all in the Javascript community. While promises
 have been at a certain point considered for use in Polyphony, they were not
 found to offer enough of a benefit. Promises still cause split logic, are quite

data/docs/getting-started/overview.md

@@ -114,7 +114,7 @@ single CPU core.
 
 Nevertheless, Polyphony fully supports multithreading, with each thread having
 its own fiber run queue and its own libev event loop. Polyphony even enables
-cross-thread communication using [fiber messaging](#message-passing). 
+cross-thread communication using [fiber messaging](#message-passing).
 
 ## Fibers vs Callbacks
 
@@ -138,7 +138,7 @@ Fibers, in contrast, let the developer express the business logic in a
 sequential, easy to read manner: do this, then that. State can be stored right
 in the business logic, as local variables. And finally, the sequential
 programming style makes it much easier to debug your code, since stack traces
-contain the entire history of execution from the app's inception. 
+contain the entire history of execution from the app's inception.
 
 ## Structured Concurrency
 
@@ -229,7 +229,7 @@ normally or with an exception:
 fiber1 = spin { sleep 1; raise 'foo' }
 fiber2 = spin { sleep 1 }
 
-supervise # blocks and then propagates the error raised in fiber1 
+supervise # blocks and then propagates the error raised in fiber1
 ```
 
 ## Message Passing
@@ -266,7 +266,7 @@ end
 Notice how the state (the `subscribers` variable) stays local, and how the logic
 of the chat room is expressed in a way that is both compact and easy to extend.
 Also notice how the chat room is written as an infinite loop. This is a common
-pattern in Polyphony, since fibers can always be stopped at any moment. 
+pattern in Polyphony, since fibers can always be stopped at any moment.
 
 The code for handling a chat room user might be expressed as follows:
 
@@ -350,14 +350,14 @@ operations.
 
 While a standard event loop-based solution would implement a blocking call
 separately from the fiber scheduling, the system backend integrates the two to
-create a blocking call that is already knows how to switch and schedule fibers.
+create a blocking call that already knows how to switch and schedule fibers.
 For example, in Polyphony all APIs having to do with reading from files or
 sockets end up calling `Thread.current.backend.read`, which does all the work.
 
 This design offers some major advantages over other designs. It minimizes memory
-allocations, of both Ruby objects and C structures. For example, instead of
+allocations of both Ruby objects and C structures. For example, instead of
 having to allocate libev watchers on the heap and then pass them around, they
-are allocated on the stack instead, which saves up on both memory and CPU cycles.
+are allocated on the stack instead, which saves both memory and CPU cycles.
 
 In addition, the backend interface includes two methods that allow maximizing
 server performance by accepting connections and reading from sockets in a tight
@@ -482,5 +482,5 @@ reach version 1.0. Here are some of the exciting directions we're working on.
 
 - Support for more core and stdlib APIs
 - More adapters for gems with C-extensions, such as `mysql`, `sqlite3` etc
-- Use `io_uring` backend as alternative to the libev backend 
+- Use `io_uring` backend as alternative to the libev backend
 - More concurrency constructs for building highly concurrent applications

data/docs/getting-started/tutorial.md

@@ -51,7 +51,7 @@ multiple threads or processes, but this approach has numerous disavantages:
 - Both threads and processes are limited to a few thousand at best on a single
   machine. Trying to spawn a thread per client essentially limits the scaling
   capacity of your system.
-  
+
 Polyphony eschews both threads and processes in favor of fibers as the basic
 unit of concurrency. The idea is that any time a blocking I/O operation occurs,
 the current fiber is paused, and another fiber which has been marked as
@@ -117,7 +117,7 @@ Here's the actual sequence of execution (in pseudo-code)
 sleeper = spin { ... } # The main fiber spins up a new fiber marked as runnable
 suspend # The main fiber suspends, waiting for all other work to finish
   Thread.current.switch_fiber # Polyphony looks for other runnable fibers
-  
+
   # (sleeper fiber)
   puts "Going to sleep..." # The sleeper fiber starts running
   sleep 1 # The sleeper fiber goes to sleep
@@ -280,7 +280,7 @@ def handle_client(client)
 rescue Polyphony::Cancel
   client.puts 'Closing connection due to inactivity.'
 rescue Polyphony::Terminate
-  # We add a handler for the Terminate exception, and give 
+  # We add a handler for the Terminate exception, and give
   client.puts 'Server is shutting down. You have 5 more seconds...'
   move_on_after(5) do
     client_loop(client)

data/docs/main-concepts/concurrency.md

@@ -133,7 +133,7 @@ fiber that's blocking on any arbitrary operation.
 Some other constructs offered by Polyphony:
 
 * `Mutex` - a mutex used to synchronize access to a single shared resource.
-* `ResourcePool` - used for synchronizing access to a limited amount of shared 
+* `ResourcePool` - used for synchronizing access to a limited amount of shared
   resources, for example a pool of database connections.
 * `Throttler` - used for throttling repeating operations, for example throttling
   access to a shared resource, or throttling incoming requests.

data/docs/main-concepts/extending.md

@@ -9,9 +9,9 @@ next_title: The Design of Polyphony
 ---
 # Extending Polyphony
 
-Polyphony was designed to ease the transition from blocking APIs and 
+Polyphony was designed to ease the transition from blocking APIs and
 callback-based API to non-blocking, fiber-based ones. It is important to
-understand that not all blocking calls can be easily converted into 
+understand that not all blocking calls can be easily converted into
 non-blocking calls. That might be the case with Ruby gems based on C-extensions,
 such as database libraries. In that case, Polyphony's built-in
 [thread pool](#threadpool) might be used for offloading such blocking calls.
@@ -22,7 +22,7 @@ Some of the most common patterns in Ruby APIs is the callback pattern, in which
 the API takes a block as a callback to be called upon completion of a task. One
 such example can be found in the excellent
 [http_parser.rb](https://github.com/tmm1/http_parser.rb/) gem, which is used by
-Polyphony itself to provide HTTP 1 functionality. The `HTTP:Parser` provides 
+Polyphony itself to provide HTTP 1 functionality. The `HTTP:Parser` provides
 multiple hooks, or callbacks, for being notified when an HTTP request is
 complete. The typical callback-based setup is as follows:
 

data/docs/main-concepts/fiber-scheduling.md

@@ -145,7 +145,7 @@ def with_timeout(duration)
     sleep duration
     interruptible_fiber.raise 'timeout'
   end
-  
+
   # do work
   yield
 ensure

data/examples/core/calc.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'polyphony'
+
+# Kernel#spin starts a new fiber
+@controller = spin do
+  @worker = spin do
+    loop do
+      # Each fiber has a mailbox for receiving messages
+      peer, op, x, y = receive
+      result = x.send(op, y)
+      # The result is sent back to the "client"
+      peer << result
+    end
+  end
+  # The controller fiber will block until the worker is done (but notice that
+  # the worker runs an infinite loop.)
+  @worker.await
+end
+
+def calc(op, x, y)
+  # Send the job to the worker fiber...
+  @worker << [Fiber.current, op, x, y]
+  # ... and wait for the result
+  receive
+end
+
+# wait for worker to start
+snooze until @worker
+
+p calc(:+, 2, 3)
+p calc(:**, 2, 3)
+p calc(:+, 2, nil)
+
+# wait for the controller to terminate
+@controller.await

data/examples/core/calc_with_restart.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'polyphony'
+
+# Kernel#spin starts a new fiber
+@controller = spin do
+  @worker = spin do
+    loop do
+      # Each fiber has a mailbox for receiving messages
+      peer, op, x, y = receive
+      result = x.send(op, y)
+      # The result is sent back to the "client"
+      peer << result
+    end
+  end
+  # The controller fiber will block until the worker is done (but notice that
+  # the worker runs an infinite loop.)
+  @worker.await
+rescue => e
+  puts "Uncaught exception in worker: #{e}. Restarting..."
+  @worker.restart
+end
+
+def calc(op, x, y)
+  # Send the job to the worker fiber...
+  @worker << [Fiber.current, op, x, y]
+  # ... and wait for the result
+  receive
+end
+
+# wait for worker to start
+snooze until @worker
+
+p calc(:+, 2, 3)
+p calc(:**, 2, 3)
+p calc(:+, 2, nil)
+
+# wait for the controller to terminate
+@controller.await

data/examples/core/calc_with_supervise.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'polyphony'
+
+# Kernel#spin starts a new fiber
+@controller = spin do
+  @worker = spin do
+    loop do
+      # Each fiber has a mailbox for receiving messages
+      peer, op, x, y = receive
+      result = x.send(op, y)
+      # The result is sent back to the "client"
+      peer << result
+    end
+  end
+  # The controller fiber will block until the worker is done (but notice that
+  # the worker runs an infinite loop.)
+  @worker.supervise(@worker, restart: :always)
+end
+
+def calc(op, x, y)
+  # Send the job to the worker fiber...
+  @worker << [Fiber.current, op, x, y]
+  # ... and wait for the result
+  receive
+end
+
+# wait for worker to start
+snooze until @worker
+
+p calc(:+, 2, 3)
+p calc(:**, 2, 3)
+p calc(:+, 2, nil)
+
+# wait for the controller to terminate
+@controller.await

data/examples/core/message_based_supervision.rb

@@ -45,7 +45,7 @@ def start_worker(id)
     sleep duration
     raise 'foo' if rand > 0.7
     break if rand > 0.6
-  end  
+  end
 end
 
 supervise(start_worker(1), start_worker(2))

data/examples/core/ring.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'polyphony'
+
+M = 100
+N = 10000
+
+GC.disable
+
+def monotonic_clock
+  ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+end
+
+def spin_proc(next_fiber)
+  spin_loop { next_fiber << receive }
+end
+
+last = Fiber.current
+N.times { last = spin_proc(last) }
+
+snooze
+t0 = monotonic_clock
+M.times do
+  last << 'hello'
+  receive
+end
+elapsed = monotonic_clock - t0
+puts "M=#{M} N=#{N} elapsed: #{elapsed}"

data/examples/io/rack_server.rb

@@ -14,7 +14,7 @@ module RackAdapter
     def env(req)
       {}
     end
-    
+
     def respond(socket, request, (status_code, headers, body))
       body = body.join
       headers = "Content-Type: text/plain\r\nContent-Length: #{body.bytesize}\r\n"

data/examples/io/tunnel.rb

@@ -21,7 +21,7 @@ def endpoint_loop(idx, peer_idx)
     '0.0.0.0',
     port,
     reuse_addr: true
-  ) 
+  )
   # server = TCPServer.open('0.0.0.0', port)
   log "Listening on port #{port}"
   loop do

data/examples/performance/fiber_transfer.rb

@@ -35,7 +35,7 @@ def run(num_fibers)
   end
 
   last.next = first
-  
+
   t0 = Time.now
   puts "start transfer..."
   first.transfer

data/examples/performance/line_splitting.rb

@@ -8,7 +8,7 @@ def slice
   while true
     idx = str.index("\n")
     break unless idx
-    
+
     lines << str.slice!(0, idx + 1)
   end
   raise unless lines.size == 4

data/examples/performance/thread-vs-fiber/compare.rb

@@ -27,7 +27,7 @@ def run_test(name, port, cmd, setting)
   puts "*" * 80
   puts "Run #{name} (#{port}): #{setting}"
   puts "*" * 80
-  
+
   pid = spawn("#{cmd} > /dev/null 2>&1")
   sleep 1