polyphony 0.29 → 0.30

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fb34882b5a9cb5bbf2b1b27909795f96e5291fe953aeea53f6ebd4e95d8bd19f
-  data.tar.gz: c38af8c17d62b70fed44988182e70caba305eb9abbb3c4d1a8fdeda349031bae
+  metadata.gz: cf1cef97b80aa22506d7b5d1547062cfb62909ef12411fe0877ff3972f00198f
+  data.tar.gz: 5418b1912ea214600fbbc435e91f56e6db325dd21710530932fef944b27fc7ab
 SHA512:
-  metadata.gz: ab9f3c212357aa7eedaee5f43c1b85ff28337e8bb1cc3d693fcc72ff02c49ddf55b211daccfd0eb516cfca1544e02312f10f5e59eb0903c55279cc5983a69ca3
-  data.tar.gz: 9d871f6e1c8013a13fc03f191d6e0ac7fb7766e723c73be3afcf2d870cea9595bc750c4d45c7efb1a61104ef0611e00a430baae6b3a0c5afcd4b2a859c8677f5
+  metadata.gz: f878c45416276bb13e40c8641ab614d75c47f82b82859ab21fa37ae7012e487d040919b50a8f588d097fa2ab95295f20b79a91ab5ae4ec8177f0846c6cbd3892
+  data.tar.gz: 9d97c148ea0f54e15d1c55209a53131d09278fa7c8dd426c1f3d945ba26f86c0ffebb056dde9f4e324719a3dc424afcde2bbf99cddbadfe781bc9e1fab965ef9

data/.gitignore

@@ -53,6 +53,7 @@ test.rb
 .vscode
 
 lib/gyro_ext.bundle
+lib/gyro_ext.so
 
 _site
 .sass-cache

data/CHANGELOG.md

@@ -1,3 +1,17 @@
+0.30 2020-14-02
+---------------
+
+* Add support for awaiting a fiber from multiple monitor fibers at once
+* Implemented child fibers
+* Fix TERM and INT signal handling (close #11)
+* Fix compiling on Linux
+* Do not reset runnable value in Gyro_suspend (prevents interrupting timers)
+* Don't snooze when stopping a fiber
+* Fix IO#read for files larger than 8KB (#10)
+* Fix fiber messaging in main fiber
+* Prevent signalling of inactive async watcher
+* Better fiber messaging
+
 0.29 2020-02-02
 ---------------
 

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    polyphony (0.29)
+    polyphony (0.30)
       modulation (~> 1.0)
 
 GEM

data/TODO.md

@@ -1,15 +1,20 @@
-## 0.29 Multithreaded fiber scheduling - some rough corners
+## 0.31 Working Sinatra application
+
+- Accept rate/interval in `spin_loop` and `spin_worker_loop`:
+
+  ```ruby
+  spin_loop(10) { ... } # 10 times per second
+  spin_loop(rate: 10) { ... } # 10 times per second
+  spin_loop(interval: 10) { ... } # once every ten seconds
+  ```
 
 - Docs: explain difference between `sleep` and `suspend`
 - Check why first call to `#sleep` returns too early in tests. Check the
   sleep behaviour in a spawned thread.
-
-## 0.30 Working Sinatra application
-
 - app with database access (postgresql)
 - benchmarks!
 
-## 0.31 Sidekick
+## 0.32 Sidekick
 
 Plan of action:
 
@@ -17,13 +22,13 @@ Plan of action:
 - test performance
 - proceed from there
 
-## 0.32 Testing && Docs
+## 0.33 Testing && Docs
 
 - Pull out redis/postgres code, put into new `polyphony-xxx` gems
 
-## 0.33 Integration
+## 0.34 Integration
 
-## 0.34 Real IO#gets and IO#read
+## 0.35 Real IO#gets and IO#read
 
 - More tests
 - Implement some basic stuff missing:
@@ -33,11 +38,11 @@ Plan of action:
   - `IO.foreach`
   - `Process.waitpid`
 
-## 0.35 Rails
+## 0.36 Rails
 
 - Rails?
 
-## 0.36 DNS
+## 0.37 DNS
 
 ### DNS client
 

data/docs/getting-started/tutorial.md

@@ -5,7 +5,7 @@ nav_order: 2
 parent: Getting Started
 permalink: /getting-started/tutorial/
 prev_title: Installing Polyphony
-next_title: Design Principles
+next_title: Concurrency the Easy Way
 ---
 # A Gentle Introduction to Polyphony
 
@@ -332,7 +332,7 @@ fiber.raise 'foo'
 ```
 
 For more information on how exceptions are handled in Polyphony, see [exception
-handling](../../technical-overview/exception-handling/).
+handling](../../main-concepts/exception-handling/).
 
 ## Supervising - controlling multiple fibers at once
 
@@ -441,6 +441,6 @@ possibilities for Ruby. Polyphony has the performance characteristics and
 provides the necessary tools for transforming how concurrent Ruby apps are
 written. Polyphony is still new, and the present documentation is far from being
 complete. To learn more about Polyphony, read the [technical
-overview](../../technical-overview/design-principles/). For more examples please
+overview](../../main-concepts/design-principles/). For more examples please
 consult the [Github
 repository](https://github.com/digital-fabric/polyphony/tree/master/examples).

data/docs/index.md

@@ -78,10 +78,9 @@ Polyphony draws inspiration from the following, in no particular order:
 
 ## Going further
 
-To learn more about using Polyphony to build concurrent applications, read the
-technical overview below, or look at the [included
+To learn more about using Polyphony to build concurrent applications, continue reading, or look at the [bundled
 examples](https://github.com/digital-fabric/polyphony/tree/9e0f3b09213156bdf376ef33684ef267517f06e8/examples/README.md).
-A thorough reference is forthcoming.
+A thorough API reference is forthcoming.
 
 ## Contributing to Polyphony
 

data/docs/{technical-overview → main-concepts}/concurrency.md

@@ -1,10 +1,10 @@
 ---
 layout: page
 title: Concurrency the Easy Way
-nav_order: 2
-parent: Technical Overview
-permalink: /technical-overview/concurrency/
-prev_title: Design Principles
+nav_order: 1
+parent: Main Concepts
+permalink: /main-concepts/concurrency/
+prev_title: A Gentle Introduction to Polyphony
 next_title: How Fibers are Scheduled
 ---
 # Concurrency the Easy Way
@@ -65,10 +65,64 @@ understand, compared to callback-style programming.
 
 Polyphony extends the core `Fiber` class with additional functionality that
 allows scheduling, synchronizing, interrupting and otherwise controlling running
-fibers. Polyphony makes sure any exception raised while a fiber is running is
-[handled correctly](exception-handling.md). Moreover, fibers can communicate
-with each other using message passing, turning them into autonomous actors in a
-highly concurrent environment.
+fibers. Starting a concurrent operation inside a fiber is as simple as a `spin`
+method call:
+
+```ruby
+while (connection = server.accept)
+  spin { handle_connection(connection) }
+end
+```
+
+In order to facilitate developing applications that employ complex concurrent
+patterns and can scale easily, Polyphony employs a [structured
+approach](https://en.wikipedia.org/wiki/Structured_concurrency) to controlling
+fiber lifetime. A spun fiber is considered the *child* of the fiber from which
+it was spun, and is always limited to the life time of its parent:
+
+```ruby
+parent = spin do
+  do_something
+  child = spin do
+    do_some_other_stuff
+  end
+  # the child fiber is guaranteed to stop executing before the parent fiber
+  # terminates
+end
+```
+
+Any uncaught exception raised in a fiber will be
+[propagated]((exception-handling.md)) to its parent, and potentially further up
+the fiber hierarchy, all the way to the main fiber:
+
+```ruby
+parent = spin do
+  child = spin do
+    raise 'foo'
+  end
+  sleep
+end
+
+sleep
+# the exception will be propagated from the child fiber to the parent fiber,
+# and from the parent fiber to the main fiber, which will cause the program to
+# abort.
+```
+
+In addition, fibers can communicate with each other using message passing,
+turning them into autonomous actors in a highly concurrent environment. Message
+passing is in many ways a superior way to pass data between concurrent entities,
+obviating the need to synchronize access to shared resources:
+
+```ruby
+writer = spin do
+  while (write_request = receive)
+    do_write(write_request)
+  end
+end
+...
+writer << { stamp: Time.now, value: rand }
+```
 
 ## Higher-Order Concurrency Constructs
 
@@ -80,19 +134,13 @@ Cancel scopes \(borrowed from the brilliant Python library
 [Trio](https://trio.readthedocs.io/en/stable/)\) allows cancelling ongoing
 operations for any reason with more control over cancelling behaviour.
 
-Supervisors allow controlling multiple fibers. They offer enhanced exception
-handling and can be nested to create complex supervision trees ala
-[Erlang](https://adoptingerlang.org/docs/development/supervision_trees/).
-
 Some other constructs offered by Polyphony:
 
 * `Mutex` - a mutex used to synchronize access to a single shared resource.
 * `ResourcePool` - used for synchronizing access to a limited amount of shared 
-
   resources, for example a pool of database connections.
 
 * `Throttler` - used for throttling repeating operations, for example throttling
-
   access to a shared resource, or throttling incoming requests.
 
 ## A Compelling Concurrency Solution for Ruby
@@ -106,4 +154,3 @@ way to write concurrent applications in Ruby. Polyphony aims to show that Ruby
 can be used for developing sufficiently high-performance applications, while
 offering all the advantages of Ruby, with source code that is easy to read and
 understand.
-

data/docs/{technical-overview → main-concepts}/design-principles.md

@@ -1,16 +1,26 @@
 ---
 layout: page
 title: Design Principles
-nav_order: 1
-parent: Technical Overview
-permalink: /technical-overview/design-principles/
-prev_title: A Gentle Introduction to Polyphony
-next_title: Concurrency the Easy Way
+nav_order: 5
+parent: Main Concepts
+permalink: /main-concepts/design-principles/
+prev_title: Extending Polyphony
 ---
 # Design Principles
 
-Polyphony was created in order to enable creating high-performance concurrent
-applications in Ruby, by utilizing Ruby fibers together with the
+Polyphony was created in order to enable developing high-performance concurrent
+applications in Ruby using a fluent, compact syntax and API. Polyphony enables
+fine-grained concurrency - the splitting up of operations into a large number of
+concurrent tasks, each concerned with small part of the whole and advancing at
+its own pace.
+
+
+
+
+a single Ruby process may spin up millions of
+concurrent fibers.
+
+, by utilizing Ruby fibers together with the
 [libev](http://pod.tst.eu/http://cvs.schmorp.de/libev/ev.pod) event reactor
 library. Polyphony's design is based on the following principles:
 
@@ -57,8 +67,11 @@ library. Polyphony's design is based on the following principles:
   }
   ```
 
+- Breaking up operations into 
+
 - Polyphony should embrace Ruby's standard `raise/rescue/ensure` exception
-  handling mechanism:
+  handling mechanism. Exception handling in a highly concurrent environment
+  should be robust and foolproof:
 
   ```ruby
   cancel_after(0.5) do

data/docs/{technical-overview → main-concepts}/exception-handling.md

@@ -1,21 +1,21 @@
 ---
 layout: page
 title: Exception Handling
-nav_order: 4
-parent: Technical Overview
-permalink: /technical-overview/exception-handling/
+nav_order: 3
+parent: Main Concepts
+permalink: /main-concepts/exception-handling/
 prev_title: How Fibers are Scheduled
 next_title: Extending Polyphony
 ---
 # Exception Handling
 
 Ruby employs a pretty robust exception handling mechanism. An raised exception
-will bubble up the call stack until a suitable exception handler is found, based
-on the exception's class. In addition, the exception will include a stack trace
-showing the execution path from the exception's locus back to the program's
-entry point. Unfortunately, when exceptions are raised while switching between
-fibers, stack traces will only include partial information. Here's a simple
-demonstration:
+will propagate up the fiber tree until a suitable exception handler is found,
+based on the exception's class. In addition, the exception will include a stack
+trace showing the execution path from the exception's locus back to the
+program's entry point. Unfortunately, when exceptions are raised while switching
+between fibers, stack traces will only include partial information. Here's a
+simple demonstration:
 
 _fiber\_exception.rb_
 
@@ -124,7 +124,7 @@ fired or not. We call `timer.stop` inside an ensure block, thus ensuring that
 the timer will have stopped once the awaiting fiber has resumed, even if it has
 not fired.
 
-## Bubbling Up - A Robust Solution for Uncaught Exceptions
+## Exception Propagation
 
 One of the "annoying" things about exceptions is that for them to be useful, you
 have to intercept them \(using `rescue`\). If you forget to do that, you'll end
@@ -132,9 +132,9 @@ up with uncaught exceptions that can wreak havoc. For example, by default a Ruby
 `Thread` in which an exception was raised without being caught, will simply
 terminate with the exception silently swallowed.
 
-To prevent the same from happening with fibers, Polyphony provides a mechanism
-that lets uncaught exceptions bubble up through the chain of calling fibers.
-Let's discuss the following example:
+To prevent the same from happening with fibers, Polyphony provides a robust
+mechanism that propagates uncaught exceptions up through the chain of parent
+fibers. Let's discuss the following example:
 
 ```ruby
 require 'polyphony'
@@ -144,17 +144,23 @@ spin do
     spin do
       spin do
         raise 'foo'
-      end.await
-    end.await
-  end.await
-end.await
+      end
+      sleep
+    end
+    sleep
+  end
+  sleep
+end
+
+sleep
 ```
 
-In this example, there are four fibers, nested one within the other. An
-exception is raised in the inner most fiber, and having no exception handler,
-will bubble up through the different enclosing fibers, until reaching the
+In the above example, four nested fibers are created, and each of them, except
+for the innermost fiber, goes to sleep for an unlimited duration. An exception
+is raised in the innermost fiber, and having no corresponding exception handler,
+will propagate up through the enclosing fibers, until reaching the
 top-most level, that of the root fiber, at which point the exception will cause
-the program to halt and print an error message.
+the program to abort and print an error message.
 
 ## MoveOn and Cancel - Interrupting Fiber Execution
 
@@ -163,8 +169,8 @@ provides two exception classes that used exclusively to interrupt fiber
 execution: `MoveOn` and `Cancel`. Both of these classes are used in various
 fiber-control APIs, and `MoveOn` exceptions in particular are handled in a
 particular manner by Polyphony. The difference between `MoveOn` and `Cancel` is
-that `MoveOn` stops fiber execution without the exception bubbling up. It can
-optionally provide an arbitrary return value for the fiber. `Cancel` will bubble
+that `MoveOn` stops fiber execution without the exception propagating. It can
+optionally provide an arbitrary return value for the fiber. `Cancel` will propagate
 up like all exceptions.
 
 The `MoveOn` and `Cancel` classes are normally used indirectly, through the
@@ -183,22 +189,58 @@ f3 = spin { sleep 100 }
 f3.cancel #=> will raise a Cancel exception
 ```
 
-## Signal Handling and Termination
+In addition to `MoveOn` and `Cancel`, Polyphony employs internally another
+exception class, `Terminate` for terminating a fiber once its parent has
+finished executing.
+
+## The Special Problem of Signal Handling
+
+Ruby by default handles process signals by generating exceptions, allowing the
+handling of signals in a structured manner. However, process signals may arrive
+at any moment, and may be trapped while any arbitrary fiber is running, and even
+while an event loop is running.
 
-Polyphony does not normally intercept process signals, though it is possible to
-intercept them using `Gyro::Signal` watchers. It is, however, recommended for
-the time being to not interfere with Ruby's normal signal processing.
+Two signals in particular require special care as they involve the stopping of
+the entire process: `TERM` and `INT`. The `TERM` signal should be handled
+gracefully, i.e. with proper cleanup, which also means terminating all fibers.
+The `INT` signal requires halting the process and printing a correct stack
+trace.
+
+To ensure correct behaviour for these two signals, polyphony installs signal
+handlers that ensure that the main thread's event loop stops if it's currently
+running, and that the corresponding exceptions (namely `SystemExit` and
+`Interrupt`) are handled correctly by propagating them using Polyphony's normal
+exception handling mechanisms.
+
+Care should be taken when handling other signals. There are two options for
+correctly handling the signals: using Ruby's stock `trap` method, and using
+Polyphony's signal watchers. The stock method involves trapping signals as
+usual, but making sure we're not inside the event loop:
+
+```ruby
+trap('SIGHUP') do
+  Thread.current.break_out_of_ev_loop(nil)
+  handle_hup_signal
+end
+```
+
+The alternative is to use `Polyphony.wait_for_signal`:
+
+```ruby
+hup_handler = spin_loop do
+  Polyphony.wait_for_signal
+  handle_hup_signal
+end
+```
 
-In Ruby there are three core exception classes are related to signal handling
-and process termination: `Interrupt` - raised upon receiving an `INT` signal;
-`SystemExit` - raised upon calling `Kernel#exit`; and `SignalException` - raised
-upon receiving other signals.
+## The Special Problem of Thread Termination
 
-These exceptions are raised on the main thread and in a multi-fiber environment
-can occur in any fiber, as long as it is the currently running fiber. In
-Polyphony, when these exceptions are raised in a fiber other than the main
-fiber, they will be effectively tranferred to the main fiber for processing.
+Thread termination using `Thread#kill` or `Thread#raise` also presents the same
+problems as signal handling in a multi-fiber environment. The termination can
+occur while any fiber is running, and even while running the thread's event
+loop.
 
-This means that any handlers for these three exception classes should be put
-only in the main fiber. This mechanism also helps with showing a correct
-backtrace for these exceptions.
+To ensure proper thread termination, including the termination of all the
+thread's fibers, Polyphony patches the `Thread#kill` and `Thread#raise` methods
+to schedule the thread's main fiber with the corresponding exceptions, thus
+ensuring an orderly termination or exception handling.

data/docs/{technical-overview → main-concepts}/extending.md

@@ -1,10 +1,11 @@
 ---
 layout: page
 title: Extending Polyphony
-nav_order: 5
-parent: Technical Overview
-permalink: /technical-overview/extending/
+nav_order: 4
+parent: Main Concepts
+permalink: /main-concepts/extending/
 prev_title: Exception Handling
+next_title: Design Principles
 ---
 # Extending Polyphony
 

data/docs/{technical-overview → main-concepts}/fiber-scheduling.md

@@ -1,9 +1,9 @@
 ---
 layout: page
 title: How Fibers are Scheduled
-nav_order: 3
-parent: Technical Overview
-permalink: /technical-overview/fiber-scheduling/
+nav_order: 2
+parent: Main Concepts
+permalink: /main-concepts/fiber-scheduling/
 prev_title: Concurrency the Easy Way
 next_title: Exception Handling
 ---

data/docs/{technical-overview.md → main-concepts.md}

@@ -1,10 +1,10 @@
 ---
 layout: page
-title: Technical Overview
+title: Main Concepts
 description: Lorem ipsum
 has_children: true
 section: true
 has_toc: false
 nav_order: 3
-section_link: /technical-overview/design-principles
+section_link: /main-concepts/concurrency
 ---

data/examples/core/xx-at_exit.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'polyphony'
+
+Exception.__disable_sanitized_backtrace__ = true
+
+child_pid = Polyphony.fork do
+  at_exit do
+    puts "at_exit"
+    f = spin { sleep 10 }
+    trap('SIGINT') { f.stop }
+    f.await
+  end
+  
+  f1 = spin { sleep 100 }
+  
+  puts "pid: #{Process.pid}"
+  
+  pid = Process.pid
+  
+  f1.join
+end
+
+sleep 0.1
+Process.kill('INT', child_pid)
+sleep 0.1
+Process.kill('INT', child_pid)
+Process.wait(child_pid)

data/examples/core/xx-fork-terminate.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'polyphony'
+
+Exception.__disable_sanitized_backtrace__ = true
+
+pid = Polyphony.fork do
+  f = spin do
+    p 1
+    sleep 1
+    p 2
+  ensure
+    p 2.5
+  end
+  p 3
+  snooze
+  p 4
+  # f.stop
+  # f.join
+  # Fiber.current.terminate_all_children
+  # Fiber.current.await_all_children
+  p 5
+end
+
+puts "Child pid: #{pid}"
+Process.wait(pid)

data/examples/core/xx-pingpong.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'polyphony'
+
+pong = spin_loop do
+  msg, ping = receive
+  puts msg
+  ping << 'pong'
+end
+
+ping = spin_loop do
+  pong << ['ping', Fiber.current]
+  msg = receive
+  puts msg
+end
+
+suspend

data/examples/core/xx-stop.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'polyphony'
+
+f1 = spin do
+  f2 = spin { sleep 60 }
+  f3 = spin { sleep 60 }
+  sleep 60
+ensure
+  p 1
+  f2.stop
+  p 2
+  f3.stop
+  p "should reach here!"
+end
+
+sleep 0.1
+f1.stop
+snooze

data/ext/gyro/async.c

@@ -78,7 +78,7 @@ static VALUE Gyro_Async_signal(int argc, VALUE *argv, VALUE self) {
   struct Gyro_Async *async;
   GetGyro_Async(self, async);
 
-  if (!async->ev_loop) {
+  if (!async->active) {
     // printf("signal! called before await\n");
     return Qnil;
   }

data/ext/gyro/extconf.rb

@@ -16,8 +16,5 @@ $defs << "-DHAVE_SYS_RESOURCE_H" if have_header("sys/resource.h")
 
 CONFIG["optflags"] << " -fno-strict-aliasing" unless RUBY_PLATFORM =~ /mswin/
 
-CONFIG["optflags"] << " -Wcomment"
-CONFIG["optflags"] << " -Wbitwise-op-parentheses"
-
 dir_config "gyro_ext"
 create_makefile "gyro_ext"