polyphony 0.44.0 → 0.45.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 04c1c1221d46b1b9169aa04c5896c8d68c20fa11d64554e76b7452710423f142
-  data.tar.gz: 519528e0a3dfd323620869d4a7e43e4769cf3c1b5f0ea405cf341efc6fd534f6
+  metadata.gz: fe5d86b07fc6f29d01897a7790deac8f0544bb61dbd486fb8ea104dd315b0e80
+  data.tar.gz: 3e1086b9395d63835a09051c52b7f8496013f190149df35532333b6c27b74169
 SHA512:
-  metadata.gz: de8ac460dd05b051057ffab1c155c6b2d675cc623b98549333ba1defbc99f4efb738ff4a34a9f8c14bd3cb9ab0cab5356aa34a2308daa73273a6ef8719565bc4
-  data.tar.gz: c26b254559e484774574555dc895cf072bda8061b92fed26d8e61e7172523fe0faf2750cd2f924db3a25b59583b673ebe5f14749fd6f1279ca2e7e7b7745490b
+  metadata.gz: d6c7a78a46b9084f60ee838e82b7cde6f287eea2c500f779e5014103a400da98dc066adef58631033591a8de5278cc4f7b95ba8dcdac3ad22ecd68668a7d68e4
+  data.tar.gz: d32e7def9aa63ecb5c53b46020079e3384809ff786478e0b87fce1b141abb85d3c69d1f490b1d0a3bcbfaf972a3779939ac42bec512b12132d3b46b159754de5

data/.rubocop.yml

@@ -81,6 +81,7 @@ Lint/SuppressedException:
     - examples/**/*.rb
 
 Metrics/MethodLength:
+  Max: 12
   Exclude:
     - lib/polyphony/http/server/rack.rb
     - lib/polyphony/extensions/io.rb
@@ -95,6 +96,7 @@ Metrics/ModuleLength:
 Metrics/ClassLength:
   Exclude:
     - lib/polyphony/http/server/http1.rb
+    - lib/polyphony/extensions/io.rb
     - test/**/*.rb
     - examples/**/*.rb
 
@@ -111,6 +113,7 @@ Style/Documentation:
   Exclude:
     - test/**/*.rb
     - examples/**/*.rb
+    - lib/polyphony/adapters/**/*.rb
 
 Style/FormatString:
   Exclude:
@@ -172,4 +175,8 @@ Style/RedundantRegexpEscape:
   Enabled: true
 
 Style/SlicingWithRange:
-  Enabled: true
+  Enabled: true
+
+Style/RaiseArgs:
+  Exclude:
+    - lib/polyphony/extensions/fiber.rb

data/CHANGELOG.md

@@ -1,3 +1,44 @@
+## 0.45.5
+
+* Fix compilation error (#43)
+* Add support for resetting move_on_after, cancel_after timeouts
+* Optimize anti-event starvation polling
+* Implement optimized runqueue for better performance
+* Schedule parent with priority on uncaught exception
+* Fix race condition in `Mutex#synchronize` (#41)
+
+## 0.45.4
+
+* Improve signal trapping mechanism
+
+## 0.45.3
+
+* Don't swallow error in `Process#kill_and_await`
+* Add `Fiber#mailbox` attribute reader
+* Fix bug in `Fiber.await`
+* Implement `IO#getc`, `IO#getbyte`
+
+## 0.45.2
+
+* Rewrite `Fiber#<<`, `Fiber#await`, `Fiber#receive` in C
+
+## 0.45.1
+
+* Fix Net::HTTP compatibility
+* Fix fs adapter
+* Improve performance of IO#puts
+* Mutex#synchronize
+* Fix Socket#connect
+* Cleanup code
+* Improve support for Ruby 3 keyword args
+
+## 0.45.0
+
+* Cleanup code
+* Rename `Agent` to `Backend`
+* Implement `Polyphony::ConditionVariable`
+* Fix Kernel.system
+
 ## 0.44.0 2020-07-25
 
 * Fix reentrant `ResourcePool` (#38)

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    polyphony (0.44.0)
+    polyphony (0.45.5)
 
 GEM
   remote: https://rubygems.org/
@@ -11,6 +11,7 @@ GEM
     ansi (1.5.0)
     ast (2.4.0)
     builder (3.2.4)
+    coderay (1.1.3)
     colorator (1.1.0)
     concurrent-ruby (1.1.6)
     docile (1.3.2)
@@ -22,7 +23,7 @@ GEM
     forwardable-extended (2.6.0)
     hiredis (0.6.3)
     http_parser.rb (0.6.0)
-    httparty (0.17.0)
+    httparty (0.17.1)
       mime-types (~> 3.0)
       multi_xml (>= 0.5.2)
     i18n (0.9.5)
@@ -60,11 +61,11 @@ GEM
     listen (3.2.1)
       rb-fsevent (~> 0.10, >= 0.10.3)
       rb-inotify (~> 0.9, >= 0.9.10)
-    localhost (1.1.4)
     mercenary (0.3.6)
+    method_source (1.0.0)
     mime-types (3.3.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2019.1009)
+    mime-types-data (3.2020.0512)
     minitest (5.13.0)
     minitest-reporters (1.4.2)
       ansi
@@ -79,10 +80,14 @@ GEM
     pathutil (0.16.2)
       forwardable-extended (~> 2.6)
     pg (1.1.4)
+    pry (0.13.1)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
     public_suffix (4.0.3)
+    rack (2.2.3)
     rainbow (3.0.0)
     rake (12.3.3)
-    rake-compiler (1.0.5)
+    rake-compiler (1.1.1)
       rake
     rb-fsevent (0.10.3)
     rb-inotify (0.10.1)
@@ -124,18 +129,19 @@ PLATFORMS
 DEPENDENCIES
   hiredis (= 0.6.3)
   http_parser.rb (~> 0.6.0)
-  httparty (= 0.17.0)
+  httparty (= 0.17.1)
   jekyll (~> 3.8.6)
   jekyll-remote-theme (~> 0.4.1)
   jekyll-seo-tag (~> 2.6.1)
   just-the-docs (~> 0.3.0)
-  localhost (= 1.1.4)
   minitest (= 5.13.0)
   minitest-reporters (= 1.4.2)
   mysql2 (= 0.5.3)
   pg (= 1.1.4)
   polyphony!
-  rake-compiler (= 1.0.5)
+  pry (= 0.13.1)
+  rack (>= 2.0.8, < 2.3.0)
+  rake-compiler (= 1.1.1)
   redis (= 4.1.0)
   rubocop (= 0.85.1)
   sequel (= 5.34.0)

data/Rakefile

@@ -20,7 +20,7 @@ task :stress_test do
 end
 
 task :docs do
-  exec 'RUBYOPT=-W0 jekyll serve -s docs -H ec2-35-158-110-38.eu-central-1.compute.amazonaws.com'
+  exec 'RUBYOPT=-W0 jekyll serve -s docs -H ec2-18-156-117-172.eu-central-1.compute.amazonaws.com'
 end
 
 CLEAN.include "**/*.o", "**/*.so", "**/*.bundle", "**/*.jar", "pkg", "tmp"

data/TODO.md

@@ -1,10 +1,15 @@
-- Adapter for Pry, or maybe just an adapter for readline
-- Rename Agent to Backend
-- Review all code
-  - Cleanup C code
-  - Cleanup Ruby code (use Rubocop)
-  - Cleanup and annotate examples (and remove all the examples used for
-    debugging). Focus on examples that serve as "how-to".
+- io_uring
+
+0.46
+
+- Adapter for io/console (what does `IO#raw` do?)
+- Adapter for Pry and IRB (Which fixes #5 and #6)
+- Improve `#supervise`. It does not work as advertised, and seems to exhibit an
+  inconsistent behaviour (see supervisor example).
+- Fix backtrace for `Timeout.timeout` API (see timeout example).
+- Check why worker-thread example doesn't work.
+
+0.47
 
 - Debugging
   - Eat your own dogfood: need a good tool to check what's going on when some
@@ -118,8 +123,6 @@
   - discuss using `snooze` for ensuring responsiveness when executing CPU-bound work
 
 
-## 0.45
-
 ### Some more API work, more docs
 
 - sintra app with database access (postgresql)
@@ -131,14 +134,10 @@
   - proceed from there
 
 
-## 0.46
-
 ### Sinatra / Sidekiq
 
 - Pull out redis/postgres code, put into new `polyphony-xxx` gems
 
-## 0.47
-
 ### Testing && Docs
 
 - More tests
@@ -149,8 +148,6 @@
   - `IO.foreach`
   - `Process.waitpid`
 
-## 0.48 DNS
-
 ### DNS client
 
 ```ruby

data/docs/_posts/2020-07-26-polyphony-0.44.md

@@ -0,0 +1,77 @@
+# Polyphony 0.44.1
+
+## More performance, more compatibility, more robustness
+
+The last three weeks have been very busy for Polyphony. Since I first presented
+Polyphony here and elsewhere, 17 issues were closed, 10 pull requests were
+merged, and 144 commits were made by 4 different authors. I'm really
+excited about Polyphony and the momentum it seems to be gathering. Your
+reactions have been very positive so far (it even got [tweeted by
+Matz!](https://twitter.com/yukihiro_matz/status/1279289318083715073))
+
+I'm even more excited about the contributions Polyphony is starting to get from
+other developers. Thank you [Will](https://github.com/wjordan),
+[Máximo](https://github.com/ElMassimo) and [Trent](https://github.com/misfo) for
+your valuable contributions! Also, the Polyphony project has now got a logo
+designed by my friend [Gérald Morales](https://webocube.com/).
+
+I'd like to encourage other developers to get in on the action and start
+contributing by testing Polyphony, creating issues and writing code and
+documentation. Together we can make Polyphony a game-changer for developing
+concurrent apps in Ruby, and finally put to rest the notion that "Ruby is slow"!
+
+Since the last public release of Polyphony, we have focused on fixing bugs,
+improving performance and introducing new features that improve the Polyphony
+developer experience. Polyphony 0.44 is up to 20% percent faster than the
+previous release, due notably to a new ring-buffer implementation used by the
+fiber run queue and the `Polyphony::Queue` class, a new `Backend#read_loop` API
+for tighter server loops, and minimizing `fcntl` syscalls when doing I/O. These
+and other minor improvements have resulted in Polyphony first crossing the
+50,000 requests per second threshold for the first time in a minimal [rack
+server
+example](https://github.com/digital-fabric/polyphony/blob/master/examples/io/xx-rack_server.rb).
+
+Notable new features include a MySQL adapter, a Sequel adapter, and a new
+`Fiber#interject` API that allows executing arbitrary code on arbitrary fibers.
+
+We have also fixed numerous bugs, among which an issue building Polyphony on
+MacOS, problems issuing `Net::HTTP` requests with secure URLs, an issue with
+`YAML.load` and much more...
+
+For the full list of changes please consult the [change log](https://github.com/digital-fabric/polyphony/blob/master/CHANGELOG.md).
+
+## What's next for Polyphony?
+
+The next release of Polyphony will focus on full support IRB and Pry. Being able
+to run operations in the background in IRB and Pry can be very beneficial, most
+of all when developing and when debugging running processes using `binding.pry`
+for example.
+
+Subsequent releases will introduce a whole new full-featured debugger for
+fiber-aware concurrent apps, and eventually full support for Sequel, Sinatra,
+Hanami, Sidekiq and other major areas of the Ruby ecosystem.
+
+## Tipi - a polyphonic web server for Ruby
+
+[Tipi](https://github.com/digital-fabric/tipi) is a new web server for Ruby
+apps. It is intended to be *the* go-to app server for Ruby apps looking for
+robustness, scalability and performance. Tipi already supports HTTP/1, HTTP/2,
+WebSockets and SSL termination. It can currently drive simple Rack apps. In the
+future Tipi will be fully compliant with the Rack specification, and will also
+offer a static file server, a rich configuration  and automatic TLS certificates
+(using Let's Encrypt) out of the box.
+
+For those wondering about performance, here are some preliminary numbers (see
+disclaimer below):
+
+- HTTP, hello world, single process: ~50000 requests/second
+- HTTP, Rack hello world app, single process: ~33000 requests/second
+- HTTP, Rack hello world, 4 worker processes: ~95000 requests/second
+- HTTPS, hello world, single process: ~20000 requests/second
+- HTTPS, hello world, 4 worker processes: ~72000 requests/second
+
+Disclaimer: these numbers should be taken with a grain of salt. They do not
+follow any established benchmarking methodology, and may vary significantly. The
+different configurtations were benchmarked using the command: `wrk -d10 -t1 -c10
+"<http|https>://127.0.0.1:1234/"` on the same machine (an `m2.xlarge` instance)
+as the server. In the future Tipi's performance might substantially change. YMMV.

data/docs/api-reference/thread.md

@@ -12,7 +12,7 @@ Polyphony enhances the core `Thread` class with APIs for switching and
 scheduling fibers, and reimplements some of its APIs such as `Thread#raise`
 using fibers which, incidentally, make it safe.
 
-Each thread has its own run queue and its own system agent. While running
+Each thread has its own run queue and its own system backend. While running
 multiple threads does not result in true parallelism in MRI Ruby, sometimes
 multithreading is inevitable, for instance when using third-party gems that
 spawn threads, or when calling blocking APIs that are not fiber-aware.

data/docs/getting-started/overview.md

@@ -341,25 +341,25 @@ move_on_after(10) { perform_query }
 cancel_after(10) { perform_query }
 ```
 
-## The System Agent
+## The Polyphony Backend
 
 In order to implement automatic fiber switching when performing blocking
-operations, Polyphony introduces a concept called the *system agent*. The system
-agent is an object having a uniform interface, that performs all blocking
+operations, Polyphony introduces a concept called the *system backend*. The system
+backend is an object having a uniform interface, that performs all blocking
 operations.
 
 While a standard event loop-based solution would implement a blocking call
-separately from the fiber scheduling, the system agent integrates the two to
+separately from the fiber scheduling, the system backend integrates the two to
 create a blocking call that is already knows how to switch and schedule fibers.
 For example, in Polyphony all APIs having to do with reading from files or
-sockets end up calling `Thread.current.agent.read`, which does all the work.
+sockets end up calling `Thread.current.backend.read`, which does all the work.
 
 This design offers some major advantages over other designs. It minimizes memory
 allocations, of both Ruby objects and C structures. For example, instead of
 having to allocate libev watchers on the heap and then pass them around, they
 are allocated on the stack instead, which saves up on both memory and CPU cycles.
 
-In addition, the agent interface includes two methods that allow maximizing
+In addition, the backend interface includes two methods that allow maximizing
 server performance by accepting connections and reading from sockets in a tight
 loop. Here's a naive implementation of an HTTP/1 server:
 
@@ -372,7 +372,7 @@ def handle_client(socket)
   reqs = []
   parser.on_message_complete = proc { |env| reqs << { foo: :bar } }
 
-  Thread.current.agent.read_loop(socket) do |data|
+  Thread.current.backend.read_loop(socket) do |data|
     parser << data
     reqs.each { |r| reply(socket, r) }
     reqs.clear
@@ -388,20 +388,20 @@ end
 server = TCPServer.open('0.0.0.0', 1234)
 puts "listening on port 1234"
 
-Thread.current.agent.accept_loop(server) do |client|
+Thread.current.backend.accept_loop(server) do |client|
   spin { handle_client(client) }
 end
 ```
 
-The `#read_loop` and `#accept_loop` agent methods implement tight loops that
+The `#read_loop` and `#accept_loop` backend methods implement tight loops that
 provide a significant boost to performance (up to +30% better throughput.)
 
-Currently, Polyphony includes a single system agent based on
+Currently, Polyphony includes a single system backend based on
 [libev](http://pod.tst.eu/http://cvs.schmorp.de/libev/ev.pod). In the future,
-Polyphony will include other platform-specific system agents, such as a Windows
-agent using
+Polyphony will include other platform-specific system backends, such as a Windows
+backend using
 [IOCP](https://docs.microsoft.com/en-us/windows/win32/fileio/i-o-completion-ports),
-or an [io_uring](https://unixism.net/loti/what_is_io_uring.html) agent,
+or an [io_uring](https://unixism.net/loti/what_is_io_uring.html) backend,
 which might be a game-changer for writing highly-concurrent Ruby-based web apps.
 
 ## Writing Web Apps with Polyphony
@@ -482,5 +482,5 @@ reach version 1.0. Here are some of the exciting directions we're working on.
 
 - Support for more core and stdlib APIs
 - More adapters for gems with C-extensions, such as `mysql`, `sqlite3` etc
-- Use `io_uring` agent as alternative to the libev agent 
+- Use `io_uring` backend as alternative to the libev backend 
 - More concurrency constructs for building highly concurrent applications

data/docs/getting-started/tutorial.md

@@ -123,7 +123,7 @@ suspend # The main fiber suspends, waiting for all other work to finish
   sleep 1 # The sleeper fiber goes to sleep
     Gyro::Timer.new(1, 0).await # A timer event watcher is setup and yields
       Thread.current.switch_fiber # Polyphony looks for other runnable fibers
-        Thread.current.agent.poll # With no work left, the event loop is ran
+        Thread.current.backend.poll # With no work left, the event loop is ran
           fiber.schedule # The timer event fires, scheduling the sleeper fiber
   # <= The sleep method returns
   puts "Woke up"

data/examples/adapters/redis_client.rb

@@ -3,7 +3,9 @@
 require 'bundler/setup'
 require 'polyphony/adapters/redis'
 
-redis = Redis.new
+::Exception.__disable_sanitized_backtrace__ = true
+
+redis = Redis.new(host: ENV['REDIS_HOST'] || 'localhost')
 
 X = 10
 

data/examples/adapters/redis_pubsub_perf.rb

@@ -6,7 +6,7 @@ require 'json'
 
 X_SESSIONS = 1000
 X_NODES = 10_000
-X_SUBSCRIPTIONS_PER_SESSION = 100
+X_SUBSCRIPTIONS_PER_SESSION = 1000
 
 $sessions = []
 X_SESSIONS.times do
@@ -17,8 +17,11 @@ X_SESSIONS.times do
   }
 end
 
+REDIS_HOST = ENV['REDIS_HOST'] || 'localhost'
+p redis_host: REDIS_HOST
+
 spin do
-  redis = Redis.new
+  redis = Redis.new(host: REDIS_HOST)
   redis.subscribe('events') do |on|
     on.message do |_, message|
       distribute_event(JSON.parse(message, symbolize_names: true))
@@ -30,18 +33,18 @@ $update_count = 0
 
 def distribute_event(event)
   $update_count += 1
-  # t0 = Time.now
+  t0 = Time.now
   count = 0
   $sessions.each do |s|
     count += 1 if s[:subscriptions].include?(event[:path])
   end
-  # elapsed = Time.now - t0
-  # rate = X_SESSIONS / elapsed
-  # puts "elapsed: #{elapsed} (#{rate}/s)" if $update_count % 100 == 0
+  elapsed = Time.now - t0
+  rate = X_SESSIONS / elapsed
+  puts "elapsed: #{elapsed} (#{rate}/s)" if $update_count % 100 == 0
 end
 
 spin do
-  redis = Redis.new
+  redis = Redis.new(host: REDIS_HOST)
   throttled_loop(1000) do
     redis.publish('events', { path: "node#{rand(X_NODES)}" }.to_json)
   end
@@ -60,7 +63,7 @@ spin do
   end
 end
 
-trap(:int) do
+trap('SIGINT') do
   puts 'bye...'
   exit!
 end