polyphony 0.43.10 → 0.45.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b3000a280d995b187518c1e12f585939b4e67312db5458d105c6e6b40c784d5a
-  data.tar.gz: 5a582fc77ae044238521619b1d5364a0dd2aac13ca8a6b82d006ec1f23284d2d
+  metadata.gz: a2fa4fa731ff52272d3aed456f7a26751e76c63b4335366919bffaf089bf96dd
+  data.tar.gz: 1afe67258c5c73d9cc1082ca238eb358dac0f5b7a975f90a77b4f6de6a23b2a4
 SHA512:
-  metadata.gz: 6d403109b1bbf55e1b0ab799d8c74752677784ca0c12cb5b1e21f62c9bdd244245966db8020b20a8d4ed988263a9535aef5a8aff24b50587c9583a67109d7677
-  data.tar.gz: 205d6c9859ec8d12ddf7630cbd3893a13d3efb3537dd734e53f17efe31d52399488761da45684d0da7e5074ce950c47a7e51f4a1446577e36f7c3c7838604a7a
+  metadata.gz: 175edf315d759f85d2c0d934be36b9fcacd68342206d984ad64f800120a8b65668e9ea81595f47cbbb439402c256b47fde78f0cdad4b42e0c3b1d6ee321135b9
+  data.tar.gz: 01cfe1d8b51d60a736adb80062eb22e7983a1879a5491cb7a2e47ec4ad5734e0c8bd8664653870a4393276b42fa912953721658ecff634521a672419c8d3cab6

data/.rubocop.yml

@@ -81,6 +81,7 @@ Lint/SuppressedException:
     - examples/**/*.rb
 
 Metrics/MethodLength:
+  Max: 12
   Exclude:
     - lib/polyphony/http/server/rack.rb
     - lib/polyphony/extensions/io.rb
@@ -95,6 +96,7 @@ Metrics/ModuleLength:
 Metrics/ClassLength:
   Exclude:
     - lib/polyphony/http/server/http1.rb
+    - lib/polyphony/extensions/io.rb
     - test/**/*.rb
     - examples/**/*.rb
 
@@ -111,6 +113,7 @@ Style/Documentation:
   Exclude:
     - test/**/*.rb
     - examples/**/*.rb
+    - lib/polyphony/adapters/**/*.rb
 
 Style/FormatString:
   Exclude:
@@ -172,4 +175,8 @@ Style/RedundantRegexpEscape:
   Enabled: true
 
 Style/SlicingWithRange:
-  Enabled: true
+  Enabled: true
+
+Style/RaiseArgs:
+  Exclude:
+    - lib/polyphony/extensions/fiber.rb

data/CHANGELOG.md

@@ -1,3 +1,40 @@
+## 0.45.2
+
+* Rewrite `Fiber#<<`, `Fiber#await`, `Fiber#receive` in C
+
+## 0.45.1
+
+* Fix Net::HTTP compatibility
+* Fix fs adapter
+* Improve performance of IO#puts
+* Mutex#synchronize
+* Fix Socket#connect
+* Cleanup code
+* Improve support for Ruby 3 keyword args
+
+## 0.45.0
+
+* Cleanup code
+* Rename `Agent` to `Backend`
+* Implement `Polyphony::ConditionVariable`
+* Fix Kernel.system
+
+## 0.44.0 2020-07-25
+
+* Fix reentrant `ResourcePool` (#38)
+* Add `ResourcePool#discard!` (#35)
+* Add `Mysql2::Client` and `Sequel::ConnectionPool` adapters (#35)
+* Reimplement `Kernel.trap` using `Fiber#interject`
+* Add `Fiber#interject` for running arbitrary code on arbitrary fibers (#39)
+
+## 0.43.11 2020-07-24
+
+* Dump uncaught exception info for forked process (#36)
+* Add additional socket config options (#37)
+  - :reuse_port (`SO_REUSEPORT`)
+  - :backlog (listen backlog, default `SOMAXCONN`)
+* Fix possible race condition in Queue#shift (#34)
+
 ## 0.43.10 2020-07-23
 
 * Fix race condition when terminating fibers (#33)

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    polyphony (0.43.10)
+    polyphony (0.45.2)
 
 GEM
   remote: https://rubygems.org/
@@ -11,6 +11,7 @@ GEM
     ansi (1.5.0)
     ast (2.4.0)
     builder (3.2.4)
+    coderay (1.1.3)
     colorator (1.1.0)
     concurrent-ruby (1.1.6)
     docile (1.3.2)
@@ -22,7 +23,7 @@ GEM
     forwardable-extended (2.6.0)
     hiredis (0.6.3)
     http_parser.rb (0.6.0)
-    httparty (0.17.0)
+    httparty (0.17.1)
       mime-types (~> 3.0)
       multi_xml (>= 0.5.2)
     i18n (0.9.5)
@@ -60,11 +61,11 @@ GEM
     listen (3.2.1)
       rb-fsevent (~> 0.10, >= 0.10.3)
       rb-inotify (~> 0.9, >= 0.9.10)
-    localhost (1.1.4)
     mercenary (0.3.6)
+    method_source (1.0.0)
     mime-types (3.3.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2019.1009)
+    mime-types-data (3.2020.0512)
     minitest (5.13.0)
     minitest-reporters (1.4.2)
       ansi
@@ -72,13 +73,18 @@ GEM
       minitest (>= 5.0)
       ruby-progressbar
     multi_xml (0.6.0)
+    mysql2 (0.5.3)
     parallel (1.19.1)
     parser (2.7.0.2)
       ast (~> 2.4.0)
     pathutil (0.16.2)
       forwardable-extended (~> 2.6)
     pg (1.1.4)
+    pry (0.13.1)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
     public_suffix (4.0.3)
+    rack (2.2.3)
     rainbow (3.0.0)
     rake (12.3.3)
     rake-compiler (1.0.5)
@@ -109,6 +115,7 @@ GEM
     sass-listen (4.0.0)
       rb-fsevent (~> 0.9, >= 0.9.4)
       rb-inotify (~> 0.9, >= 0.9.7)
+    sequel (5.34.0)
     simplecov (0.17.1)
       docile (~> 1.1)
       json (>= 1.8, < 3)
@@ -122,19 +129,22 @@ PLATFORMS
 DEPENDENCIES
   hiredis (= 0.6.3)
   http_parser.rb (~> 0.6.0)
-  httparty (= 0.17.0)
+  httparty (= 0.17.1)
   jekyll (~> 3.8.6)
   jekyll-remote-theme (~> 0.4.1)
   jekyll-seo-tag (~> 2.6.1)
   just-the-docs (~> 0.3.0)
-  localhost (= 1.1.4)
   minitest (= 5.13.0)
   minitest-reporters (= 1.4.2)
+  mysql2 (= 0.5.3)
   pg (= 1.1.4)
   polyphony!
+  pry (= 0.13.1)
+  rack (>= 2.0.8, < 2.3.0)
   rake-compiler (= 1.0.5)
   redis (= 4.1.0)
   rubocop (= 0.85.1)
+  sequel (= 5.34.0)
   simplecov (= 0.17.1)
 
 BUNDLED WITH

data/Rakefile

@@ -20,7 +20,7 @@ task :stress_test do
 end
 
 task :docs do
-  exec 'RUBYOPT=-W0 jekyll serve -s docs -H ec2-35-158-110-38.eu-central-1.compute.amazonaws.com'
+  exec 'RUBYOPT=-W0 jekyll serve -s docs -H ec2-18-156-117-172.eu-central-1.compute.amazonaws.com'
 end
 
 CLEAN.include "**/*.o", "**/*.so", "**/*.bundle", "**/*.jar", "pkg", "tmp"

data/TODO.md

@@ -1,9 +1,18 @@
--- Add `Fiber#schedule_with_priority` method
+0.45.2
+
+- Adapter for Pry and IRB (Which fixes #5 and #6)
+- Redesign signal handling - the current mechanism is problematic in that it
+  does not address signals that do not kill, for instance HUP or USR1.
+- Improve `#supervise`. It does not work as advertised, and seems to exhibit an
+  inconsistent behaviour (see supervisor example).
+- Fix backtrace for `Timeout.timeout` API (see timeout example).
+- Check why worker-thread example doesn't work.
+
+0.46.0
 
 - Debugging
   - Eat your own dogfood: need a good tool to check what's going on when some
     test fails
-  - Needs to work with Pry (can write perhaps an extension for pry)
   - First impl in Ruby using `TracePoint` API
   - Mode of operation:
     - Two parts: tracer and controller
@@ -113,7 +122,7 @@
   - discuss using `snooze` for ensuring responsiveness when executing CPU-bound work
 
 
-## 0.44
+## 0.47
 
 ### Some more API work, more docs
 
@@ -126,13 +135,13 @@
   - proceed from there
 
 
-## 0.45
+## 0.48
 
 ### Sinatra / Sidekiq
 
 - Pull out redis/postgres code, put into new `polyphony-xxx` gems
 
-## 0.46
+## 0.49
 
 ### Testing && Docs
 
@@ -144,11 +153,7 @@
   - `IO.foreach`
   - `Process.waitpid`
 
-## 0.47
-
-### Real IO#gets and IO#read
-
-## 0.48 DNS
+## 0.50 DNS
 
 ### DNS client
 

data/docs/_posts/2020-07-26-polyphony-0.44.md

@@ -0,0 +1,77 @@
+# Polyphony 0.44.1
+
+## More performance, more compatibility, more robustness
+
+The last three weeks have been very busy for Polyphony. Since I first presented
+Polyphony here and elsewhere, 17 issues were closed, 10 pull requests were
+merged, and 144 commits were made by 4 different authors. I'm really
+excited about Polyphony and the momentum it seems to be gathering. Your
+reactions have been very positive so far (it even got [tweeted by
+Matz!](https://twitter.com/yukihiro_matz/status/1279289318083715073))
+
+I'm even more excited about the contributions Polyphony is starting to get from
+other developers. Thank you [Will](https://github.com/wjordan),
+[Máximo](https://github.com/ElMassimo) and [Trent](https://github.com/misfo) for
+your valuable contributions! Also, the Polyphony project has now got a logo
+designed by my friend [Gérald Morales](https://webocube.com/).
+
+I'd like to encourage other developers to get in on the action and start
+contributing by testing Polyphony, creating issues and writing code and
+documentation. Together we can make Polyphony a game-changer for developing
+concurrent apps in Ruby, and finally put to rest the notion that "Ruby is slow"!
+
+Since the last public release of Polyphony, we have focused on fixing bugs,
+improving performance and introducing new features that improve the Polyphony
+developer experience. Polyphony 0.44 is up to 20% percent faster than the
+previous release, due notably to a new ring-buffer implementation used by the
+fiber run queue and the `Polyphony::Queue` class, a new `Backend#read_loop` API
+for tighter server loops, and minimizing `fcntl` syscalls when doing I/O. These
+and other minor improvements have resulted in Polyphony first crossing the
+50,000 requests per second threshold for the first time in a minimal [rack
+server
+example](https://github.com/digital-fabric/polyphony/blob/master/examples/io/xx-rack_server.rb).
+
+Notable new features include a MySQL adapter, a Sequel adapter, and a new
+`Fiber#interject` API that allows executing arbitrary code on arbitrary fibers.
+
+We have also fixed numerous bugs, among which an issue building Polyphony on
+MacOS, problems issuing `Net::HTTP` requests with secure URLs, an issue with
+`YAML.load` and much more...
+
+For the full list of changes please consult the [change log](https://github.com/digital-fabric/polyphony/blob/master/CHANGELOG.md).
+
+## What's next for Polyphony?
+
+The next release of Polyphony will focus on full support IRB and Pry. Being able
+to run operations in the background in IRB and Pry can be very beneficial, most
+of all when developing and when debugging running processes using `binding.pry`
+for example.
+
+Subsequent releases will introduce a whole new full-featured debugger for
+fiber-aware concurrent apps, and eventually full support for Sequel, Sinatra,
+Hanami, Sidekiq and other major areas of the Ruby ecosystem.
+
+## Tipi - a polyphonic web server for Ruby
+
+[Tipi](https://github.com/digital-fabric/tipi) is a new web server for Ruby
+apps. It is intended to be *the* go-to app server for Ruby apps looking for
+robustness, scalability and performance. Tipi already supports HTTP/1, HTTP/2,
+WebSockets and SSL termination. It can currently drive simple Rack apps. In the
+future Tipi will be fully compliant with the Rack specification, and will also
+offer a static file server, a rich configuration  and automatic TLS certificates
+(using Let's Encrypt) out of the box.
+
+For those wondering about performance, here are some preliminary numbers (see
+disclaimer below):
+
+- HTTP, hello world, single process: ~50000 requests/second
+- HTTP, Rack hello world app, single process: ~33000 requests/second
+- HTTP, Rack hello world, 4 worker processes: ~95000 requests/second
+- HTTPS, hello world, single process: ~20000 requests/second
+- HTTPS, hello world, 4 worker processes: ~72000 requests/second
+
+Disclaimer: these numbers should be taken with a grain of salt. They do not
+follow any established benchmarking methodology, and may vary significantly. The
+different configurtations were benchmarked using the command: `wrk -d10 -t1 -c10
+"<http|https>://127.0.0.1:1234/"` on the same machine (an `m2.xlarge` instance)
+as the server. In the future Tipi's performance might substantially change. YMMV.

data/docs/api-reference/thread.md

@@ -12,7 +12,7 @@ Polyphony enhances the core `Thread` class with APIs for switching and
 scheduling fibers, and reimplements some of its APIs such as `Thread#raise`
 using fibers which, incidentally, make it safe.
 
-Each thread has its own run queue and its own system agent. While running
+Each thread has its own run queue and its own system backend. While running
 multiple threads does not result in true parallelism in MRI Ruby, sometimes
 multithreading is inevitable, for instance when using third-party gems that
 spawn threads, or when calling blocking APIs that are not fiber-aware.

data/docs/getting-started/overview.md

@@ -341,25 +341,25 @@ move_on_after(10) { perform_query }
 cancel_after(10) { perform_query }
 ```
 
-## The System Agent
+## The Polyphony Backend
 
 In order to implement automatic fiber switching when performing blocking
-operations, Polyphony introduces a concept called the *system agent*. The system
-agent is an object having a uniform interface, that performs all blocking
+operations, Polyphony introduces a concept called the *system backend*. The system
+backend is an object having a uniform interface, that performs all blocking
 operations.
 
 While a standard event loop-based solution would implement a blocking call
-separately from the fiber scheduling, the system agent integrates the two to
+separately from the fiber scheduling, the system backend integrates the two to
 create a blocking call that is already knows how to switch and schedule fibers.
 For example, in Polyphony all APIs having to do with reading from files or
-sockets end up calling `Thread.current.agent.read`, which does all the work.
+sockets end up calling `Thread.current.backend.read`, which does all the work.
 
 This design offers some major advantages over other designs. It minimizes memory
 allocations, of both Ruby objects and C structures. For example, instead of
 having to allocate libev watchers on the heap and then pass them around, they
 are allocated on the stack instead, which saves up on both memory and CPU cycles.
 
-In addition, the agent interface includes two methods that allow maximizing
+In addition, the backend interface includes two methods that allow maximizing
 server performance by accepting connections and reading from sockets in a tight
 loop. Here's a naive implementation of an HTTP/1 server:
 
@@ -372,7 +372,7 @@ def handle_client(socket)
   reqs = []
   parser.on_message_complete = proc { |env| reqs << { foo: :bar } }
 
-  Thread.current.agent.read_loop(socket) do |data|
+  Thread.current.backend.read_loop(socket) do |data|
     parser << data
     reqs.each { |r| reply(socket, r) }
     reqs.clear
@@ -388,20 +388,20 @@ end
 server = TCPServer.open('0.0.0.0', 1234)
 puts "listening on port 1234"
 
-Thread.current.agent.accept_loop(server) do |client|
+Thread.current.backend.accept_loop(server) do |client|
   spin { handle_client(client) }
 end
 ```
 
-The `#read_loop` and `#accept_loop` agent methods implement tight loops that
+The `#read_loop` and `#accept_loop` backend methods implement tight loops that
 provide a significant boost to performance (up to +30% better throughput.)
 
-Currently, Polyphony includes a single system agent based on
+Currently, Polyphony includes a single system backend based on
 [libev](http://pod.tst.eu/http://cvs.schmorp.de/libev/ev.pod). In the future,
-Polyphony will include other platform-specific system agents, such as a Windows
-agent using
+Polyphony will include other platform-specific system backends, such as a Windows
+backend using
 [IOCP](https://docs.microsoft.com/en-us/windows/win32/fileio/i-o-completion-ports),
-or an [io_uring](https://unixism.net/loti/what_is_io_uring.html) agent,
+or an [io_uring](https://unixism.net/loti/what_is_io_uring.html) backend,
 which might be a game-changer for writing highly-concurrent Ruby-based web apps.
 
 ## Writing Web Apps with Polyphony
@@ -482,5 +482,5 @@ reach version 1.0. Here are some of the exciting directions we're working on.
 
 - Support for more core and stdlib APIs
 - More adapters for gems with C-extensions, such as `mysql`, `sqlite3` etc
-- Use `io_uring` agent as alternative to the libev agent 
+- Use `io_uring` backend as alternative to the libev backend 
 - More concurrency constructs for building highly concurrent applications

data/docs/getting-started/tutorial.md

@@ -123,7 +123,7 @@ suspend # The main fiber suspends, waiting for all other work to finish
   sleep 1 # The sleeper fiber goes to sleep
     Gyro::Timer.new(1, 0).await # A timer event watcher is setup and yields
       Thread.current.switch_fiber # Polyphony looks for other runnable fibers
-        Thread.current.agent.poll # With no work left, the event loop is ran
+        Thread.current.backend.poll # With no work left, the event loop is ran
           fiber.schedule # The timer event fires, scheduling the sleeper fiber
   # <= The sleep method returns
   puts "Woke up"

data/examples/adapters/redis_client.rb

@@ -3,7 +3,9 @@
 require 'bundler/setup'
 require 'polyphony/adapters/redis'
 
-redis = Redis.new
+::Exception.__disable_sanitized_backtrace__ = true
+
+redis = Redis.new(host: ENV['REDIS_HOST'] || 'localhost')
 
 X = 10
 

data/examples/adapters/redis_pubsub_perf.rb

@@ -6,7 +6,7 @@ require 'json'
 
 X_SESSIONS = 1000
 X_NODES = 10_000
-X_SUBSCRIPTIONS_PER_SESSION = 100
+X_SUBSCRIPTIONS_PER_SESSION = 1000
 
 $sessions = []
 X_SESSIONS.times do
@@ -17,8 +17,11 @@ X_SESSIONS.times do
   }
 end
 
+REDIS_HOST = ENV['REDIS_HOST'] || 'localhost'
+p redis_host: REDIS_HOST
+
 spin do
-  redis = Redis.new
+  redis = Redis.new(host: REDIS_HOST)
   redis.subscribe('events') do |on|
     on.message do |_, message|
       distribute_event(JSON.parse(message, symbolize_names: true))
@@ -30,18 +33,18 @@ $update_count = 0
 
 def distribute_event(event)
   $update_count += 1
-  # t0 = Time.now
+  t0 = Time.now
   count = 0
   $sessions.each do |s|
     count += 1 if s[:subscriptions].include?(event[:path])
   end
-  # elapsed = Time.now - t0
-  # rate = X_SESSIONS / elapsed
-  # puts "elapsed: #{elapsed} (#{rate}/s)" if $update_count % 100 == 0
+  elapsed = Time.now - t0
+  rate = X_SESSIONS / elapsed
+  puts "elapsed: #{elapsed} (#{rate}/s)" if $update_count % 100 == 0
 end
 
 spin do
-  redis = Redis.new
+  redis = Redis.new(host: REDIS_HOST)
   throttled_loop(1000) do
     redis.publish('events', { path: "node#{rand(X_NODES)}" }.to_json)
   end
@@ -60,7 +63,7 @@ spin do
   end
 end
 
-trap(:int) do
+trap('SIGINT') do
   puts 'bye...'
   exit!
 end