polyphony 0.44.0 → 0.45.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 04c1c1221d46b1b9169aa04c5896c8d68c20fa11d64554e76b7452710423f142
-  data.tar.gz: 519528e0a3dfd323620869d4a7e43e4769cf3c1b5f0ea405cf341efc6fd534f6
+  metadata.gz: f9c8ab74213c6cc5e3852f73ee027ee496f2b04c6844e09856c9771ea5e7839a
+  data.tar.gz: 83d7c533024b6d6d633b9e18abc392911adfc07f728af826bd84cce35cafb20c
 SHA512:
-  metadata.gz: de8ac460dd05b051057ffab1c155c6b2d675cc623b98549333ba1defbc99f4efb738ff4a34a9f8c14bd3cb9ab0cab5356aa34a2308daa73273a6ef8719565bc4
-  data.tar.gz: c26b254559e484774574555dc895cf072bda8061b92fed26d8e61e7172523fe0faf2750cd2f924db3a25b59583b673ebe5f14749fd6f1279ca2e7e7b7745490b
+  metadata.gz: c42e49ebcd6fb10b384438cb7688801a387963bc2a6fb2c1d3de6d022a7731a7d88db138a34b364b35c9d71830b975a69cb1f14fb277166a9db1802e24e82dec
+  data.tar.gz: 6c578eacded00dd7a77a35124c69ebc966e4f9aa42264913df82af147f43955a2ecd86087630c085fcd4d6e964aa705202c5aa24bb4032648d3ba278d60d4f63

data/.rubocop.yml

@@ -81,6 +81,7 @@ Lint/SuppressedException:
     - examples/**/*.rb
 
 Metrics/MethodLength:
+  Max: 12
   Exclude:
     - lib/polyphony/http/server/rack.rb
     - lib/polyphony/extensions/io.rb
@@ -111,6 +112,7 @@ Style/Documentation:
   Exclude:
     - test/**/*.rb
     - examples/**/*.rb
+    - lib/polyphony/adapters/**/*.rb
 
 Style/FormatString:
   Exclude:
@@ -172,4 +174,8 @@ Style/RedundantRegexpEscape:
   Enabled: true
 
 Style/SlicingWithRange:
-  Enabled: true
+  Enabled: true
+
+Style/RaiseArgs:
+  Exclude:
+    - lib/polyphony/extensions/fiber.rb

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+## 0.45.0
+
+* Cleanup code
+* Rename `Agent` to `Backend`
+* Implement `Polyphony::ConditionVariable`
+* Fix Kernel.system
+
 ## 0.44.0 2020-07-25
 
 * Fix reentrant `ResourcePool` (#38)

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    polyphony (0.44.0)
+    polyphony (0.45.0)
 
 GEM
   remote: https://rubygems.org/
@@ -11,6 +11,7 @@ GEM
     ansi (1.5.0)
     ast (2.4.0)
     builder (3.2.4)
+    coderay (1.1.3)
     colorator (1.1.0)
     concurrent-ruby (1.1.6)
     docile (1.3.2)
@@ -22,9 +23,6 @@ GEM
     forwardable-extended (2.6.0)
     hiredis (0.6.3)
     http_parser.rb (0.6.0)
-    httparty (0.17.0)
-      mime-types (~> 3.0)
-      multi_xml (>= 0.5.2)
     i18n (0.9.5)
       concurrent-ruby (~> 1.0)
     jekyll (3.8.6)
@@ -60,18 +58,14 @@ GEM
     listen (3.2.1)
       rb-fsevent (~> 0.10, >= 0.10.3)
       rb-inotify (~> 0.9, >= 0.9.10)
-    localhost (1.1.4)
     mercenary (0.3.6)
-    mime-types (3.3.1)
-      mime-types-data (~> 3.2015)
-    mime-types-data (3.2019.1009)
+    method_source (1.0.0)
     minitest (5.13.0)
     minitest-reporters (1.4.2)
       ansi
       builder
       minitest (>= 5.0)
       ruby-progressbar
-    multi_xml (0.6.0)
     mysql2 (0.5.3)
     parallel (1.19.1)
     parser (2.7.0.2)
@@ -79,7 +73,11 @@ GEM
     pathutil (0.16.2)
       forwardable-extended (~> 2.6)
     pg (1.1.4)
+    pry (0.13.1)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
     public_suffix (4.0.3)
+    rack (2.2.3)
     rainbow (3.0.0)
     rake (12.3.3)
     rake-compiler (1.0.5)
@@ -124,17 +122,17 @@ PLATFORMS
 DEPENDENCIES
   hiredis (= 0.6.3)
   http_parser.rb (~> 0.6.0)
-  httparty (= 0.17.0)
   jekyll (~> 3.8.6)
   jekyll-remote-theme (~> 0.4.1)
   jekyll-seo-tag (~> 2.6.1)
   just-the-docs (~> 0.3.0)
-  localhost (= 1.1.4)
   minitest (= 5.13.0)
   minitest-reporters (= 1.4.2)
   mysql2 (= 0.5.3)
   pg (= 1.1.4)
   polyphony!
+  pry (= 0.13.1)
+  rack (>= 2.0.8, < 2.3.0)
   rake-compiler (= 1.0.5)
   redis (= 4.1.0)
   rubocop (= 0.85.1)

data/Rakefile

@@ -20,7 +20,7 @@ task :stress_test do
 end
 
 task :docs do
-  exec 'RUBYOPT=-W0 jekyll serve -s docs -H ec2-35-158-110-38.eu-central-1.compute.amazonaws.com'
+  exec 'RUBYOPT=-W0 jekyll serve -s docs -H ec2-18-156-117-172.eu-central-1.compute.amazonaws.com'
 end
 
 CLEAN.include "**/*.o", "**/*.so", "**/*.bundle", "**/*.jar", "pkg", "tmp"

data/TODO.md

@@ -1,11 +1,16 @@
-- Adapter for Pry, or maybe just an adapter for readline
-- Rename Agent to Backend
+0.45
+
 - Review all code
   - Cleanup C code
-  - Cleanup Ruby code (use Rubocop)
   - Cleanup and annotate examples (and remove all the examples used for
     debugging). Focus on examples that serve as "how-to".
 
+0.45.1
+
+- Adapter for Pry and IRB (Which fixes #5 and #6)
+
+0.46.0
+
 - Debugging
   - Eat your own dogfood: need a good tool to check what's going on when some
     test fails
@@ -118,7 +123,7 @@
   - discuss using `snooze` for ensuring responsiveness when executing CPU-bound work
 
 
-## 0.45
+## 0.47
 
 ### Some more API work, more docs
 
@@ -131,13 +136,13 @@
   - proceed from there
 
 
-## 0.46
+## 0.48
 
 ### Sinatra / Sidekiq
 
 - Pull out redis/postgres code, put into new `polyphony-xxx` gems
 
-## 0.47
+## 0.49
 
 ### Testing && Docs
 
@@ -149,7 +154,7 @@
   - `IO.foreach`
   - `Process.waitpid`
 
-## 0.48 DNS
+## 0.50 DNS
 
 ### DNS client
 

data/docs/_posts/2020-07-26-polyphony-0.44.md

@@ -0,0 +1,77 @@
+# Polyphony 0.44.1
+
+## More performance, more compatibility, more robustness
+
+The last three weeks have been very busy for Polyphony. Since I first presented
+Polyphony here and elsewhere, 17 issues were closed, 10 pull requests were
+merged, and 144 commits were made by 4 different authors. I'm really
+excited about Polyphony and the momentum it seems to be gathering. Your
+reactions have been very positive so far (it even got [tweeted by
+Matz!](https://twitter.com/yukihiro_matz/status/1279289318083715073))
+
+I'm even more excited about the contributions Polyphony is starting to get from
+other developers. Thank you [Will](https://github.com/wjordan),
+[Máximo](https://github.com/ElMassimo) and [Trent](https://github.com/misfo) for
+your valuable contributions! Also, the Polyphony project has now got a logo
+designed by my friend [Gérald Morales](https://webocube.com/).
+
+I'd like to encourage other developers to get in on the action and start
+contributing by testing Polyphony, creating issues and writing code and
+documentation. Together we can make Polyphony a game-changer for developing
+concurrent apps in Ruby, and finally put to rest the notion that "Ruby is slow"!
+
+Since the last public release of Polyphony, we have focused on fixing bugs,
+improving performance and introducing new features that improve the Polyphony
+developer experience. Polyphony 0.44 is up to 20% percent faster than the
+previous release, due notably to a new ring-buffer implementation used by the
+fiber run queue and the `Polyphony::Queue` class, a new `Backend#read_loop` API
+for tighter server loops, and minimizing `fcntl` syscalls when doing I/O. These
+and other minor improvements have resulted in Polyphony first crossing the
+50,000 requests per second threshold for the first time in a minimal [rack
+server
+example](https://github.com/digital-fabric/polyphony/blob/master/examples/io/xx-rack_server.rb).
+
+Notable new features include a MySQL adapter, a Sequel adapter, and a new
+`Fiber#interject` API that allows executing arbitrary code on arbitrary fibers.
+
+We have also fixed numerous bugs, among which an issue building Polyphony on
+MacOS, problems issuing `Net::HTTP` requests with secure URLs, an issue with
+`YAML.load` and much more...
+
+For the full list of changes please consult the [change log](https://github.com/digital-fabric/polyphony/blob/master/CHANGELOG.md).
+
+## What's next for Polyphony?
+
+The next release of Polyphony will focus on full support IRB and Pry. Being able
+to run operations in the background in IRB and Pry can be very beneficial, most
+of all when developing and when debugging running processes using `binding.pry`
+for example.
+
+Subsequent releases will introduce a whole new full-featured debugger for
+fiber-aware concurrent apps, and eventually full support for Sequel, Sinatra,
+Hanami, Sidekiq and other major areas of the Ruby ecosystem.
+
+## Tipi - a polyphonic web server for Ruby
+
+[Tipi](https://github.com/digital-fabric/tipi) is a new web server for Ruby
+apps. It is intended to be *the* go-to app server for Ruby apps looking for
+robustness, scalability and performance. Tipi already supports HTTP/1, HTTP/2,
+WebSockets and SSL termination. It can currently drive simple Rack apps. In the
+future Tipi will be fully compliant with the Rack specification, and will also
+offer a static file server, a rich configuration  and automatic TLS certificates
+(using Let's Encrypt) out of the box.
+
+For those wondering about performance, here are some preliminary numbers (see
+disclaimer below):
+
+- HTTP, hello world, single process: ~50000 requests/second
+- HTTP, Rack hello world app, single process: ~33000 requests/second
+- HTTP, Rack hello world, 4 worker processes: ~95000 requests/second
+- HTTPS, hello world, single process: ~20000 requests/second
+- HTTPS, hello world, 4 worker processes: ~72000 requests/second
+
+Disclaimer: these numbers should be taken with a grain of salt. They do not
+follow any established benchmarking methodology, and may vary significantly. The
+different configurtations were benchmarked using the command: `wrk -d10 -t1 -c10
+"<http|https>://127.0.0.1:1234/"` on the same machine (an `m2.xlarge` instance)
+as the server. In the future Tipi's performance might substantially change. YMMV.

data/docs/api-reference/thread.md

@@ -12,7 +12,7 @@ Polyphony enhances the core `Thread` class with APIs for switching and
 scheduling fibers, and reimplements some of its APIs such as `Thread#raise`
 using fibers which, incidentally, make it safe.
 
-Each thread has its own run queue and its own system agent. While running
+Each thread has its own run queue and its own system backend. While running
 multiple threads does not result in true parallelism in MRI Ruby, sometimes
 multithreading is inevitable, for instance when using third-party gems that
 spawn threads, or when calling blocking APIs that are not fiber-aware.

data/docs/getting-started/overview.md

@@ -341,25 +341,25 @@ move_on_after(10) { perform_query }
 cancel_after(10) { perform_query }
 ```
 
-## The System Agent
+## The Polyphony Backend
 
 In order to implement automatic fiber switching when performing blocking
-operations, Polyphony introduces a concept called the *system agent*. The system
-agent is an object having a uniform interface, that performs all blocking
+operations, Polyphony introduces a concept called the *system backend*. The system
+backend is an object having a uniform interface, that performs all blocking
 operations.
 
 While a standard event loop-based solution would implement a blocking call
-separately from the fiber scheduling, the system agent integrates the two to
+separately from the fiber scheduling, the system backend integrates the two to
 create a blocking call that is already knows how to switch and schedule fibers.
 For example, in Polyphony all APIs having to do with reading from files or
-sockets end up calling `Thread.current.agent.read`, which does all the work.
+sockets end up calling `Thread.current.backend.read`, which does all the work.
 
 This design offers some major advantages over other designs. It minimizes memory
 allocations, of both Ruby objects and C structures. For example, instead of
 having to allocate libev watchers on the heap and then pass them around, they
 are allocated on the stack instead, which saves up on both memory and CPU cycles.
 
-In addition, the agent interface includes two methods that allow maximizing
+In addition, the backend interface includes two methods that allow maximizing
 server performance by accepting connections and reading from sockets in a tight
 loop. Here's a naive implementation of an HTTP/1 server:
 
@@ -372,7 +372,7 @@ def handle_client(socket)
   reqs = []
   parser.on_message_complete = proc { |env| reqs << { foo: :bar } }
 
-  Thread.current.agent.read_loop(socket) do |data|
+  Thread.current.backend.read_loop(socket) do |data|
     parser << data
     reqs.each { |r| reply(socket, r) }
     reqs.clear
@@ -388,20 +388,20 @@ end
 server = TCPServer.open('0.0.0.0', 1234)
 puts "listening on port 1234"
 
-Thread.current.agent.accept_loop(server) do |client|
+Thread.current.backend.accept_loop(server) do |client|
   spin { handle_client(client) }
 end
 ```
 
-The `#read_loop` and `#accept_loop` agent methods implement tight loops that
+The `#read_loop` and `#accept_loop` backend methods implement tight loops that
 provide a significant boost to performance (up to +30% better throughput.)
 
-Currently, Polyphony includes a single system agent based on
+Currently, Polyphony includes a single system backend based on
 [libev](http://pod.tst.eu/http://cvs.schmorp.de/libev/ev.pod). In the future,
-Polyphony will include other platform-specific system agents, such as a Windows
-agent using
+Polyphony will include other platform-specific system backends, such as a Windows
+backend using
 [IOCP](https://docs.microsoft.com/en-us/windows/win32/fileio/i-o-completion-ports),
-or an [io_uring](https://unixism.net/loti/what_is_io_uring.html) agent,
+or an [io_uring](https://unixism.net/loti/what_is_io_uring.html) backend,
 which might be a game-changer for writing highly-concurrent Ruby-based web apps.
 
 ## Writing Web Apps with Polyphony
@@ -482,5 +482,5 @@ reach version 1.0. Here are some of the exciting directions we're working on.
 
 - Support for more core and stdlib APIs
 - More adapters for gems with C-extensions, such as `mysql`, `sqlite3` etc
-- Use `io_uring` agent as alternative to the libev agent 
+- Use `io_uring` backend as alternative to the libev backend 
 - More concurrency constructs for building highly concurrent applications

data/docs/getting-started/tutorial.md

@@ -123,7 +123,7 @@ suspend # The main fiber suspends, waiting for all other work to finish
   sleep 1 # The sleeper fiber goes to sleep
     Gyro::Timer.new(1, 0).await # A timer event watcher is setup and yields
       Thread.current.switch_fiber # Polyphony looks for other runnable fibers
-        Thread.current.agent.poll # With no work left, the event loop is ran
+        Thread.current.backend.poll # With no work left, the event loop is ran
           fiber.schedule # The timer event fires, scheduling the sleeper fiber
   # <= The sleep method returns
   puts "Woke up"

data/examples/core/{xx-agent.rb → xx-backend.rb}

@@ -27,17 +27,17 @@ class Test
   
   def test_file
     f = File.open(__FILE__, 'r')
-    puts Thread.current.agent.read(f, +'', 10000, true)
+    puts Thread.current.backend.read(f, +'', 10000, true)
     
-    Thread.current.agent.write(STDOUT, "Write something: ")
+    Thread.current.backend.write(STDOUT, "Write something: ")
     str = +''
-    Thread.current.agent.read(STDIN, str, 5, false)
+    Thread.current.backend.read(STDIN, str, 5, false)
     puts str
   end
   
   def test_fork
     pid = fork do
-      Thread.current.agent.post_fork
+      Thread.current.backend.post_fork
       puts 'child going to sleep'
       sleep 1
       puts 'child done sleeping'
@@ -45,7 +45,7 @@ class Test
     end
     
     puts "Waiting for pid #{pid}"
-    result = Thread.current.agent.waitpid(pid)
+    result = Thread.current.backend.waitpid(pid)
     puts "Done waiting"
     p result
   end

data/examples/io/xx-pry.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'polyphony'
+require 'polyphony/adapters/readline'
+require 'pry'
+
+$counter = 0
+timer = spin do
+  throttled_loop(5) do
+    $counter += 1
+  end
+end
+
+at_exit { timer.stop }
+
+puts 'try typing $counter to see the counter incremented in the background'
+binding.pry

data/examples/io/xx-rack_server.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'polyphony'
+require 'http/parser'
+require 'rack'
+
+module RackAdapter
+  class << self
+    def run(app)
+      ->(socket, req) { respond(socket, req, app.(env(req))) }
+    end
+
+    def env(req)
+      {}
+    end
+    
+    def respond(socket, request, (status_code, headers, body))
+      body = body.join
+      headers = "Content-Type: text/plain\r\nContent-Length: #{body.bytesize}\r\n"
+      socket.write "HTTP/1.1 #{status_code}\r\n#{headers}\r\n#{body}"
+    end
+  end
+end
+
+$connection_count = 0
+
+def handle_client(socket, &handler)
+  $connection_count += 1
+  parser = Http::Parser.new
+  reqs = []
+  parser.on_message_complete = proc do |env|
+    reqs << Object.new # parser
+  end
+  socket.read_loop do |data|
+    parser << data
+    while (req = reqs.shift)
+      handler.call(socket, req)
+      req = nil
+    end
+  end
+rescue IOError, SystemCallError => e
+  # do nothing
+ensure
+  $connection_count -= 1
+  socket&.close
+end
+
+def handle_request(client, parser)
+  status_code = "200 OK"
+  data = "Hello world!\n"
+  headers = "Content-Type: text/plain\r\nContent-Length: #{data.bytesize}\r\n"
+  client.write "HTTP/1.1 #{status_code}\r\n#{headers}\r\n#{data}"
+end
+
+server = TCPServer.open('0.0.0.0', 1234)
+puts "pid #{Process.pid}"
+puts "listening on port 1234"
+
+app = RackAdapter.run(lambda { |env|
+  [
+    200,
+    {"Content-Type" => "text/plain"},
+    ["Hello, world!\n"]
+  ]
+})
+
+loop do
+  client = server.accept
+  spin { handle_client(client, &app) }
+end