polyphony 0.43.6 → 0.44.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b048cfa1e0d7cd542840ab9f2198349b2474d40a6d943b584d1bcb57593ca41d
-  data.tar.gz: 0d9ed9fc45ec2165018e61f8befbdefb5668cfcf4b7628a648a055f5b6052e4c
+  metadata.gz: 04c1c1221d46b1b9169aa04c5896c8d68c20fa11d64554e76b7452710423f142
+  data.tar.gz: 519528e0a3dfd323620869d4a7e43e4769cf3c1b5f0ea405cf341efc6fd534f6
 SHA512:
-  metadata.gz: a962cb0db032fd58a8db4024d9a6ad2d1be5307caff72d9508bdf879bafbd6c55a5c79e9bc69a9d3bda96d2f72fda51ed2f978e96fa0ca636458284e9e9dbbaf
-  data.tar.gz: 2b419313309e898003399f780abcf7641de34c3350ae5d7c10c785209e3e6c12e4d741b01676a49ec1ec2f3f15ea8f9001636ec44880e13583d68168f82c9cb7
+  metadata.gz: de8ac460dd05b051057ffab1c155c6b2d675cc623b98549333ba1defbc99f4efb738ff4a34a9f8c14bd3cb9ab0cab5356aa34a2308daa73273a6ef8719565bc4
+  data.tar.gz: c26b254559e484774574555dc895cf072bda8061b92fed26d8e61e7172523fe0faf2750cd2f924db3a25b59583b673ebe5f14749fd6f1279ca2e7e7b7745490b

data/CHANGELOG.md

@@ -1,3 +1,48 @@
+## 0.44.0 2020-07-25
+
+* Fix reentrant `ResourcePool` (#38)
+* Add `ResourcePool#discard!` (#35)
+* Add `Mysql2::Client` and `Sequel::ConnectionPool` adapters (#35)
+* Reimplement `Kernel.trap` using `Fiber#interject`
+* Add `Fiber#interject` for running arbitrary code on arbitrary fibers (#39)
+
+## 0.43.11 2020-07-24
+
+* Dump uncaught exception info for forked process (#36)
+* Add additional socket config options (#37)
+  - :reuse_port (`SO_REUSEPORT`)
+  - :backlog (listen backlog, default `SOMAXCONN`)
+* Fix possible race condition in Queue#shift (#34)
+
+## 0.43.10 2020-07-23
+
+* Fix race condition when terminating fibers (#33)
+* Fix lock release in `Mutex` (#32)
+* Virtualize agent interface
+* Implement `LibevAgent_connect`
+
+## 0.43.9 2020-07-22
+
+* Rewrite `Channel` using `Queue`
+* Rewrite `Mutex` using `Queue`
+* Reimplement `Event` in C to prevent cross-thread race condition
+* Reimplement `ResourcePool` using `Queue`
+* Implement `Queue#size`
+
+## 0.43.8 2020-07-21
+
+* Rename `LibevQueue` to `Queue`
+* Reimplement Event using `Agent#wait_event`
+* Improve Queue shift queue performance
+* Introduce `Agent#wait_event` API for waiting on asynchronous events
+* Minimize `fcntl` syscalls in IO operations 
+
+## 0.43.7 2020-07-20
+
+* Fix memory leak in ResourcePool (#31)
+* Check and adjust file position before reading (#30)
+* Minor documentation fixes
+
 ## 0.43.6 2020-07-18
 
 * Allow brute-force interrupting with second Ctrl-C

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    polyphony (0.43.6)
+    polyphony (0.44.0)
 
 GEM
   remote: https://rubygems.org/
@@ -72,6 +72,7 @@ GEM
       minitest (>= 5.0)
       ruby-progressbar
     multi_xml (0.6.0)
+    mysql2 (0.5.3)
     parallel (1.19.1)
     parser (2.7.0.2)
       ast (~> 2.4.0)
@@ -109,6 +110,7 @@ GEM
     sass-listen (4.0.0)
       rb-fsevent (~> 0.9, >= 0.9.4)
       rb-inotify (~> 0.9, >= 0.9.7)
+    sequel (5.34.0)
     simplecov (0.17.1)
       docile (~> 1.1)
       json (>= 1.8, < 3)
@@ -130,11 +132,13 @@ DEPENDENCIES
   localhost (= 1.1.4)
   minitest (= 5.13.0)
   minitest-reporters (= 1.4.2)
+  mysql2 (= 0.5.3)
   pg (= 1.1.4)
   polyphony!
   rake-compiler (= 1.0.5)
   redis (= 4.1.0)
   rubocop (= 0.85.1)
+  sequel (= 5.34.0)
   simplecov (= 0.17.1)
 
 BUNDLED WITH

data/README.md

@@ -1,10 +1,25 @@
-<p align="center"><img src="docs/polyphony-logo.png" /></p>
+<h1 align="center">
+  <a href="https://digital-fabric.github.io/polyphony/">
+    <img src="docs/polyphony-logo.png" alt="Polyphony">
+  </a>
+  <br>
+  Polyphony
+  <br>
+</h1>
 
-# Polyphony - Fine-Grained Concurrency for Ruby
+<h4 align="center">Fine-Grained Concurrency for Ruby</h4>
 
-[![Gem Version](https://badge.fury.io/rb/polyphony.svg)](http://rubygems.org/gems/polyphony)
-[![Modulation Test](https://github.com/digital-fabric/polyphony/workflows/Tests/badge.svg)](https://github.com/digital-fabric/polyphony/actions?query=workflow%3ATests)
-[![MIT licensed](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/digital-fabric/polyphony/blob/master/LICENSE)
+<p align="center">
+  <a href="http://rubygems.org/gems/polyphony">
+    <img src="https://badge.fury.io/rb/polyphony.svg" alt="Ruby gem">
+  </a>
+  <a href="https://github.com/digital-fabric/polyphony/actions?query=workflow%3ATests">
+    <img src="https://github.com/digital-fabric/polyphony/workflows/Tests/badge.svg" alt="Tests">
+  </a>
+  <a href="https://github.com/digital-fabric/polyphony/blob/master/LICENSE">
+    <img src="https://img.shields.io/badge/license-MIT-blue.svg" alt="MIT License">
+  </a>
+</p>
 
 [DOCS](https://digital-fabric.github.io/polyphony/) |
 [EXAMPLES](examples)

data/TODO.md

@@ -1,14 +1,14 @@
-## 0.43 
-
-- Reimplement ResourcePool, Channel, Mutex using LibevQueue
--- Add `Fiber#schedule_with_priority` method, aliased by `Fiber#wakeup`
-- Implement agent interface is virtual function table
-- Implement proxy agent for plugging in a user-provided agent class
+- Adapter for Pry, or maybe just an adapter for readline
+- Rename Agent to Backend
+- Review all code
+  - Cleanup C code
+  - Cleanup Ruby code (use Rubocop)
+  - Cleanup and annotate examples (and remove all the examples used for
+    debugging). Focus on examples that serve as "how-to".
 
 - Debugging
   - Eat your own dogfood: need a good tool to check what's going on when some
     test fails
-  - Needs to work with Pry (can write perhaps an extension for pry)
   - First impl in Ruby using `TracePoint` API
   - Mode of operation:
     - Two parts: tracer and controller
@@ -118,7 +118,7 @@
   - discuss using `snooze` for ensuring responsiveness when executing CPU-bound work
 
 
-## 0.44
+## 0.45
 
 ### Some more API work, more docs
 
@@ -131,13 +131,13 @@
   - proceed from there
 
 
-## 0.45
+## 0.46
 
 ### Sinatra / Sidekiq
 
 - Pull out redis/postgres code, put into new `polyphony-xxx` gems
 
-## 0.46
+## 0.47
 
 ### Testing && Docs
 
@@ -149,10 +149,6 @@
   - `IO.foreach`
   - `Process.waitpid`
 
-## 0.47
-
-### Real IO#gets and IO#read
-
 ## 0.48 DNS
 
 ### DNS client

data/bin/stress.rb

@@ -0,0 +1,28 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+`rake recompile`
+
+count = ARGV[0] ? ARGV[0].to_i : 100
+
+TEST_CMD = 'ruby test/run.rb'
+
+def run_test(count)
+  puts "#{count}: running tests..."
+  system(TEST_CMD)
+  return if $?.exitstatus == 0
+
+  puts "Failure after #{count} tests"
+  exit!
+end
+
+trap('INT') { exit! }
+t0 = Time.now
+count.times { |i| run_test(i + 1) }
+elapsed = Time.now - t0
+puts format(
+  "Successfully ran %d tests in %f seconds (%f per test)",
+  count,
+  elapsed,
+  elapsed / count
+)

data/docs/getting-started/overview.md

@@ -113,8 +113,8 @@ active concurrent connections, each advancing at its own pace, consuming only a
 single CPU core.
 
 Nevertheless, Polyphony fully supports multithreading, with each thread having
-its own fiber run queue and its own libev event loop. In addition, Polyphony
-enables cross-thread communication using  
+its own fiber run queue and its own libev event loop. Polyphony even enables
+cross-thread communication using [fiber messaging](#message-passing). 
 
 ## Fibers vs Callbacks
 

data/examples/adapters/sequel_mysql.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'polyphony/adapters/sequel'
+require 'polyphony/adapters/mysql2'
+
+time_printer = spin do
+  last = Time.now
+  throttled_loop(10) do
+    now = Time.now
+    puts now - last
+    last = now
+  end
+end
+
+db = Sequel.connect('mysql2://localhost/test')
+
+x = 10_000
+t0 = Time.now
+x.times { db.execute('select 1 as test') }
+puts "query rate: #{x / (Time.now - t0)} reqs/s"
+
+time_printer.stop

data/examples/adapters/sequel_mysql_pool.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'polyphony/adapters/sequel'
+require 'polyphony/adapters/mysql2'
+
+CONCURRENCY = ARGV.first ? ARGV.first.to_i : 1000
+puts "concurrency: #{CONCURRENCY}"
+
+db = Sequel.connect(
+  'mysql2://localhost/test',
+  max_connections: 100,
+  preconnect: true
+)
+
+t0 = Time.now
+count = 0
+
+fibers = Array.new(CONCURRENCY) do
+  spin do
+    loop do
+      db.execute('select sleep(0.001) as test')
+      count += 1
+    end
+  end
+end
+
+sleep 0.1
+fibers.first.terminate # Interrupt mid-query
+
+sleep 2
+puts "query rate: #{count / (Time.now - t0)} reqs/s"
+fibers.each(&:interrupt)

data/examples/core/xx-channels.rb

@@ -2,10 +2,12 @@
 
 require 'bundler/setup'
 require 'polyphony'
+require 'polyphony/core/channel'
 
 def echo(cin, cout)
   puts 'start echoer'
   while (msg = cin.receive)
+    puts "echoer received #{msg}"
     cout << "you said: #{msg}"
   end
 ensure
@@ -20,7 +22,7 @@ spin do
   puts 'start receiver'
   while (msg = chan2.receive)
     puts msg
-    $main.resume if msg =~ /world/
+    $main.schedule if msg =~ /world/
   end
 ensure
   puts 'receiver stopped'
@@ -42,4 +44,4 @@ $main = spin do
   puts "done #{Time.now - t0}"
 end
 
-suspend
+$main.await

data/examples/core/xx-using-a-mutex.rb

@@ -2,6 +2,7 @@
 
 require 'bundler/setup'
 require 'polyphony'
+require 'polyphony/core/sync'
 
 def loop_it(number, lock)
   loop do
@@ -13,7 +14,7 @@ def loop_it(number, lock)
   end
 end
 
-lock = Polyphony::Sync::Mutex.new
+lock = Polyphony::Mutex.new
 spin { loop_it(1, lock) }
 spin { loop_it(2, lock) }
 spin { loop_it(3, lock) }

data/examples/performance/fiber_transfer.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require 'fiber'
+
+class Fiber
+  attr_accessor :next
+end
+
+# This program shows how the performance 
+
+def run(num_fibers)
+  count = 0
+
+  GC.disable
+
+  first = nil
+  last = nil
+  supervisor = Fiber.current
+  num_fibers.times do
+    fiber = Fiber.new do
+      loop do
+        count += 1
+        if count == 1_000_000
+          supervisor.transfer
+        else
+          Fiber.current.next.transfer
+        end
+      end
+    end
+    first ||= fiber
+    last.next = fiber if last
+    last = fiber
+  end
+
+  last.next = first
+  
+  t0 = Time.now
+  first.transfer
+  elapsed = Time.now - t0
+
+  puts "fibers: #{num_fibers} count: #{count} rate: #{count / elapsed}"
+  GC.start
+end
+
+run(100)
+run(1000)
+run(10000)

data/ext/polyphony/agent.h

@@ -0,0 +1,41 @@
+#ifndef AGENT_H
+#define AGENT_H
+
+#include "ruby.h"
+
+// agent interface function signatures 
+
+// VALUE LibevAgent_accept(VALUE self, VALUE sock);
+// VALUE LibevAgent_accept_loop(VALUE self, VALUE sock);
+// VALUE libev_agent_await(VALUE self);
+// VALUE LibevAgent_connect(VALUE self, VALUE sock, VALUE host, VALUE port);
+// VALUE LibevAgent_finalize(VALUE self);
+// VALUE LibevAgent_post_fork(VALUE self);
+// VALUE LibevAgent_read(VALUE self, VALUE io, VALUE str, VALUE length, VALUE to_eof);
+// VALUE LibevAgent_read_loop(VALUE self, VALUE io);
+// VALUE LibevAgent_sleep(VALUE self, VALUE duration);
+// VALUE LibevAgent_wait_io(VALUE self, VALUE io, VALUE write);
+// VALUE LibevAgent_wait_pid(VALUE self, VALUE pid);
+// VALUE LibevAgent_write(int argc, VALUE *argv, VALUE self);
+
+typedef VALUE (* agent_pending_count_t)(VALUE self);
+typedef VALUE (*agent_poll_t)(VALUE self, VALUE nowait, VALUE current_fiber, VALUE queue);
+typedef VALUE (* agent_ref_t)(VALUE self);
+typedef int (* agent_ref_count_t)(VALUE self);
+typedef void (* agent_reset_ref_count_t)(VALUE self);
+typedef VALUE (* agent_unref_t)(VALUE self);
+typedef VALUE (* agent_wait_event_t)(VALUE self, VALUE raise_on_exception);
+typedef VALUE (* agent_wakeup_t)(VALUE self);
+
+typedef struct agent_interface {
+  agent_pending_count_t   pending_count;
+  agent_poll_t            poll;
+  agent_ref_t             ref;
+  agent_ref_count_t       ref_count;
+  agent_reset_ref_count_t reset_ref_count;
+  agent_unref_t           unref;
+  agent_wait_event_t      wait_event;
+  agent_wakeup_t          wakeup;
+} agent_interface_t;
+
+#endif /* AGENT_H */

data/ext/polyphony/event.c

@@ -0,0 +1,86 @@
+#include "polyphony.h"
+#include "ring_buffer.h"
+
+typedef struct event {
+  VALUE waiting_fiber;
+} Event_t;
+
+VALUE cEvent = Qnil;
+
+static void Event_mark(void *ptr) {
+  Event_t *event = ptr;
+  rb_gc_mark(event->waiting_fiber);
+}
+
+static void Event_free(void *ptr) {
+  xfree(ptr);
+}
+
+static size_t Event_size(const void *ptr) {
+  return sizeof(Event_t);
+}
+
+static const rb_data_type_t Event_type = {
+  "Event",
+  {Event_mark, Event_free, Event_size,},
+  0, 0, 0
+};
+
+static VALUE Event_allocate(VALUE klass) {
+  Event_t *event;
+
+  event = ALLOC(Event_t);
+  return TypedData_Wrap_Struct(klass, &Event_type, event);
+}
+
+#define GetEvent(obj, event) \
+  TypedData_Get_Struct((obj), Event_t, &Event_type, (event))
+
+static VALUE Event_initialize(VALUE self) {
+  Event_t *event;
+  GetEvent(self, event);
+
+  event->waiting_fiber = Qnil;
+
+  return self;
+}
+
+VALUE Event_signal(int argc, VALUE *argv, VALUE self) {
+  VALUE value = argc > 0 ? argv[0] : Qnil;
+  Event_t *event;
+  GetEvent(self, event);
+
+  if (event->waiting_fiber != Qnil) {
+    Fiber_make_runnable(event->waiting_fiber, value);
+    event->waiting_fiber = Qnil;
+  }
+  return self;
+}
+
+VALUE Event_await(VALUE self) {
+  Event_t *event;
+  GetEvent(self, event);
+
+  if (event->waiting_fiber != Qnil)
+    rb_raise(rb_eRuntimeError, "Event is already awaited by another fiber");
+
+  VALUE agent = rb_ivar_get(rb_thread_current(), ID_ivar_agent);
+  event->waiting_fiber = rb_fiber_current();
+  VALUE switchpoint_result = __AGENT__.wait_event(agent, Qnil);
+  event->waiting_fiber = Qnil;
+
+  TEST_RESUME_EXCEPTION(switchpoint_result);
+  RB_GC_GUARD(agent);
+  RB_GC_GUARD(switchpoint_result);
+
+  return switchpoint_result;
+}
+
+void Init_Event() {
+  cEvent = rb_define_class_under(mPolyphony, "Event", rb_cData);
+  rb_define_alloc_func(cEvent, Event_allocate);
+
+  rb_define_method(cEvent, "initialize", Event_initialize, 0);
+  rb_define_method(cEvent, "await", Event_await, 0);
+  rb_define_method(cEvent, "signal", Event_signal, -1);
+}