polyphony 0.43.6 → 0.44.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +45 -0
  3. data/Gemfile.lock +5 -1
  4. data/README.md +20 -5
  5. data/TODO.md +10 -14
  6. data/bin/stress.rb +28 -0
  7. data/docs/getting-started/overview.md +2 -2
  8. data/examples/adapters/sequel_mysql.rb +23 -0
  9. data/examples/adapters/sequel_mysql_pool.rb +33 -0
  10. data/examples/core/xx-channels.rb +4 -2
  11. data/examples/core/xx-using-a-mutex.rb +2 -1
  12. data/examples/performance/fiber_transfer.rb +47 -0
  13. data/ext/polyphony/agent.h +41 -0
  14. data/ext/polyphony/event.c +86 -0
  15. data/ext/polyphony/fiber.c +0 -5
  16. data/ext/polyphony/libev_agent.c +201 -128
  17. data/ext/polyphony/polyphony.c +4 -2
  18. data/ext/polyphony/polyphony.h +24 -24
  19. data/ext/polyphony/polyphony_ext.c +4 -2
  20. data/ext/polyphony/queue.c +208 -0
  21. data/ext/polyphony/ring_buffer.c +0 -24
  22. data/ext/polyphony/thread.c +53 -38
  23. data/lib/polyphony.rb +13 -31
  24. data/lib/polyphony/adapters/mysql2.rb +19 -0
  25. data/lib/polyphony/adapters/sequel.rb +45 -0
  26. data/lib/polyphony/core/channel.rb +3 -34
  27. data/lib/polyphony/core/exceptions.rb +11 -0
  28. data/lib/polyphony/core/resource_pool.rb +23 -72
  29. data/lib/polyphony/core/sync.rb +12 -9
  30. data/lib/polyphony/extensions/core.rb +15 -8
  31. data/lib/polyphony/extensions/fiber.rb +4 -0
  32. data/lib/polyphony/extensions/socket.rb +9 -9
  33. data/lib/polyphony/extensions/thread.rb +1 -1
  34. data/lib/polyphony/net.rb +2 -1
  35. data/lib/polyphony/version.rb +1 -1
  36. data/polyphony.gemspec +2 -0
  37. data/test/helper.rb +2 -2
  38. data/test/test_agent.rb +2 -2
  39. data/test/test_event.rb +12 -0
  40. data/test/test_fiber.rb +17 -1
  41. data/test/test_io.rb +14 -0
  42. data/test/test_queue.rb +33 -0
  43. data/test/test_resource_pool.rb +34 -43
  44. data/test/test_signal.rb +2 -26
  45. data/test/test_socket.rb +0 -43
  46. data/test/test_trace.rb +18 -17
  47. metadata +40 -5
  48. data/ext/polyphony/libev_queue.c +0 -288
  49. data/lib/polyphony/event.rb +0 -27
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: b048cfa1e0d7cd542840ab9f2198349b2474d40a6d943b584d1bcb57593ca41d
4
- data.tar.gz: 0d9ed9fc45ec2165018e61f8befbdefb5668cfcf4b7628a648a055f5b6052e4c
3
+ metadata.gz: 04c1c1221d46b1b9169aa04c5896c8d68c20fa11d64554e76b7452710423f142
4
+ data.tar.gz: 519528e0a3dfd323620869d4a7e43e4769cf3c1b5f0ea405cf341efc6fd534f6
5
5
  SHA512:
6
- metadata.gz: a962cb0db032fd58a8db4024d9a6ad2d1be5307caff72d9508bdf879bafbd6c55a5c79e9bc69a9d3bda96d2f72fda51ed2f978e96fa0ca636458284e9e9dbbaf
7
- data.tar.gz: 2b419313309e898003399f780abcf7641de34c3350ae5d7c10c785209e3e6c12e4d741b01676a49ec1ec2f3f15ea8f9001636ec44880e13583d68168f82c9cb7
6
+ metadata.gz: de8ac460dd05b051057ffab1c155c6b2d675cc623b98549333ba1defbc99f4efb738ff4a34a9f8c14bd3cb9ab0cab5356aa34a2308daa73273a6ef8719565bc4
7
+ data.tar.gz: c26b254559e484774574555dc895cf072bda8061b92fed26d8e61e7172523fe0faf2750cd2f924db3a25b59583b673ebe5f14749fd6f1279ca2e7e7b7745490b
@@ -1,3 +1,48 @@
1
+ ## 0.44.0 2020-07-25
2
+
3
+ * Fix reentrant `ResourcePool` (#38)
4
+ * Add `ResourcePool#discard!` (#35)
5
+ * Add `Mysql2::Client` and `Sequel::ConnectionPool` adapters (#35)
6
+ * Reimplement `Kernel.trap` using `Fiber#interject`
7
+ * Add `Fiber#interject` for running arbitrary code on arbitrary fibers (#39)
8
+
9
+ ## 0.43.11 2020-07-24
10
+
11
+ * Dump uncaught exception info for forked process (#36)
12
+ * Add additional socket config options (#37)
13
+ - :reuse_port (`SO_REUSEPORT`)
14
+ - :backlog (listen backlog, default `SOMAXCONN`)
15
+ * Fix possible race condition in Queue#shift (#34)
16
+
17
+ ## 0.43.10 2020-07-23
18
+
19
+ * Fix race condition when terminating fibers (#33)
20
+ * Fix lock release in `Mutex` (#32)
21
+ * Virtualize agent interface
22
+ * Implement `LibevAgent_connect`
23
+
24
+ ## 0.43.9 2020-07-22
25
+
26
+ * Rewrite `Channel` using `Queue`
27
+ * Rewrite `Mutex` using `Queue`
28
+ * Reimplement `Event` in C to prevent cross-thread race condition
29
+ * Reimplement `ResourcePool` using `Queue`
30
+ * Implement `Queue#size`
31
+
32
+ ## 0.43.8 2020-07-21
33
+
34
+ * Rename `LibevQueue` to `Queue`
35
+ * Reimplement Event using `Agent#wait_event`
36
+ * Improve Queue shift queue performance
37
+ * Introduce `Agent#wait_event` API for waiting on asynchronous events
38
+ * Minimize `fcntl` syscalls in IO operations
39
+
40
+ ## 0.43.7 2020-07-20
41
+
42
+ * Fix memory leak in ResourcePool (#31)
43
+ * Check and adjust file position before reading (#30)
44
+ * Minor documentation fixes
45
+
1
46
  ## 0.43.6 2020-07-18
2
47
 
3
48
  * Allow brute-force interrupting with second Ctrl-C
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- polyphony (0.43.6)
4
+ polyphony (0.44.0)
5
5
 
6
6
  GEM
7
7
  remote: https://rubygems.org/
@@ -72,6 +72,7 @@ GEM
72
72
  minitest (>= 5.0)
73
73
  ruby-progressbar
74
74
  multi_xml (0.6.0)
75
+ mysql2 (0.5.3)
75
76
  parallel (1.19.1)
76
77
  parser (2.7.0.2)
77
78
  ast (~> 2.4.0)
@@ -109,6 +110,7 @@ GEM
109
110
  sass-listen (4.0.0)
110
111
  rb-fsevent (~> 0.9, >= 0.9.4)
111
112
  rb-inotify (~> 0.9, >= 0.9.7)
113
+ sequel (5.34.0)
112
114
  simplecov (0.17.1)
113
115
  docile (~> 1.1)
114
116
  json (>= 1.8, < 3)
@@ -130,11 +132,13 @@ DEPENDENCIES
130
132
  localhost (= 1.1.4)
131
133
  minitest (= 5.13.0)
132
134
  minitest-reporters (= 1.4.2)
135
+ mysql2 (= 0.5.3)
133
136
  pg (= 1.1.4)
134
137
  polyphony!
135
138
  rake-compiler (= 1.0.5)
136
139
  redis (= 4.1.0)
137
140
  rubocop (= 0.85.1)
141
+ sequel (= 5.34.0)
138
142
  simplecov (= 0.17.1)
139
143
 
140
144
  BUNDLED WITH
data/README.md CHANGED
@@ -1,10 +1,25 @@
1
- <p align="center"><img src="docs/polyphony-logo.png" /></p>
1
+ <h1 align="center">
2
+ <a href="https://digital-fabric.github.io/polyphony/">
3
+ <img src="docs/polyphony-logo.png" alt="Polyphony">
4
+ </a>
5
+ <br>
6
+ Polyphony
7
+ <br>
8
+ </h1>
2
9
 
3
- # Polyphony - Fine-Grained Concurrency for Ruby
10
+ <h4 align="center">Fine-Grained Concurrency for Ruby</h4>
4
11
 
5
- [![Gem Version](https://badge.fury.io/rb/polyphony.svg)](http://rubygems.org/gems/polyphony)
6
- [![Modulation Test](https://github.com/digital-fabric/polyphony/workflows/Tests/badge.svg)](https://github.com/digital-fabric/polyphony/actions?query=workflow%3ATests)
7
- [![MIT licensed](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/digital-fabric/polyphony/blob/master/LICENSE)
12
+ <p align="center">
13
+ <a href="http://rubygems.org/gems/polyphony">
14
+ <img src="https://badge.fury.io/rb/polyphony.svg" alt="Ruby gem">
15
+ </a>
16
+ <a href="https://github.com/digital-fabric/polyphony/actions?query=workflow%3ATests">
17
+ <img src="https://github.com/digital-fabric/polyphony/workflows/Tests/badge.svg" alt="Tests">
18
+ </a>
19
+ <a href="https://github.com/digital-fabric/polyphony/blob/master/LICENSE">
20
+ <img src="https://img.shields.io/badge/license-MIT-blue.svg" alt="MIT License">
21
+ </a>
22
+ </p>
8
23
 
9
24
  [DOCS](https://digital-fabric.github.io/polyphony/) |
10
25
  [EXAMPLES](examples)
data/TODO.md CHANGED
@@ -1,14 +1,14 @@
1
- ## 0.43
2
-
3
- - Reimplement ResourcePool, Channel, Mutex using LibevQueue
4
- -- Add `Fiber#schedule_with_priority` method, aliased by `Fiber#wakeup`
5
- - Implement agent interface is virtual function table
6
- - Implement proxy agent for plugging in a user-provided agent class
1
+ - Adapter for Pry, or maybe just an adapter for readline
2
+ - Rename Agent to Backend
3
+ - Review all code
4
+ - Cleanup C code
5
+ - Cleanup Ruby code (use Rubocop)
6
+ - Cleanup and annotate examples (and remove all the examples used for
7
+ debugging). Focus on examples that serve as "how-to".
7
8
 
8
9
  - Debugging
9
10
  - Eat your own dogfood: need a good tool to check what's going on when some
10
11
  test fails
11
- - Needs to work with Pry (can write perhaps an extension for pry)
12
12
  - First impl in Ruby using `TracePoint` API
13
13
  - Mode of operation:
14
14
  - Two parts: tracer and controller
@@ -118,7 +118,7 @@
118
118
  - discuss using `snooze` for ensuring responsiveness when executing CPU-bound work
119
119
 
120
120
 
121
- ## 0.44
121
+ ## 0.45
122
122
 
123
123
  ### Some more API work, more docs
124
124
 
@@ -131,13 +131,13 @@
131
131
  - proceed from there
132
132
 
133
133
 
134
- ## 0.45
134
+ ## 0.46
135
135
 
136
136
  ### Sinatra / Sidekiq
137
137
 
138
138
  - Pull out redis/postgres code, put into new `polyphony-xxx` gems
139
139
 
140
- ## 0.46
140
+ ## 0.47
141
141
 
142
142
  ### Testing && Docs
143
143
 
@@ -149,10 +149,6 @@
149
149
  - `IO.foreach`
150
150
  - `Process.waitpid`
151
151
 
152
- ## 0.47
153
-
154
- ### Real IO#gets and IO#read
155
-
156
152
  ## 0.48 DNS
157
153
 
158
154
  ### DNS client
@@ -0,0 +1,28 @@
1
+ #!/usr/bin/env ruby
2
+ # frozen_string_literal: true
3
+
4
+ `rake recompile`
5
+
6
+ count = ARGV[0] ? ARGV[0].to_i : 100
7
+
8
+ TEST_CMD = 'ruby test/run.rb'
9
+
10
+ def run_test(count)
11
+ puts "#{count}: running tests..."
12
+ system(TEST_CMD)
13
+ return if $?.exitstatus == 0
14
+
15
+ puts "Failure after #{count} tests"
16
+ exit!
17
+ end
18
+
19
+ trap('INT') { exit! }
20
+ t0 = Time.now
21
+ count.times { |i| run_test(i + 1) }
22
+ elapsed = Time.now - t0
23
+ puts format(
24
+ "Successfully ran %d tests in %f seconds (%f per test)",
25
+ count,
26
+ elapsed,
27
+ elapsed / count
28
+ )
@@ -113,8 +113,8 @@ active concurrent connections, each advancing at its own pace, consuming only a
113
113
  single CPU core.
114
114
 
115
115
  Nevertheless, Polyphony fully supports multithreading, with each thread having
116
- its own fiber run queue and its own libev event loop. In addition, Polyphony
117
- enables cross-thread communication using
116
+ its own fiber run queue and its own libev event loop. Polyphony even enables
117
+ cross-thread communication using [fiber messaging](#message-passing).
118
118
 
119
119
  ## Fibers vs Callbacks
120
120
 
@@ -0,0 +1,23 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'bundler/setup'
4
+ require 'polyphony/adapters/sequel'
5
+ require 'polyphony/adapters/mysql2'
6
+
7
+ time_printer = spin do
8
+ last = Time.now
9
+ throttled_loop(10) do
10
+ now = Time.now
11
+ puts now - last
12
+ last = now
13
+ end
14
+ end
15
+
16
+ db = Sequel.connect('mysql2://localhost/test')
17
+
18
+ x = 10_000
19
+ t0 = Time.now
20
+ x.times { db.execute('select 1 as test') }
21
+ puts "query rate: #{x / (Time.now - t0)} reqs/s"
22
+
23
+ time_printer.stop
@@ -0,0 +1,33 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'bundler/setup'
4
+ require 'polyphony/adapters/sequel'
5
+ require 'polyphony/adapters/mysql2'
6
+
7
+ CONCURRENCY = ARGV.first ? ARGV.first.to_i : 1000
8
+ puts "concurrency: #{CONCURRENCY}"
9
+
10
+ db = Sequel.connect(
11
+ 'mysql2://localhost/test',
12
+ max_connections: 100,
13
+ preconnect: true
14
+ )
15
+
16
+ t0 = Time.now
17
+ count = 0
18
+
19
+ fibers = Array.new(CONCURRENCY) do
20
+ spin do
21
+ loop do
22
+ db.execute('select sleep(0.001) as test')
23
+ count += 1
24
+ end
25
+ end
26
+ end
27
+
28
+ sleep 0.1
29
+ fibers.first.terminate # Interrupt mid-query
30
+
31
+ sleep 2
32
+ puts "query rate: #{count / (Time.now - t0)} reqs/s"
33
+ fibers.each(&:interrupt)
@@ -2,10 +2,12 @@
2
2
 
3
3
  require 'bundler/setup'
4
4
  require 'polyphony'
5
+ require 'polyphony/core/channel'
5
6
 
6
7
  def echo(cin, cout)
7
8
  puts 'start echoer'
8
9
  while (msg = cin.receive)
10
+ puts "echoer received #{msg}"
9
11
  cout << "you said: #{msg}"
10
12
  end
11
13
  ensure
@@ -20,7 +22,7 @@ spin do
20
22
  puts 'start receiver'
21
23
  while (msg = chan2.receive)
22
24
  puts msg
23
- $main.resume if msg =~ /world/
25
+ $main.schedule if msg =~ /world/
24
26
  end
25
27
  ensure
26
28
  puts 'receiver stopped'
@@ -42,4 +44,4 @@ $main = spin do
42
44
  puts "done #{Time.now - t0}"
43
45
  end
44
46
 
45
- suspend
47
+ $main.await
@@ -2,6 +2,7 @@
2
2
 
3
3
  require 'bundler/setup'
4
4
  require 'polyphony'
5
+ require 'polyphony/core/sync'
5
6
 
6
7
  def loop_it(number, lock)
7
8
  loop do
@@ -13,7 +14,7 @@ def loop_it(number, lock)
13
14
  end
14
15
  end
15
16
 
16
- lock = Polyphony::Sync::Mutex.new
17
+ lock = Polyphony::Mutex.new
17
18
  spin { loop_it(1, lock) }
18
19
  spin { loop_it(2, lock) }
19
20
  spin { loop_it(3, lock) }
@@ -0,0 +1,47 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'fiber'
4
+
5
+ class Fiber
6
+ attr_accessor :next
7
+ end
8
+
9
+ # This program shows how the performance
10
+
11
+ def run(num_fibers)
12
+ count = 0
13
+
14
+ GC.disable
15
+
16
+ first = nil
17
+ last = nil
18
+ supervisor = Fiber.current
19
+ num_fibers.times do
20
+ fiber = Fiber.new do
21
+ loop do
22
+ count += 1
23
+ if count == 1_000_000
24
+ supervisor.transfer
25
+ else
26
+ Fiber.current.next.transfer
27
+ end
28
+ end
29
+ end
30
+ first ||= fiber
31
+ last.next = fiber if last
32
+ last = fiber
33
+ end
34
+
35
+ last.next = first
36
+
37
+ t0 = Time.now
38
+ first.transfer
39
+ elapsed = Time.now - t0
40
+
41
+ puts "fibers: #{num_fibers} count: #{count} rate: #{count / elapsed}"
42
+ GC.start
43
+ end
44
+
45
+ run(100)
46
+ run(1000)
47
+ run(10000)
@@ -0,0 +1,41 @@
1
+ #ifndef AGENT_H
2
+ #define AGENT_H
3
+
4
+ #include "ruby.h"
5
+
6
+ // agent interface function signatures
7
+
8
+ // VALUE LibevAgent_accept(VALUE self, VALUE sock);
9
+ // VALUE LibevAgent_accept_loop(VALUE self, VALUE sock);
10
+ // VALUE libev_agent_await(VALUE self);
11
+ // VALUE LibevAgent_connect(VALUE self, VALUE sock, VALUE host, VALUE port);
12
+ // VALUE LibevAgent_finalize(VALUE self);
13
+ // VALUE LibevAgent_post_fork(VALUE self);
14
+ // VALUE LibevAgent_read(VALUE self, VALUE io, VALUE str, VALUE length, VALUE to_eof);
15
+ // VALUE LibevAgent_read_loop(VALUE self, VALUE io);
16
+ // VALUE LibevAgent_sleep(VALUE self, VALUE duration);
17
+ // VALUE LibevAgent_wait_io(VALUE self, VALUE io, VALUE write);
18
+ // VALUE LibevAgent_wait_pid(VALUE self, VALUE pid);
19
+ // VALUE LibevAgent_write(int argc, VALUE *argv, VALUE self);
20
+
21
+ typedef VALUE (* agent_pending_count_t)(VALUE self);
22
+ typedef VALUE (*agent_poll_t)(VALUE self, VALUE nowait, VALUE current_fiber, VALUE queue);
23
+ typedef VALUE (* agent_ref_t)(VALUE self);
24
+ typedef int (* agent_ref_count_t)(VALUE self);
25
+ typedef void (* agent_reset_ref_count_t)(VALUE self);
26
+ typedef VALUE (* agent_unref_t)(VALUE self);
27
+ typedef VALUE (* agent_wait_event_t)(VALUE self, VALUE raise_on_exception);
28
+ typedef VALUE (* agent_wakeup_t)(VALUE self);
29
+
30
+ typedef struct agent_interface {
31
+ agent_pending_count_t pending_count;
32
+ agent_poll_t poll;
33
+ agent_ref_t ref;
34
+ agent_ref_count_t ref_count;
35
+ agent_reset_ref_count_t reset_ref_count;
36
+ agent_unref_t unref;
37
+ agent_wait_event_t wait_event;
38
+ agent_wakeup_t wakeup;
39
+ } agent_interface_t;
40
+
41
+ #endif /* AGENT_H */
@@ -0,0 +1,86 @@
1
+ #include "polyphony.h"
2
+ #include "ring_buffer.h"
3
+
4
+ typedef struct event {
5
+ VALUE waiting_fiber;
6
+ } Event_t;
7
+
8
+ VALUE cEvent = Qnil;
9
+
10
+ static void Event_mark(void *ptr) {
11
+ Event_t *event = ptr;
12
+ rb_gc_mark(event->waiting_fiber);
13
+ }
14
+
15
+ static void Event_free(void *ptr) {
16
+ xfree(ptr);
17
+ }
18
+
19
+ static size_t Event_size(const void *ptr) {
20
+ return sizeof(Event_t);
21
+ }
22
+
23
+ static const rb_data_type_t Event_type = {
24
+ "Event",
25
+ {Event_mark, Event_free, Event_size,},
26
+ 0, 0, 0
27
+ };
28
+
29
+ static VALUE Event_allocate(VALUE klass) {
30
+ Event_t *event;
31
+
32
+ event = ALLOC(Event_t);
33
+ return TypedData_Wrap_Struct(klass, &Event_type, event);
34
+ }
35
+
36
+ #define GetEvent(obj, event) \
37
+ TypedData_Get_Struct((obj), Event_t, &Event_type, (event))
38
+
39
+ static VALUE Event_initialize(VALUE self) {
40
+ Event_t *event;
41
+ GetEvent(self, event);
42
+
43
+ event->waiting_fiber = Qnil;
44
+
45
+ return self;
46
+ }
47
+
48
+ VALUE Event_signal(int argc, VALUE *argv, VALUE self) {
49
+ VALUE value = argc > 0 ? argv[0] : Qnil;
50
+ Event_t *event;
51
+ GetEvent(self, event);
52
+
53
+ if (event->waiting_fiber != Qnil) {
54
+ Fiber_make_runnable(event->waiting_fiber, value);
55
+ event->waiting_fiber = Qnil;
56
+ }
57
+ return self;
58
+ }
59
+
60
+ VALUE Event_await(VALUE self) {
61
+ Event_t *event;
62
+ GetEvent(self, event);
63
+
64
+ if (event->waiting_fiber != Qnil)
65
+ rb_raise(rb_eRuntimeError, "Event is already awaited by another fiber");
66
+
67
+ VALUE agent = rb_ivar_get(rb_thread_current(), ID_ivar_agent);
68
+ event->waiting_fiber = rb_fiber_current();
69
+ VALUE switchpoint_result = __AGENT__.wait_event(agent, Qnil);
70
+ event->waiting_fiber = Qnil;
71
+
72
+ TEST_RESUME_EXCEPTION(switchpoint_result);
73
+ RB_GC_GUARD(agent);
74
+ RB_GC_GUARD(switchpoint_result);
75
+
76
+ return switchpoint_result;
77
+ }
78
+
79
+ void Init_Event() {
80
+ cEvent = rb_define_class_under(mPolyphony, "Event", rb_cData);
81
+ rb_define_alloc_func(cEvent, Event_allocate);
82
+
83
+ rb_define_method(cEvent, "initialize", Event_initialize, 0);
84
+ rb_define_method(cEvent, "await", Event_await, 0);
85
+ rb_define_method(cEvent, "signal", Event_signal, -1);
86
+ }