rbgo 0.2.0 → 0.2.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5bccb96948128dee0adbfb357a273c55af918c35148099b90280e09c4af29a88
-  data.tar.gz: 753b306467acaf5b46882b57a42a38d05c64a65496d6d285dca08e99330dc197
+  metadata.gz: d77329adc2e8773f5ffb22ef40da51ad2f4451f2032804e684a4e7287df6c387
+  data.tar.gz: dd92c1cd5c9ec8f0fa307cc1d8ae6731d51cb70e39d02fe9641aba0dd8d598f9
 SHA512:
-  metadata.gz: '090379aa03738950d87b5b399a83f5e11b3aef62e6189e94b9df633f6ad7db41f5b5b53dd871b73831b6c1e09d210d71221d9702ecd1d844687d9d6d37128902'
-  data.tar.gz: b1689789770582e853220b084157b3477e7f095e1e0ea4725ef9b7f8b46572e0e71dc4c55c8d78b76cc5adeb12e5dbacd972813728ae8b0dcc66fa9a18e9629b
+  metadata.gz: b5519621d6a4c7f2354037157901a86d2a060ca71d459ca9ac42f692e6c9f8dab32138dfeb571ea3a562f640f4118bcbaa720a79175724200259a1cb51a8ab78
+  data.tar.gz: e41d09d68fbe4d61222408d6899625627edebf56f2141b6c13a1e5e45a4ad181149520361b8a27a7f236b283875e8d0042dc989a059511e205c0e7814f9e5f00

data/README.md

@@ -130,19 +130,111 @@ actor = Rbgo::Actor.new do|msg, actor|
 actor.send_msg :msg1 #won't block
 
 ```            
+# IOMachine
+IOMachine wrap nio4r to do IO operation asynchronously.
+
+support platforms: MRI, JRuby.
+
+*Not supported platform: Truffleruby*
+
+
+```ruby  
+require 'rbgo'
+
+io_r, io_w = IO.pipe
+machine = Rbgo::IOMachine.new
+
+receipt1 = machine.do_read(io_r, length: 100) # when length > 0, result nil if have not read anything yet when read complete.
+receipt2 = machine.do_read(io_r, length: 100) # when length >0, result data bytes length up to 100 if have read some when read complete.
+receipt3 = machine.do_read(io_r, length: 0) # when length == 0, result ""
+receipt4 = machine.do_read(io_r, length: nil) # when length == nil, read until EOF. return "" if have not read anything.
+
+io_w.write("a"*100)
+io_w.write("b"*100)
+io_w.write("c"*100)
+io_w.close # cause EOF
+
+# if the same io object, and the same read/write operation, operations will complete in sequence.
+# so receipt1 complete first, and the receipt2 ...
+# if the same io object, but not the same read/write operation,
+# or not the same io object, operations will complete in arbitrary order.
+receipt1.wait
+p receipt1.res # aaa...
+receipt2.wait
+p receipt2.res # bbb...
+receipt3.wait
+p receipt3.res # ""
+receipt4.wait
+p receipt4.res # ccc...
+
+io_r, io_w = IO.pipe
+receipt1 = machine.do_write(io_w, str: "hello world!")
+
+receipt1.wait
+p receipt1.res # number of bytes written, may be less than str.bytesize if exception raised
+
+```          
+
+# yield_read / yield_write
+use IOMachine to do IO operation asynchronously, but write code in a sequential way. 
+
+No callback and another callback...
+```ruby                   
+require 'rbgo'
+using Rbgo::CoRunExtensions
+
+io_r, io_w = IO.pipe
+go do
+  data = io_r.yield_read(100) # have blocking semantics, but execute asynchronously
+                              # this operation will *NOT* block the current thread
+                              # when io operation complete, thread will resume to execute from this point.
+  p data
+end
+
+sleep 2
+
+io_w.write("haha I'm crazy.")
+io_w.close
+
+# NOTICE yield_read / yield_write will do yield only in CoRun::Routine.new(*args, new_thread: false, &blk) block
+# in other place yield_read / yield_write will do normal IO#read / IO#write
+# for example: 
+# go do
+#   io_r.yield_read # will do yield
+# end
+# 
+# go do
+#   fiber = Fiber.new do
+#     io_r.yield_read # will not do yield. just do normal IO#read
+#   end
+#   fiber.resume
+# end
+# 
+# go! do
+#   io_r.yield_read # will not do yield. just do normal IO#read
+# end
+
+```
+
+
 # NetworkService
 
 open TCP or UDP service 
 
 Because service handles network request in async mode, it can handle many requests concurrently. If use some Non-GIL ruby implementations such as TruffleRuby or JRuby, it can utilize all your CPU cores.  
 
+Use IO#yield_read IO#yield_write to do IO operations
 
 ```ruby  
 require 'rbgo'
 
+using Rbgo::CoRunExtensions
+
 #localhost, port 3000
 tcp_service = Rbgo::NetworkServiceFactory.open_tcp_service(3000) do|sock, clientAddrInfo|
   p [sock, clientAddrInfo] 
+  p sock.yield_read
+  sock.yield_write("hello")
   sock.close  #SHOULD CLOSE SOCK MANUALLY since version 0.2.0
 end                  
 

data/lib/rbgo/actor.rb

@@ -1,7 +1,5 @@
 require 'thread'
-require_relative 'corun'
 
-using Rbgo::CoRunExtensions
 
 module Rbgo
   class Actor
@@ -37,7 +35,7 @@ module Rbgo
     private
 
     def start_msg_loop
-      go! do
+      CoRun::Routine.new(new_thread: true) do
         while msg = mail_box.deq
           handler.call(msg, self) rescue nil
         end

data/lib/rbgo/corun.rb

@@ -2,9 +2,35 @@ require 'thread'
 require 'fiber'
 require 'system'
 require 'singleton'
+require_relative 'io_machine'
 
 module Rbgo
   module CoRun
+    IS_CORUN_FIBER     = :is_corun_fiber_bbc0f70e
+    YIELD_IO_OPERATION = :yield_bbc0f70e
+
+    def self.is_in_corun_fiber?
+      !!Thread.current[IS_CORUN_FIBER]
+    end
+
+    def self.read_from(io, length: nil)
+      if is_in_corun_fiber?
+        return "" if length == 0
+        receipt = Scheduler.instance.io_machine.do_read(io, length: length)
+        Fiber.yield [YIELD_IO_OPERATION, receipt]
+      else
+        io.read(length)
+      end
+    end
+
+    def self.write_to(io, str:)
+      if is_in_corun_fiber?
+        receipt = Scheduler.instance.io_machine.do_write(io, str: str)
+        Fiber.yield [YIELD_IO_OPERATION, receipt]
+      else
+        io.write(str)
+      end
+    end
 
     class Routine
       attr_accessor :error
@@ -16,7 +42,7 @@ module Rbgo
 
       private
 
-      attr_accessor :args, :blk, :fiber
+      attr_accessor :args, :blk, :fiber, :io_receipt
 
       def initialize(*args, new_thread: false, &blk)
         self.args = args
@@ -33,6 +59,7 @@ module Rbgo
               end
             rescue Exception => ex
               self.error = ex
+              STDERR.puts ex
             end
           end
 
@@ -43,11 +70,28 @@ module Rbgo
 
       def perform
         self.fiber = Fiber.new do |args|
+          Thread.current[IS_CORUN_FIBER] = true
           blk.call(*args)
         end if fiber.nil?
 
         if fiber.alive?
-          fiber.resume(*args)
+          if io_receipt.nil?
+            obj = fiber.resume(*args)
+          else
+            if io_receipt.done_flag
+              obj = fiber.resume(io_receipt.res)
+            else
+              return nil
+            end
+          end
+          if obj.is_a?(Array) &&
+            obj.size == 2 &&
+            obj[0] == YIELD_IO_OPERATION &&
+            obj[1].is_a?(IOReceipt)
+            self.io_receipt = obj[1]
+          else
+            self.io_receipt = nil
+          end
         end
         nil
       end
@@ -56,7 +100,7 @@ module Rbgo
     class Scheduler
 
       include Singleton
-      attr_accessor :num_thread, :check_interval
+      attr_accessor :num_thread, :check_interval, :io_machine
 
       private
 
@@ -66,7 +110,7 @@ module Rbgo
       attr_accessor :supervisor_thread
 
       def initialize
-        self.num_thread  = System::CPU.count rescue 8
+        self.num_thread = System::CPU.count rescue 8
         self.thread_pool = []
 
         self.msg_queue  = Queue.new
@@ -74,6 +118,8 @@ module Rbgo
 
         self.check_interval = 0.1
 
+        self.io_machine = IOMachine.new
+
         msg_queue << :init
         create_supervisor_thread
         generate_check_msg
@@ -85,22 +131,38 @@ module Rbgo
           thread_pool << Thread.new do
             Thread.current.report_on_exception = false
             begin
-              should_exit      = false
-              yield_task_queue = Queue.new
-              local_task_queue = Queue.new
+              should_exit           = false
+              yield_task_queue      = Queue.new
+              pending_io_task_queue = Queue.new
+              local_task_queue      = Queue.new
               loop do
                 task = nil
                 if local_task_queue.empty?
                   task = task_queue.deq(true) rescue nil
                   if task.nil?
                     task = yield_task_queue.deq unless yield_task_queue.empty?
-                    task = task_queue.deq if task.nil?
-                    local_task_queue << task
-                  else
-                    local_task_queue << task
-                    local_task_queue << yield_task_queue.deq unless yield_task_queue.empty?
+                    if task.nil?
+                      task = pending_io_task_queue.deq unless pending_io_task_queue.empty?
+                      if task.nil?
+                        task = task_queue.deq
+                      else
+                        sleep 0.1 # only pending io tasks in queue
+                      end
+                    else
+                      receipt = task.send(:io_receipt)
+                      if receipt
+                        pending_io_task_queue << task unless receipt.done_flag
+                        next
+                      end
+                    end
                   end
+
+                  local_task_queue << task
+                  local_task_queue << yield_task_queue.deq unless yield_task_queue.empty?
+                  local_task_queue << pending_io_task_queue.deq unless pending_io_task_queue.empty?
+
                 end
+
                 task = local_task_queue.deq
 
                 begin
@@ -108,6 +170,7 @@ module Rbgo
                   task.send :perform
                 rescue Exception => ex
                   task.error = ex
+                  STDERR.puts(ex)
                   next
                 ensure
                   Thread.current.thread_variable_set(:performing, false)
@@ -119,6 +182,7 @@ module Rbgo
 
                 should_exit = Thread.current.thread_variable_get(:should_exit) &&
                   yield_task_queue.empty? &&
+                  pending_io_task_queue.empty? &&
                   local_task_queue.empty?
                 break if should_exit
               end
@@ -202,6 +266,8 @@ module Rbgo
     end
   end
 
+  CoRun.freeze
+
   module CoRunExtensions
     refine Object do
       def go(*args, &blk)
@@ -212,5 +278,15 @@ module Rbgo
         CoRun::Routine.new(*args, new_thread: true, &blk)
       end
     end
+
+    refine IO do
+      def yield_read(len = nil)
+        CoRun.read_from(self, length: len)
+      end
+
+      def yield_write(str)
+        CoRun.write_to(self, str: str)
+      end
+    end
   end
 end

data/lib/rbgo/io_machine.rb

@@ -0,0 +1,266 @@
+require 'nio'
+require_relative 'actor'
+
+module Rbgo
+  class IOReceipt
+    attr_reader :registered_op, :done_flag
+    attr_accessor :res
+
+    def wait
+      mutex.synchronize do
+        until done_flag
+          cond.wait(mutex)
+        end
+      end
+      nil
+    end
+
+    def notify
+      mutex.synchronize do
+        self.done_flag = true
+        cond.signal
+      end
+      nil
+    end
+
+    private
+
+    attr_accessor :mutex, :cond
+    attr_writer :registered_op, :done_flag
+
+    def initialize(op)
+      self.done_flag     = false
+      self.mutex         = Mutex.new
+      self.cond          = ConditionVariable.new
+      self.registered_op = op
+    end
+  end
+
+
+  class IOMachine
+    def do_read(io, length: nil)
+      op      = [:register_read, io, length]
+      receipt = IOReceipt.new(op)
+      actor.send_msg(receipt)
+      receipt
+    end
+
+    def do_write(io, str:)
+      op      = [:register_write, io, str]
+      receipt = IOReceipt.new(op)
+      actor.send_msg(receipt)
+      receipt
+    end
+
+    def close
+      actor.close
+      selector.close
+    end
+
+    def closed?
+      actor.closed? && selector.closed?
+    end
+
+    private
+
+    attr_accessor :selector, :actor, :monitors
+
+    def initialize
+      self.selector = NIO::Selector.new
+      self.monitors = {}
+
+      self.actor = Actor.new do |msg, actor|
+        if msg == :do_select
+          handle_select_msg(actor)
+          next
+        end
+
+        receipt = msg
+        op      = receipt.registered_op
+
+        case
+        when param_pattern_match([:register_read, IO, Integer], op)
+          handle_read_msg(receipt, actor)
+        when param_pattern_match([:register_read, IO, nil], op)
+          handle_read_msg(receipt, actor)
+        when param_pattern_match([:register_write, IO, String], op)
+          handle_write_msg(receipt, actor)
+        end
+      end #end of actor
+
+    end
+
+    #end of initialize
+
+
+    def handle_select_msg(actor)
+      return if selector.empty?
+      selector.select(0.1) do |monitor|
+        if monitor.readiness == :r
+          monitor.value[0].call
+        elsif monitor.readiness == :w
+          monitor.value[1].call
+        else
+          monitor.value[0].call
+          monitor.value[1].call
+        end
+      end
+      return if selector.empty?
+      actor.send_msg :do_select
+      nil
+    end
+
+
+    def handle_read_msg(receipt, actor)
+      op                 = receipt.registered_op
+      io                 = op[1]
+      len                = op[2]
+      res                = StringIO.new
+      buf_size           = 1024 * 512
+      registered_monitor = monitors[io]
+      if registered_monitor && (registered_monitor.interests == :r || registered_monitor.interests == :rw)
+        actor.send_msg receipt
+        return
+      end
+
+      if registered_monitor
+        registered_monitor.add_interest(:r)
+        monitor = registered_monitor
+      else
+        monitor      = selector.register(io, :r)
+        monitors[io] = monitor
+      end
+
+      monitor.value    ||= []
+      monitor.value[0] = proc do
+        if len.nil?
+          notify_blk = proc do
+            monitors.delete(monitor.io)
+            monitor.close
+            receipt.res = res.string
+            receipt.notify
+          end
+          loop do
+            begin
+              buf = io.read_nonblock(buf_size, exception: false)
+            rescue Exception => ex
+              notify_blk.call
+              STDERR.puts ex
+              break
+            end
+            if buf == :wait_readable
+              break
+            elsif buf.nil?
+              notify_blk.call
+              break
+            end
+            res << buf
+          end
+        elsif len == 0
+          monitors.delete(monitor.io)
+          monitor.close
+          receipt.res = ""
+          receipt.notify
+          break
+        else
+          notify_blk = proc do
+            monitors.delete(monitor.io)
+            monitor.close
+            if res.string.length == 0
+              receipt.res = nil
+            else
+              receipt.res = res.string
+            end
+            receipt.notify
+          end
+
+          bytes_read_n = 0
+          loop do
+            need_read_bytes_n = len - bytes_read_n
+            if need_read_bytes_n <= 0
+              notify_blk.call
+              break
+            end
+            begin
+              buf = io.read_nonblock(need_read_bytes_n, exception: false)
+            rescue Exception => ex
+              notify_blk.call
+              STDERR.puts ex
+              break
+            end
+            if buf == :wait_readable
+              break
+            elsif buf.nil?
+              notify_blk.call
+              break
+            end
+            res << buf
+            bytes_read_n += buf.bytesize
+          end
+        end
+      end
+      actor.send_msg :do_select
+    end
+
+
+    def handle_write_msg(receipt, actor)
+      op  = receipt.registered_op
+      io  = op[1]
+      str = op[2].to_s
+
+      registered_monitor = monitors[io]
+      if registered_monitor && (registered_monitor.interests == :w || registered_monitor.interests == :rw)
+        actor.send_msg receipt
+        return
+      end
+
+      if registered_monitor
+        registered_monitor.add_interest(:w)
+        monitor = registered_monitor
+      else
+        monitor      = selector.register(io, :w)
+        monitors[io] = monitor
+      end
+
+      buf = NIO::ByteBuffer.new(str.bytesize)
+      buf << str
+      buf.flip
+      bytes_written_n  = 0
+      monitor.value    ||= []
+      monitor.value[1] = proc do
+        begin
+          if buf.remaining > 0
+            n               = buf.write_to(io)
+            bytes_written_n += n
+          else
+            monitors.delete(monitor.io)
+            monitor.close
+            receipt.res = str.bytesize
+            receipt.notify
+          end
+        rescue Exception => ex
+          monitors.delete(monitor.io)
+          monitor.close
+          receipt.res = bytes_written_n
+          receipt.notify
+          STDERR.puts ex
+        end
+      end
+      actor.send_msg :do_select
+    end
+
+
+    def param_pattern_match(pattern, params)
+      return false unless pattern.is_a?(Array) && params.is_a?(Array)
+      return false if pattern.size != params.size
+      match = true
+      pattern.zip(params) do |type, param|
+        unless type === param
+          match = false
+          break
+        end
+      end
+      match
+    end
+  end
+end

data/lib/rbgo/select_chan.rb

@@ -19,6 +19,7 @@ module Rbgo
 
       def register(io)
         register_mutex.synchronize do
+          ios.delete_if { |io| io.closed? }
           ios << io
         end
       end
@@ -53,7 +54,7 @@ module Rbgo
         self.have_enq_waiting_flag = false
         self.have_deq_waiting_flag = false
 
-        self.ios = []
+        self.ios            = []
         self.register_mutex = Mutex.new
       end
 
@@ -202,7 +203,7 @@ module Rbgo
         super(max)
         @mutex = Mutex.new
 
-        self.ios = []
+        self.ios            = []
         self.register_mutex = Mutex.new
       end
 

data/lib/rbgo/version.rb

@@ -1,3 +1,3 @@
 module Rbgo
-  VERSION = "0.2.0"
+  VERSION = "0.2.5"
 end

data/lib/rbgo.rb

@@ -3,5 +3,6 @@ require_relative 'rbgo/select_chan'
 require_relative 'rbgo/wait_group'
 require_relative 'rbgo/once'
 require_relative 'rbgo/actor'
+require_relative 'rbgo/io_machine'
 require_relative 'rbgo/network_service'
 require_relative 'rbgo/version'

data/rbgo.gemspec

@@ -20,6 +20,7 @@ Gem::Specification.new do |s|
                     In MRI write program to run concurrently even not parallelly is also important.
                   END
   s.add_dependency "system", "~> 0.1.3"
+  s.add_dependency "nio4r", "~> 2.4"
   s.files         = `git ls-files`.split("\n")
   s.license       = 'MIT'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbgo
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.5
 platform: ruby
 authors:
 - Wang Yin
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-08-07 00:00:00.000000000 Z
+date: 2019-08-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: system
@@ -24,6 +24,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.1.3
+- !ruby/object:Gem::Dependency
+  name: nio4r
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.4'
 description: "                    Write concurrent program with Ruby in Golang style.
   \ \n\n                    You can produce a light weight routine easily with a method
   'go' and communicate with each routine by channel.\n\n                    In MRI
@@ -44,6 +58,7 @@ files:
 - lib/rbgo.rb
 - lib/rbgo/actor.rb
 - lib/rbgo/corun.rb
+- lib/rbgo/io_machine.rb
 - lib/rbgo/network_service.rb
 - lib/rbgo/once.rb
 - lib/rbgo/select_chan.rb