thimble 0.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 8b7df5f35312f4bea62c895b077a13d6e3d080de
+  data.tar.gz: c82c709be630358c4c30aa637e2ceac80653dcda
+SHA512:
+  metadata.gz: 755797989277b7d74fbdb549ffa0f1fb9037c0394f1da765aa5ae1da1c9b23f7e0b54165457cacb9c0f4d07edd77e6f7614baac50e9815afb84c69e16e363d4d
+  data.tar.gz: 2ec821b444f98d3c8cc85b35a3b0e977c8928abc2a531ebd36effc406a4e20963d784bdd74fdca07c30a73c837ea4ea4f4f18f15aed983961f676e1a83dfce09

data/lib/Manager.rb

@@ -0,0 +1,94 @@
+module Thimble
+  class Manager
+    attr_reader :max_workers, :batch_size, :queue_size, :worker_type
+    def initialize(max_workers: 6,batch_size: 1000, queue_size: 1000, worker_type: :fork)
+      raise ArgumentError.new ("worker type must be either :fork or :thread") unless worker_type == :thread || worker_type == :fork
+      raise ArgumentError.new ("Your system does not respond to fork please use threads.") unless worker_type == :thread || Process.respond_to?(:fork)
+      raise ArgumentError.new ("max_workers must be greater than 0") if max_workers < 1
+      raise ArgumentError.new ("batch size must be greater than 0") if batch_size < 1
+      @worker_type = worker_type
+      @max_workers = max_workers
+      @batch_size = batch_size
+      @queue_size = queue_size
+      @mutex = Mutex.new
+      @current_workers = {}
+    end
+
+    def worker_available?
+      @current_workers.size < @max_workers
+    end
+
+    def working?
+      @current_workers.size > 0
+    end
+
+    def sub_worker(worker, id)
+      raise "Worker must contain a pid!" if worker.pid.nil?
+      new_worker = OpenStruct.new
+      new_worker.worker = worker
+      new_worker.id = id
+      @mutex.synchronize do
+        @current_workers[worker.pid] = new_worker
+      end
+    end
+
+    def rem_worker(worker)
+      @mutex.synchronize do
+        @current_workers.delete(worker.pid)
+      end
+    end
+
+    def current_workers(id)
+      @mutex.synchronize do
+        @current_workers.select { |k,v| v.id == id }
+      end
+    end
+
+    def get_worker (batch)
+      @mutex.synchronize do
+        if @worker_type == :fork
+          get_fork_worker(batch, &Proc.new)    
+        else
+          get_thread_worker(batch, &Proc.new)
+        end
+      end
+    end
+
+    def get_fork_worker(batch)
+      rd, wr = IO.pipe
+      tup = OpenStruct.new
+      pid = fork do
+        Signal.trap("HUP") {exit}
+        rd.close 
+        t = Marshal.dump(batch.item.map do |item|
+          yield (item.item)
+        end)
+        wr.write(t)
+        wr.close
+      end
+      wr.close
+      tup.pid = pid
+      tup.reader = rd
+      tup
+    end
+
+    def get_thread_worker(batch)
+      tup = OpenStruct.new
+      tup.pid = Thread.new do
+        tup.result = batch.item.map do |item|
+          yield item.item
+        end
+        tup.done = true
+      end
+      tup
+    end
+
+    def self.deterministic
+      self.new(max_workers: 1, batch_size: 1, queue_size: 1)
+    end
+
+    def self.small
+      self.new(max_workers: 1, batch_size: 3, queue_size: 3)
+    end
+  end
+end

data/lib/QueueItem.rb

@@ -0,0 +1,14 @@
+require "digest"
+class QueueItem
+  attr_reader :id, :item
+  def initialize(item, name= "Item")
+    @id = Digest::SHA256.digest(rand(10**100).to_s + Time.now.to_i.to_s)
+    @item = item
+    @name = name
+  end
+
+  def to_s
+    "#{@name}: #{@item} \nID: #{@id}"
+  end
+
+end

data/lib/Thimble.rb

@@ -0,0 +1,89 @@
+
+require_relative 'Manager'
+require_relative 'ThimbleQueue'
+require_relative 'QueueItem'
+require 'io/wait'
+require 'ostruct'
+
+module Thimble
+
+  class Thimble < ThimbleQueue
+    def initialize(array, manager = Manager.new)
+      raise ArgumentError.new ("You need to pass a manager to Thimble!") unless manager.class == Manager
+      raise ArgumentError.new ("There needs to be an iterable object passed to Thimble to start.") unless array.respond_to? :each
+      @manager = manager
+      @result = ThimbleQueue.new(array.size, "Result")
+      @running = true
+      super(array.size, "Main")
+      array.each {|item| push(item)}
+      close()
+    end
+
+    # Takes a block
+    def par_map
+      @running = true
+      while @running
+        manage_workers &Proc.new
+      end
+      @result.close()
+      @result
+    end
+
+    # Will perform anything handed to this asynchronously. 
+    # Requires a block
+    def self.a_sync
+      Thread.new do |e|
+        yield e
+      end
+    end
+
+    private
+    def get_batch
+      batch = []
+      while batch.size < @manager.batch_size
+        item = take
+        if item.nil?
+          return nil if batch.size == 0
+          return QueueItem.new(batch, "Batch")
+        else
+          batch << item
+        end
+      end
+      QueueItem.new(batch, "Batch")
+    end
+
+    def manage_workers
+      while (@manager.worker_available? && batch = get_batch)
+        @manager.sub_worker( @manager.get_worker(batch, &Proc.new), @id)
+      end
+      @manager.current_workers(@id).each do |pid, pair|
+        get_result(pair.worker)
+      end
+      @running = false if !@manager.working? && !batch
+    end
+
+    def get_result(tuple)
+      if @manager.worker_type == :fork
+        if tuple.reader.ready?
+          piped_result = tuple.reader.read
+          push_result(Marshal.load(piped_result))
+          Process.kill("HUP", tuple.pid)
+          @manager.rem_worker(tuple)
+        end
+      else
+        if tuple.done == true
+          push_result(tuple.result)
+          @manager.rem_worker(tuple)
+        end
+      end
+    end
+
+    def push_result(result)
+      if result.respond_to? :each
+        result.each {|r| @result.push(r)}
+      else
+        @result.push(result)
+      end
+    end
+  end
+end

data/lib/ThimbleQueue.rb

@@ -0,0 +1,91 @@
+require 'thread'
+require_relative 'QueueItem'
+
+module Thimble
+  class ThimbleQueue
+    attr_reader :size
+    def initialize(size, name)
+      raise ArgumentError.new("make sure there is a size for the queue greater than 1! size received #{size}") unless size >= 1
+      @id = Digest::SHA256.digest(rand(10**100).to_s + Time.now.to_i.to_s)
+      @name = name
+      @size = size
+      @mutex = Mutex.new
+      @queue = []
+      @closed = false
+      @close_now = false
+      @empty = ConditionVariable.new
+      @full = ConditionVariable.new
+    end
+   
+    def take
+      @mutex.synchronize  do
+        while !@close_now
+          a = @queue.shift
+          if !a.nil?
+            @full.broadcast
+            return a
+          else 
+            return nil if @closed
+            @empty.wait(@mutex)
+          end
+        end
+      end
+    end
+
+    # This will push whatever it is handed to the queue
+    def push(x)
+      @mutex.synchronize do
+        while !offer(x)
+          @full.wait(@mutex)
+        end
+        @empty.broadcast
+      end
+    end
+
+    # This will flatten any nested arrays out and feed them one at
+    # a time to the queue.
+    def push_flat(x)
+      if x.respond_to? :each
+        x.each {|item| push(item)}
+      else
+        @mutex.synchronize do
+          while !offer(x)
+            @full.wait(@mutex)
+          end
+          @empty.broadcast
+        end
+      end
+    end
+
+    def close(now = false)
+      @mutex.synchronize do
+        @closed = true
+        @close_now = true if now
+        @full.broadcast
+        @empty.broadcast
+      end
+    end
+
+    def to_a
+      a = []
+      while item = take
+        a << item.item
+      end
+      a
+    end
+
+    def closed?
+      @close
+    end
+
+    private
+    def offer(x)
+      if @queue.size < @size
+        @queue << QueueItem.new(x)
+        true
+      else
+        false
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,47 @@
+--- !ruby/object:Gem::Specification
+name: thimble
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+platform: ruby
+authors:
+- Andrew Kovanda
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-03-15 00:00:00.000000000 Z
+dependencies: []
+description: Pass a block and get some results
+email: andrew.kovanda@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/Manager.rb
+- lib/QueueItem.rb
+- lib/Thimble.rb
+- lib/ThimbleQueue.rb
+homepage: https://github.com/akovanda/thimble
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: Concurrency and Parallelism gem that uses blocks to move data
+test_files: []