parallax 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1b5ac6d394abaaa2b0f2b6634f0cb46a1e77584ae41d0a603dd46c96bd35b3c3
-  data.tar.gz: cc568d1e88bed4cb5f7d2f7708fe6cdf1203e4623c0541c4a0326c6d8c5771c5
+  metadata.gz: 199b9a1f7d00f3ab396af10da44fbec350f059444e133a68128c33f6103efd44
+  data.tar.gz: f23cd123597d3070c5e23ab98a5db1288c5d85bf0f4170618ff304a1f17e05e9
 SHA512:
-  metadata.gz: a72b7b5fb0d24365f359ce534b0c5feba903a8e534a994c509a0f2ff13fac449023990903e6f5dcd8ac85dc6bb2d358bddfd7f40c5f3f0058b2b16d79288bdf1
-  data.tar.gz: 126f94e0441ce1bbc4acbbc86488b4f0df19bfa537c2afaca0ec0b6dd1d51ae7eb3b06d3cfd8586a3b40364ff7c54370abba4a89cdf6c9220d39575c7ddc7cfd
+  metadata.gz: a6a57b1760de4e0e857f46e2da3d0c18040846358c5f398270ec33328e661e63bff3d68d6a6d91baf09b38eb209a73691a14b3294b8aa58870847dac66084018
+  data.tar.gz: 6b8a58e098e65c5070b804cc6081dce58f616aaf06bd6b25adadb2cc2db8e5bf1606fa75732ef4fe0f73c9f98ba0f97c87ee6dc1fd8ec7c4d469bc3549edb7bc

data/Gemfile.lock

@@ -1,22 +1,108 @@
 PATH
   remote: .
   specs:
-    parallax (0.1.1)
+    parallax (0.3.0)
       activesupport
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (5.2.1.1)
+    actioncable (5.2.2)
+      actionpack (= 5.2.2)
+      nio4r (~> 2.0)
+      websocket-driver (>= 0.6.1)
+    actionmailer (5.2.2)
+      actionpack (= 5.2.2)
+      actionview (= 5.2.2)
+      activejob (= 5.2.2)
+      mail (~> 2.5, >= 2.5.4)
+      rails-dom-testing (~> 2.0)
+    actionpack (5.2.2)
+      actionview (= 5.2.2)
+      activesupport (= 5.2.2)
+      rack (~> 2.0)
+      rack-test (>= 0.6.3)
+      rails-dom-testing (~> 2.0)
+      rails-html-sanitizer (~> 1.0, >= 1.0.2)
+    actionview (5.2.2)
+      activesupport (= 5.2.2)
+      builder (~> 3.1)
+      erubi (~> 1.4)
+      rails-dom-testing (~> 2.0)
+      rails-html-sanitizer (~> 1.0, >= 1.0.3)
+    activejob (5.2.2)
+      activesupport (= 5.2.2)
+      globalid (>= 0.3.6)
+    activemodel (5.2.2)
+      activesupport (= 5.2.2)
+    activerecord (5.2.2)
+      activemodel (= 5.2.2)
+      activesupport (= 5.2.2)
+      arel (>= 9.0)
+    activestorage (5.2.2)
+      actionpack (= 5.2.2)
+      activerecord (= 5.2.2)
+      marcel (~> 0.3.1)
+    activesupport (5.2.2)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
-    concurrent-ruby (1.1.3)
+    arel (9.0.0)
+    builder (3.2.3)
+    concurrent-ruby (1.1.4)
+    crass (1.0.4)
     diff-lcs (1.3)
-    i18n (1.1.1)
+    erubi (1.8.0)
+    globalid (0.4.1)
+      activesupport (>= 4.2.0)
+    i18n (1.2.0)
       concurrent-ruby (~> 1.0)
+    loofah (2.2.3)
+      crass (~> 1.0.2)
+      nokogiri (>= 1.5.9)
+    mail (2.7.1)
+      mini_mime (>= 0.1.1)
+    marcel (0.3.3)
+      mimemagic (~> 0.3.2)
+    method_source (0.9.2)
+    mimemagic (0.3.3)
+    mini_mime (1.0.1)
+    mini_portile2 (2.4.0)
     minitest (5.11.3)
+    nio4r (2.3.1)
+    nokogiri (1.9.1)
+      mini_portile2 (~> 2.4.0)
+    rack (2.0.6)
+    rack-test (1.1.0)
+      rack (>= 1.0, < 3)
+    rails (5.2.2)
+      actioncable (= 5.2.2)
+      actionmailer (= 5.2.2)
+      actionpack (= 5.2.2)
+      actionview (= 5.2.2)
+      activejob (= 5.2.2)
+      activemodel (= 5.2.2)
+      activerecord (= 5.2.2)
+      activestorage (= 5.2.2)
+      activesupport (= 5.2.2)
+      bundler (>= 1.3.0)
+      railties (= 5.2.2)
+      sprockets-rails (>= 2.0.0)
+    rails-dev-tools (1.2.3)
+      rails (~> 5.2.2)
+      rainbow (~> 3.0)
+    rails-dom-testing (2.0.3)
+      activesupport (>= 4.2.0)
+      nokogiri (>= 1.6)
+    rails-html-sanitizer (1.0.4)
+      loofah (~> 2.2, >= 2.2.2)
+    railties (5.2.2)
+      actionpack (= 5.2.2)
+      activesupport (= 5.2.2)
+      method_source
+      rake (>= 0.8.7)
+      thor (>= 0.19.0, < 2.0)
     rainbow (3.0.0)
     rake (10.5.0)
     rspec (3.8.0)
@@ -32,19 +118,31 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.8.0)
     rspec-support (3.8.0)
+    sprockets (3.7.2)
+      concurrent-ruby (~> 1.0)
+      rack (> 1, < 3)
+    sprockets-rails (3.2.1)
+      actionpack (>= 4.0)
+      activesupport (>= 4.0)
+      sprockets (>= 3.0.0)
+    thor (0.20.3)
     thread_safe (0.3.6)
     tzinfo (1.2.5)
       thread_safe (~> 0.1)
+    websocket-driver (0.7.0)
+      websocket-extensions (>= 0.1.0)
+    websocket-extensions (0.1.3)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  bundler (~> 1.17)
+  bundler (>= 1.17)
   parallax!
+  rails-dev-tools
   rainbow
-  rake (~> 10.0)
-  rspec (~> 3.0)
+  rake (>= 10.0)
+  rspec (>= 3.0)
 
 BUNDLED WITH
-   1.17.0
+   1.17.1

data/README.md

@@ -40,9 +40,13 @@ end
 #   [2] in
 ```
 
-If you need inter-process communication, this can be done by calling `worker.send` and passing a list of arguments. The args are serialized and passed via IO pipe to a `collector` object, which is by default an instance of `Parallax::Collector` class. The collector then parses the args and treats them like a method call where the first arg is the name of the method. In the example above, the collectors calls the `log` method which prints the second arg.
+If you need inter-process communication, this can be done by calling `worker.send` and passing a list of arguments. The args are serialized and passed via IO pipe to a `collector` object, which is by default an instance of `Parallax::Collector` class. The collector then parses the args and treats them like a method call where the first arg is the name of the method.
 
-The collector object is returned by the `Parallax.execute` method, so if you need to store each worker processed data you can use the `worker.store` method or implement it in your own custom collector. For example:
+There are a number of predefined methods, build on top of `worker.send` that you can call to do a number of tasks:
+* `log`: Used in the example above, the collectors calls the `log` method which prints the message to the stdout.
+* `store`: Saves the argument object into a variable called `workers_data` in the collector.
+
+The collector object is returned by the `Parallax.execute` method, so if you need to access the stored data called with the `worker.store` method you can do:
 
 ```ruby
 numbers = (0..100).to_a
@@ -65,12 +69,47 @@ puts collector.workers_data.inspect
 
 Other options you can pass to execute are:
 * `processes`: the number of processes in which parallelize the execution. Defaults to `Etc.nprocessors` (which is equal to the number of cores of the current running machine).
+* `collector`: a custom collector object that you can implement yourself.
+
+To use a custom collector, you need to `include Parallax::Collectable` in your custom collector, and initialize your collector by calling the included `initialize_collector` method.
+Example of a custom collector:
+
+```ruby
+# custom_collector.rb
+class CustomCollector
+  include Parallax::Collectable
+
+  def initialize(worker_count, *args)
+    # Do your own initialization with *args, and then
+    initialize_collector(workers_count)
+  end
 
-Methods available for a `worker` object, which are collected in the `collector` object:
-* `log(message)`: prints a message in stdout.
-* `store(object)`: stores an object with timestamp and worker index.
-* `rescue(error)`: rescues an error from a worker and raises the same error in the collector.
-* `close`: closes worker communication with the collector.
+  def store(worker_index, object)
+    workers_data.push [ "worker #{worker_index} stored: #{object.inspect}" ]
+  end
+end
+```
+
+```ruby
+workers_count = 4
+numbers = (0..100).to_a
+
+custom_collector = CustomCollector.new(workers_count)
+Parallax.execute numbers, collector: custom_collector, do |worker, numbers_chunk|
+  numbers_chunk.each do |number|
+    worker.store number * 2
+  end
+end
+
+puts custom_collector.workers_data.inspect
+
+# Example output with 4 cores and custom collector:
+#   [ ["worker 0 stored 0"],
+#     ["worker 3 stored 152"],
+#     ["worker 1 stored 52"],
+#     ["worker 2 stored 102"],
+#     ...
+```
 
 ## Development
 

data/dev.yml

@@ -0,0 +1,4 @@
+dev:
+  project_type: single
+  project_name: parallax
+  project_folder: /Users/fballardin/progetti/francy/parallax

data/lib/parallax/collectable.rb

@@ -0,0 +1,115 @@
+module Parallax
+  module Collectable
+
+    def self.included(base)
+      
+      # @return [Integer] the number of workers running in parallel.
+      attr_accessor :workers_count
+      # @return [Array] the result of all workers' store method calls.
+      attr_accessor :workers_data
+      # @return [Integer] the number of completed worker processes.
+      attr_accessor :closed_workers_count
+      # @return [IO::Pipe] the receiving stream of data.
+      attr_accessor :receiving_stream
+      # @return [IO::Pipe] the sending stream of data.
+      attr_accessor :sending_stream
+
+      ##
+      # Inizializes the collectable object with its needed parameters.
+      #
+      # @param [Integer] workers_count the number of workers running in parallel.
+      #
+      # @return [nil]
+      def initialize_collector(workers_count)
+        @workers_count = workers_count
+        @closed_workers_count = 0
+        @receiving_stream, @sending_stream = IO.pipe
+        @workers_data = []
+      end
+
+      ##
+      # Reads data from the receiving stream.
+      #
+      # @return [String] the received data.
+      def receive
+        self.collect @receiving_stream.gets.chomp
+      end
+
+      ##
+      # Interprets a received message from the worker and
+      # executes a method in the collector.
+      #
+      # @param [String] message the message as a string.
+      #
+      # @return [Object] the execution of the interpreted method.
+      def collect(message)
+        worker_index, method, *arguments = eval(message)
+        self.send method, worker_index, *arguments
+      end
+
+      ##
+      # Prints a message from the worker.
+      #
+      # @param [Integer] worker_index the worker number.
+      # @param [String] message the worker message.
+      #
+      # @return [nil]
+      def log(worker_index, message)
+        puts message
+      end
+
+      ##
+      # Saves an object in the workers' data.
+      #
+      # @param [Integer] worker_index the worker number.
+      # @param [Object] object the object.
+      #
+      # @return [nil]
+      def store(worker_index, object)
+        workers_data.push [ Time.now, worker_index, object ]
+      end
+
+      ##
+      # In a worker raises an error, this is rescued and reraised
+      # in the collector.
+      #
+      # @param [Integer] worker_index the worker number.
+      # @param [Class] error_class the class of the error.
+      # @param [String] error_message the message of the error.
+      #
+      # @return [nil]
+      def rescue(worker_index, error_class, error_message)
+        raise error_class, "Worker #{worker_index} Error: #{error_message}"
+      end
+
+      ##
+      # Closes a worker.
+      #
+      # @param [Integer] worker_index the worker number.
+      #
+      # @return [nil]
+      def close_worker(worker_index)
+        @closed_workers_count += 1
+      end
+
+      ##
+      # Closes the collector and its data streams.
+      #
+      # @return [nil]
+      def close
+        @receiving_stream.close
+        @sending_stream.close
+      end
+
+      ##
+      # Checks if all workers have terminated.
+      #
+      # return [Boolean] if all workers have terminated.
+      def all_workers_terminated?
+        @closed_workers_count >= @workers_count
+      end
+    
+    end
+    
+  end
+end

data/lib/parallax/collector.rb

@@ -1,51 +1,10 @@
 module Parallax
   class Collector
     
-    attr_accessor :workers_count
-    attr_accessor :workers_data
-    attr_accessor :closed_workers_count
-    attr_accessor :receiving_stream
-    attr_accessor :sending_stream
+    include Parallax::Collectable
 
     def initialize(workers_count)
-      @workers_count = workers_count
-      @closed_workers_count = 0
-      @receiving_stream, @sending_stream = IO.pipe
-      @workers_data = []
-    end
-
-    def receive
-      self.collect @receiving_stream.gets.chomp
-    end
-
-    def collect(message)
-      worker_index, method, *arguments = eval(message)
-      self.send method, worker_index, *arguments
-    end
-
-    def log(worker_index, message)
-      puts message
-    end
-
-    def store(worker_index, object)
-      workers_data.push [ Time.now, worker_index, object ]
-    end
-
-    def rescue(worker_index, error_class, error_message)
-      raise error_class, "Worker #{worker_index} Error: #{error_message}"
-    end
-
-    def close_worker(worker_index)
-      @closed_workers_count += 1
-    end
-
-    def all_workers_terminated?
-      @closed_workers_count >= @workers_count
-    end
-
-    def close
-      @receiving_stream.close
-      @sending_stream.close
+      initialize_collector(workers_count)
     end
     
   end

data/lib/parallax/version.rb

@@ -1,3 +1,3 @@
 module Parallax
-  VERSION = "0.2.0"
+  VERSION = '0.3.0'
 end

data/lib/parallax/worker.rb

@@ -1,36 +1,80 @@
 module Parallax
   class Worker
 
-    attr_accessor :receiver
+    # @return [Object] the collector object.
+    attr_accessor :collector
+    # @return [Integer] the index of this worker.
     attr_accessor :index
 
-    def initialize(receiver, index)
-      @receiver = receiver
+    ##
+    # Creates a new worker referred to the specified collector,
+    # and with the given index.
+    #
+    # @param [Collector] the collector.
+    # @param [Integer] the index.
+    #
+    # @return [Worker] the worker.
+    def initialize(collector, index)
+      @collector = collector
       @index = index
     end
 
+    ##
+    # Packs the message before sending it to the sending stream.
+    #
+    # @params [Array] args the message.
+    #
+    # @return [String] a string representation of the packed message.
     def pack(*args)
       [ self.index, *args ].inspect
     end
 
+    ##
+    # Sends the message to the sending stream.
+    #
+    # @params [Array] args the message.
+    #
+    # @return [nil]
     def send(*args)
-      @receiver.sending_stream.puts pack(*args)
+      @collector.sending_stream.puts pack(*args)
     end
 
+    ##
+    # Logs the message to the collector.
+    #
+    # @params [String] message the message.
+    #
+    # @return [nil]
     def log(message)
-      @receiver.sending_stream.puts pack(:log, message)
+      @collector.sending_stream.puts pack(:log, message)
     end
 
+    ##
+    # Stores the object in the collector.
+    #
+    # @param [Object] the object.
+    #
+    # @return [nil]
     def store(object)
-      @receiver.sending_stream.puts pack(:store, object)
+      @collector.sending_stream.puts pack(:store, object)
     end
 
+    ##
+    # Rescues an error from the worker and sends it to the collector.
+    #
+    # @param [Exception] error the error.
+    #
+    # @return [nil]
     def rescue(error)
-      @receiver.sending_stream.puts pack(:rescue, error.class, error.message)
+      @collector.sending_stream.puts pack(:rescue, error.class, error.message)
     end
 
+    ##
+    # Closes the worker and alerts the collector.
+    #
+    # @return [nil]
     def close
-      @receiver.sending_stream.puts pack(:close_worker)
+      @collector.sending_stream.puts pack(:close_worker)
     end
 
   end

data/lib/parallax.rb

@@ -1,12 +1,21 @@
 require 'active_support/core_ext/array'
 
 require 'parallax/version'
+require 'parallax/collectable'
 require 'parallax/collector'
 require 'parallax/worker'
 
 module Parallax
 
   class << self
+
+    ##
+    # Get the default number of workers.
+    #
+    # @return [Integer] the workers count.
+    def workers_count
+      Etc.nprocessors
+    end
   
     ##
     # Divides the given elements in groups of N and executes
@@ -17,9 +26,13 @@ module Parallax
     #
     # @return [Collector] all processes output collector.
     def execute(elements, options = {}, &block)
-      processes = options[:processes] || Etc.nprocessors
+      processes = options[:processes] || Parallax.workers_count
 
-      collector = Parallax::Collector.new(processes)
+      if options[:collector].present?
+        collector = options[:collector]
+      else
+        collector = Parallax::Collector.new(processes)
+      end
       elements_chunks = elements.in_groups(processes, false)
       processes.times do |worker_index|
         Process.fork do

data/parallax.gemspec

@@ -1,31 +1,32 @@
 
-lib = File.expand_path("../lib", __FILE__)
+lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require "parallax/version"
+require 'parallax/version'
 
 Gem::Specification.new do |spec|
-  spec.name          = "parallax"
+  spec.name          = 'parallax'
   spec.version       = Parallax::VERSION
-  spec.authors       = ["Francesco Ballardin"]
-  spec.email         = ["francesco.ballardin@develonproject.com"]
+  spec.authors       = ['Francesco Ballardin']
+  spec.email         = ['francesco.ballardin@develonproject.com']
 
   spec.summary       = %q{Enhances Ruby inter-process communication, to boost your parallel code execution.}
   spec.description   = %q{Enhances Ruby inter-process communication, to boost your parallel code execution.}
-  spec.homepage      = "https://github.com/Pluvie/ruby-parallax"
+  spec.homepage      = 'https://github.com/Pluvie/ruby-parallax'
 
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
   spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
     `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
   end
-  spec.bindir        = "exe"
+  spec.bindir        = 'exe'
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
-  spec.require_paths = ["lib"]
+  spec.require_paths = ['lib']
   
-  spec.add_dependency "activesupport"
+  spec.add_dependency 'activesupport'
 
-  spec.add_development_dependency "bundler", "~> 1.17"
-  spec.add_development_dependency "rake", "~> 10.0"
-  spec.add_development_dependency "rspec", "~> 3.0"
-  spec.add_development_dependency "rainbow"
+  spec.add_development_dependency 'bundler', '>= 1.17'
+  spec.add_development_dependency 'rake', '>= 10.0'
+  spec.add_development_dependency 'rspec', '>= 3.0'
+  spec.add_development_dependency 'rainbow'
+  spec.add_development_dependency 'rails-dev-tools'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: parallax
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Francesco Ballardin
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-12-04 00:00:00.000000000 Z
+date: 2018-12-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -28,42 +28,42 @@ dependencies:
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.17'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.17'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '10.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '10.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '3.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '3.0'
 - !ruby/object:Gem::Dependency
@@ -80,6 +80,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rails-dev-tools
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Enhances Ruby inter-process communication, to boost your parallel code
   execution.
 email:
@@ -97,7 +111,9 @@ files:
 - Rakefile
 - bin/console
 - bin/setup
+- dev.yml
 - lib/parallax.rb
+- lib/parallax/collectable.rb
 - lib/parallax/collector.rb
 - lib/parallax/version.rb
 - lib/parallax/worker.rb