peplum 0.3.0 → 0.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 39d9ce50f924ae2ba75b9722feab0a00b40d7c18bca14fdfcfe823de5672ef4b
-  data.tar.gz: 995a13165a99b8dde616ce02fc878876e9ede71b70016f3f08928312cd969566
+  metadata.gz: 427efea5fb0daf113212ee723e2e850ccb30723af7e7d5c6ea3576fe9e51979e
+  data.tar.gz: f6563c4ac916f06e66bba1d7ba6d69fed7818b601d9148d5d111dc7c6756d1d3
 SHA512:
-  metadata.gz: 6f5eb468b00c381da6731f52295cce1636875ad4ba19df0494f2c8957e420cd4161599f65faccec22dd3e1238e1dacc2e18242ba4ecf216e5e3cb5686fe98c8a
-  data.tar.gz: bf11c45c00a5a2d1f028ac028060588448a2dbc342c133bf0ab7526c8fe09585f17eb2624897231b4bc06b40199b6b6fa24c937ec43cf30e2b12a9eee933da1f
+  metadata.gz: ecb729393728b371af870f5615a00e6d815e773932aefedce3aefa4e07e20f82eb2dbd0bee40e99add29f2a1b915654115c74df3cfb11427f87507cc4d85e584
+  data.tar.gz: 5d8ea07c8ccbac15dc63116d377b4259662634f75b105b77223b6805ded8e5176e0bbc292d67cc1a9d7768929e7c3045d16925c7b1e239c884dcb10bdead0664

data/lib/peplum/application/payload.rb

@@ -0,0 +1,67 @@
+require 'peplum/core_ext/array'
+
+module Peplum
+class Application
+module Payload
+
+  class Error < Error
+    class ImplementationMissing < Error
+    end
+  end
+
+  # Run payload against `objects` based on given `options`
+  #
+  # @param  [Array] objects Objects that this worker should process.
+  # @param  [Hash, NilClass]  options Worker options.
+  # @abstract
+  def run( objects, options )
+    fail Error::ImplementationMissing, 'Missing implementation.'
+  end
+
+  # Distribute `objects` into `groups_of` amount of groups, one for each worker.
+  #
+  # @param  [Array] objects All objects that need to be processed.
+  # @param  [Integer] groups_of  Amount of object groups that should be generated.
+  #
+  # @return [Array<Array<Object>>]  `objects` split in `chunks` amount of groups.
+  def split( objects, groups_of )
+    objects.chunk groups_of
+  end
+
+  # Merge result `data` for reporting.
+  #
+  # By default provides a generic implementation that merges the values of `Hash`es and `Array`s.
+  # If `String`s or `Numeric`s are contained, the Array is returned as is.
+  #
+  # @param  [Array] data  Report data from workers.
+  # @return [Object]  Merged results.
+  #
+  # @raise  [Error::ImplementationMissing]  When the data cannot be handled.
+  def merge( data )
+    case data.first
+    when Hash
+        f = data.pop
+        data.each do |d|
+
+          if !f.is_a? Hash
+            fail Error::ImplementationMissing, 'Missing implementation: Item not a Hash!'
+          end
+
+          f.merge! d
+        end
+        f
+
+    when Array
+      data.flatten
+
+    when String, Numeric
+      data
+
+    else
+      fail Error::ImplementationMissing, 'Missing implementation.'
+    end
+  end
+
+end
+end
+end

data/lib/peplum/application/worker.rb

@@ -0,0 +1,19 @@
+module Peplum
+class Application
+
+module Worker
+
+  # @return   [Cuboid::RPC::Server::Instance::Peers]
+  attr_reader :peers
+
+  # @return   [Cuboid::RPC::Client::Instance, NilClass]
+  attr_reader :scheduler
+
+  def peers
+    @peers ||= Cuboid::RPC::Server::Instance::Peers.new
+  end
+
+end
+
+end
+end

data/lib/peplum/application.rb

@@ -1,88 +1,93 @@
 require 'cuboid'
 require 'json'
 require 'peplum'
-require 'peplum/core_ext/array'
 
 module Peplum
   class Application < Cuboid::Application
+    require 'peplum/application/worker'
+    require 'peplum/application/payload'
+
     require 'peplum/application/services/shared_hash'
     require 'peplum/application/services/scheduler'
 
-    class Error < Peplum::Error; end
+    class Error < Peplum::Error
+      class PayloadMissing < Error
+      end
+    end
 
+    # Hijack Cuboid to set ourselves up.
     class <<self
       def inherited( application )
         super
 
+        # Don't trust Cuboid's auto-detection for this, make sure the inheriting class is the Cuboid application.
         Cuboid::Application.application = application
 
         application.validate_options_with :validate_options
-        application.serialize_with JSON
+        application.serialize_with MessagePack
 
         application.instance_service_for :scheduler,   Services::Scheduler
+
+        # Shared, in-memory key-value store for the workers, the scheduler will not be participate.
         application.instance_service_for :shared_hash, Services::SharedHash
       end
     end
 
-    attr_reader :peers
-    attr_reader :master
-
-    def initialize(*)
-      super
-
-      @peers = Cuboid::RPC::Server::Instance::Peers.new
-    end
-
     def run
-      Raktr.global.on_error do |_, e|
-        $stderr.puts e
-      end
-
       options = @options.dup
+
+      # System options.
       peplum_options = options.delete( 'peplum' )
+
+      # Payload options.
       payload_options = options.delete( 'payload' )
 
-      # We have a master so we're not the scheduler, run the payload.
+      # We have a master so we're a worker, run the payload.
       if peplum_options['master']
-        execute( peplum_options, payload_options )
+        work( peplum_options, payload_options )
 
-      # We're the scheduler Instance, get to grouping and spawning.
+      # We're the scheduler Instance, get to grouping objects and spawning workers.
       else
         schedule( peplum_options, payload_options )
       end
     end
 
-    def worker?
-      # Has a master?
-      !!@master
-    end
-
-    # Implements:
-    #   * `.run` -- Worker; executes its payload against `objects`.
-    #   * `.split` -- Splits given `objects` into groups for each worker.
-    #   * `.merge` -- Merges results from multiple workers.
+    # @return [#run, #split, #merge]
     #
-    # That's all we need to turn any application into a super version of itself.
+    #   * `#run` -- Worker; executes its payload against `objects`.
+    #   * `#split` -- Scheduler; splits given `objects` into groups for each worker.
+    #   * `#merge` -- Scheduler; merges results from multiple workers.
+    #
+    #   That's all we need to turn any application into a super version of itself.
     #
     # @abstract
-    def payload
-      fail Error, 'Missing #payload!'
+   def payload
+      fail Error::PayloadMissing, 'Missing #payload implementation!'
     end
 
+    # Overload {Cuboid#report} and delegate to the {Payload.merge} prior passing it on to {Cuboid}.
+    # @private
     def report( data )
       super payload.merge( data )
     end
 
     private
 
-    def execute( peplum_options, payload_options )
+    def work( peplum_options, payload_options )
+      # We're now a worker class!
+      self.class.include Worker
+
       master_info = peplum_options.delete( 'master' )
       @master = Processes::Instances.connect( master_info['url'], master_info['token'] )
 
+      # Configure us to know the rest of our worker peers.
+      # Required for the SharedHash service.
       self.peers.set( peplum_options.delete( 'peers' ) || {} )
 
+      # Deliver the payload.
       report_data = payload.run( peplum_options['objects'], payload_options )
 
+      # Signal that we're done by passing our report to the scheduler.
       @master.scheduler.report report_data, Cuboid::Options.rpc.url
     end
 
@@ -151,9 +156,20 @@ module Peplum
         fail Error, 'Options: Missing :max_workers'
       end
 
+      validate_payload
+
       @options = options
       true
     end
 
+    def validate_payload
+      p = self.payload
+
+      %w(run merge split).each do |m|
+        next if p.respond_to? m
+        fail Payload::Error::ImplementationMissing, "#{payload} missing implementation for: ##{m}"
+      end
+    end
+
   end
 end

data/lib/peplum/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Peplum
-  VERSION = "0.3.0"
+  VERSION = "0.3.2"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: peplum
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.2
 platform: ruby
 authors:
 - Tasos Laskos
@@ -34,8 +34,10 @@ files:
 - bin/.gitkeep
 - lib/peplum.rb
 - lib/peplum/application.rb
+- lib/peplum/application/payload.rb
 - lib/peplum/application/services/scheduler.rb
 - lib/peplum/application/services/shared_hash.rb
+- lib/peplum/application/worker.rb
 - lib/peplum/core_ext/array.rb
 - lib/peplum/version.rb
 - peplum.gemspec