peplum 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 6624c27866af54f4e3ab25e39a1fedb7ce83039b56c3bf6f43e5be42b19b0025
+  data.tar.gz: 8895326c25f1745c2074495e4a7666b2055a6e9efd765f90f32135f856c4e4d6
+SHA512:
+  metadata.gz: aa10f79c3ce4553d8237da0b0efec0578b2419e3794cefb75f036173acb5c6dfb76f34e743c150c60f919351699e4978944faa01535ed0b89bc7864232abf14a
+  data.tar.gz: bc66f042bb08248ec3c13ba01b4aea90c80e56ec50016b74e7f38e77b965a40c4745983ed470699ac897ce89df742295a0cbe675006e247719913c8fc18086ed

data/lib/peplum/application/peers.rb

@@ -0,0 +1,32 @@
+module Peplum
+class Application
+
+class Peers
+
+  def initialize
+    @peers = {}
+  end
+
+  def set( peer_info )
+    peer_info.each do |url, token|
+      next if url == self.self_url
+      @peers[url] = Peplum::Application.connect( url: url, token: token )
+    end
+
+    nil
+  end
+
+  def each( &block )
+    @peers.each do |_, client|
+      block.call client
+    end
+  end
+
+  def self_url
+    Cuboid::Options.rpc.url
+  end
+
+end
+
+end
+end

data/lib/peplum/application/services/scheduler.rb

@@ -0,0 +1,62 @@
+module Peplum
+class Application
+module Services
+
+class Scheduler
+
+  # Keep those out of RPC.
+  class <<self
+    def get_worker
+      worker_info = agent.spawn
+      return if !worker_info
+
+      worker = Peplum::Application.connect( worker_info )
+      self.workers[worker.url] = worker
+      worker
+    end
+
+    def done_signal
+      @done_signal ||= Queue.new
+    end
+
+    def wait
+      self.done_signal.pop
+    end
+
+    def done
+      self.done_signal << nil
+    end
+
+    def agent
+      @agent ||= Processes::Agents.connect( Cuboid::Options.agent.url )
+    end
+
+    def workers
+      @workers ||= {}
+    end
+  end
+
+  def report( data, url )
+    return if !(worker = self.class.workers.delete( url ))
+
+    report_data << data
+
+    worker.shutdown {}
+    return unless self.class.workers.empty?
+
+    Cuboid::Application.application.report report_data
+
+    self.class.done
+  end
+
+  private
+
+  def report_data
+    @report_data ||= []
+  end
+
+end
+
+end
+end
+end

data/lib/peplum/application/services/shared_hash.rb

@@ -0,0 +1,65 @@
+module Peplum
+class Application
+module Services
+
+class SharedHash
+
+  def initialize
+    @hash = {}
+  end
+
+  def get( k )
+    @hash[k]
+  end
+
+  def set( k, v, broadcast = true, &block )
+    if @hash[k] == v
+      block.call if block_given?
+      return
+    end
+
+    @hash[k] = v
+
+    if broadcast
+      each_peer do |peer|
+        peer.shared_hash.set( k, v, false ) {}
+      end
+    end
+
+    block.call if block_given?
+    nil
+  end
+
+  def delete( k, broadcast = true )
+    if !@hash.include? k
+      block.call if block_given?
+      return
+    end
+
+    @hash.delete( k )
+
+    if broadcast
+      each_peer do |_, peer|
+        peer.shared_hash.delete( k, false ) {}
+      end
+    end
+
+    block.call if block_given?
+    nil
+  end
+
+  def to_h
+    @hash.dup
+  end
+
+  private
+
+  def each_peer( &block )
+    Cuboid::Application.application.peers.each( &block )
+  end
+
+end
+
+end
+end
+end

data/lib/peplum/application.rb

@@ -0,0 +1,155 @@
+require 'cuboid'
+require 'json'
+require 'peplum'
+require 'peplum/core_ext/array'
+
+module Peplum
+  class Application < Cuboid::Application
+    require 'peplum/application/peers'
+
+    require 'peplum/application/services/shared_hash'
+    require 'peplum/application/services/scheduler'
+
+    class Error < Peplum::Error; end
+
+    class <<self
+      def inherited( application )
+        super
+
+        Cuboid::Application.application = application
+
+        application.validate_options_with :validate_options
+        application.serialize_with JSON
+
+        application.instance_service_for :scheduler,   Services::Scheduler
+        application.instance_service_for :shared_hash, Services::SharedHash
+      end
+    end
+
+    attr_reader :peers
+
+    def initialize(*)
+      super
+
+      @peers = Peers.new
+    end
+
+    def run
+      Raktr.global.on_error do |_, e|
+        $stderr.puts e
+      end
+
+      options = @options.dup
+      peplum_options = options.delete( 'peplum' )
+      native_options = options.delete( 'native' )
+
+      # We have a master so we're not the scheduler, run the payload.
+      if peplum_options['master']
+        execute( peplum_options, native_options )
+
+      # We're the scheduler Instance, get to grouping and spawning.
+      else
+        schedule( peplum_options, native_options )
+      end
+    end
+
+    # Implements:
+    #   * `.run` -- Worker; executes its payload against `objects`.
+    #   * `.group` -- Splits given `objects` into groups for each worker.
+    #   * `.merge` -- Merges results from multiple workers.
+    #
+    # That's all we need to turn any application into a super version of itself.
+    #
+    # @abstract
+    def native_app
+      fail Error, 'Missing native app!'
+    end
+
+    def report( data )
+      super native_app.merge( data )
+    end
+
+    private
+
+    def execute( peplum_options, native_options )
+      master_info = peplum_options.delete( 'master' )
+
+      self.peers.set( peplum_options.delete( 'peers' ) || {} )
+
+      report_data = native_app.run( peplum_options['objects'], native_options )
+
+      master = Processes::Instances.connect( master_info['url'], master_info['token'] )
+      master.scheduler.report report_data, Cuboid::Options.rpc.url
+    end
+
+    def schedule( peplum_options, native_options )
+      max_workers = peplum_options.delete('max_workers')
+      objects     = peplum_options.delete('objects')
+      groups      = native_app.group( objects, max_workers )
+
+      # Workload turned out to be less than our maximum allowed instances.
+      # Don't spawn the max if we don't have to.
+      if groups.size < max_workers
+        instance_num = groups.size
+
+        # Workload distribution turned out as expected.
+      elsif groups.size == max_workers
+        instance_num = max_workers
+
+        # What the hell did just happen1?
+      else
+        fail Error, 'Workload distribution error, uneven grouping!'
+      end
+
+      scheduler = self.scheduler.class
+      instance_num.times.each do |i|
+        # Get as many workers as necessary/possible.
+        break unless scheduler.get_worker
+      end
+
+      # We couldn't get the workers we were going for, Grid reached its capacity,
+      # re-balance distribution.
+      if scheduler.workers.size < groups.size
+        groups = native_app.group( objects, scheduler.workers.size )
+      end
+
+      peers = Hash[scheduler.workers.values.map { |client| [client.url, client.token] }]
+
+      scheduler.workers.values.each do |worker|
+        worker.run(
+          peplum: {
+            objects: groups.pop,
+            peers:   peers,
+            master:  {
+              url: Cuboid::Options.rpc.url,
+              token: Cuboid::Options.datastore.token
+            }
+          },
+          native: native_options
+        )
+      end
+
+      scheduler.wait
+    end
+
+    def validate_options( options )
+      if !Cuboid::Options.agent.url
+        fail Error, 'Missing Agent!'
+      end
+
+      peplum_options = options['peplum']
+
+      if !peplum_options.include? 'objects'
+        fail Error, 'Options: Missing :objects'
+      end
+
+      if !peplum_options['master'] && !peplum_options.include?( 'max_workers' )
+        fail Error, 'Options: Missing :max_workers'
+      end
+
+      @options = options
+      true
+    end
+
+  end
+end

data/lib/peplum/core_ext/array.rb

@@ -0,0 +1,20 @@
+class Array
+
+  def chunk( pieces = 2 )
+    return self if pieces <= 0
+
+    len    = self.length
+    mid    = len / pieces
+    chunks = []
+    start  = 0
+
+    1.upto( pieces ) do |i|
+      last = start + mid
+      last = last - 1 unless len % pieces >= i
+      chunks << self[ start..last ] || []
+      start = last + 1
+    end
+
+    chunks
+  end
+end

data/lib/peplum/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Peplum
+  VERSION = "0.1.0"
+end

data/lib/peplum.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+require "peplum/version"
+
+module Peplum
+  class Error < StandardError; end
+  # Your code goes here...
+
+  require "peplum/application"
+end

data/peplum.gemspec

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require_relative "lib/peplum/version"
+
+Gem::Specification.new do |spec|
+  spec.name = "peplum"
+  spec.version = Peplum::VERSION
+  spec.authors = ["Tasos Laskos"]
+  spec.email = ["tasos.laskos@gmail.com"]
+
+  spec.summary = "Distributed computing made easy."
+  spec.description = "Distributed computing made easy."
+  spec.homepage = "http://ecsypno.com/"
+  spec.required_ruby_version = ">= 2.6.0"
+
+  spec.files  = Dir.glob( 'bin/*')
+  spec.files += %w(bin/.gitkeep)
+  spec.files += Dir.glob( 'lib/**/*')
+  spec.files += Dir.glob( 'examples/**/*')
+  spec.files += %w(peplum.gemspec)
+
+  spec.add_dependency 'cuboid'
+end

metadata

@@ -0,0 +1,65 @@
+--- !ruby/object:Gem::Specification
+name: peplum
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Tasos Laskos
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2023-05-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: cuboid
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Distributed computing made easy.
+email:
+- tasos.laskos@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/.gitkeep
+- lib/peplum.rb
+- lib/peplum/application.rb
+- lib/peplum/application/peers.rb
+- lib/peplum/application/services/scheduler.rb
+- lib/peplum/application/services/shared_hash.rb
+- lib/peplum/core_ext/array.rb
+- lib/peplum/version.rb
+- peplum.gemspec
+homepage: http://ecsypno.com/
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.6.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.4.13
+signing_key: 
+specification_version: 4
+summary: Distributed computing made easy.
+test_files: []