cuboid 0.0.0 → 0.0.1alpha

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: db5d6348cba47291af1506eda7d15f9e0bb215d02d3f586391f57c8e18728c8c
-  data.tar.gz: 202f146996c35458cb047aeb6ea1b5bba64a0b553a3e7d3d317fea2d8970200c
+  metadata.gz: b40773cee8ae1dcd306b0d5e1832f0b9948116f0098fedf1e3f5d6b89e8f7fe4
+  data.tar.gz: 38d5ef49a0329bb16788467653875820994c604d6ac20128e95af8b26811d7eb
 SHA512:
-  metadata.gz: 9f1a80f285c1cb7be6330048f69e6a070f85cead9ddf15a22a463fa5831679a97ef801b6b27b6d9740a41098a0d1a0c73aa6d507222c6cb53b3d878b05dad660
-  data.tar.gz: 6ca864801895355a080dfeaf28736196cabcb3e8bf3922934c16ad0e2a8520ac8ab9bcaf0be9075002d90d9a423648435d5cab3aca097f0b924dc73fdfa56aa1
+  metadata.gz: 7a83686aa22ee70f5bcdb9ee597c7dcc3f00fcbc511ad337980b725dfc47e0e94ff5089e73d26835a25777b389c79d9294c1ec4a7584d9ca18341a193a258d48
+  data.tar.gz: f9eb78ace44e7754f91543a9738443a8c15fe7fdac558a96aaf7290da5fa3cf35f570b49b82a51b5de2fd78e56781b80de9496eedf4632a8eedf8e8b360d3d0a

data/Gemfile

@@ -1,8 +1,23 @@
-# frozen_string_literal: true
+source 'https://rubygems.org'
 
-source "https://rubygems.org"
+gem 'rake', '11.3.0'
 
-# Specify your gem's dependencies in cuboid.gemspec
-gemspec
+group :docs do
+    gem 'yard'
+    gem 'redcarpet'
+end
+
+group :spec do
+    gem 'simplecov', require: false, group: :test
+
+    gem 'typhoeus'
+    gem 'rspec'
+    gem 'faker'
+end
 
-gem "rake", "~> 13.0"
+group :prof do
+    gem 'benchmark-ips'
+    gem 'memory_profiler'
+end
+
+gemspec

data/LICENSE.md

@@ -0,0 +1,22 @@
+# License
+
+Copyright (c) 2021 Tasos Laskos \<tasos.laskos@gmail.com\>.
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -1,35 +1,174 @@
-# Cuboid
+# Cuboid Framework -- a decentralized distributed framework in Ruby.
+
+## Summary
+
+The Cuboid Framework offers the possibility of easily creating decentralized and
+distributed applications in Ruby.
+
+In hipper terms, you can very easily setup your own specialized _Cloud_ or
+_Cloud_ within a _Cloud_.
+
+It offers:
+
+* Load-balancing of _**Instances**_ via a software mesh network (_**Grid**_) of _**Dispatchers**_.
+  * No need to setup a topology manually, _**Dispatchers**_ will reach
+    convergence on their own, just point them to an existing _**Grid**_ member.
+  * Scaling up and down can be easily achieved by _plugging_ or _unplugging_ nodes.
+  * Fault tolerant -- one application per process (_**Instance**_).
+  * Self-healing -- keeps an eye out for disappearing and also re-appearing members.
+* A clean and simple framework for application development.
+  * Basically, Ruby -- and all the libraries and extensions that come with it.
+* Events (_pause_, _resume_, _abort_, _suspend_, _restore_).
+  * Suspend to disk is a cinch by automatically creating a _**snapshot**_
+    of the runtime environment of the application and storing it to disk
+    for later restoration of state and data.
+    * Also allows for running job transfers.
+* Management of Instances via RPC or REST APIs.
+    * Aside from what **Cuboid** uses, custom serializers can be specified for
+      application related objects.
+* Developer freedom.
+  * Apart from keeping _Data_ and _State_ separate not many other rules to follow.
+    * Only if interested in _suspensions_ and can also be left to the last minute
+      if necessary -- in cases of Ractor enforced isolation  for example.
+
+## Entities
+
+### Application
+
+A Ruby `class` which inherits from `Cuboid::Application` and complies with a few
+simple specifications; at the least, a `#run` method, serving as the execution
+entry point.
 
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/cuboid`. To experiment with that code, run `bin/console` for an interactive prompt.
+The application can use the following methods to better take advantage of the
+framework:
 
-TODO: Delete this and the text above, and describe your gem
+  * `#validate_options( options )` -- Validates _**Application**_ options.
+  * `#provision_cores( Fixnum )` -- Specifies the maximum amount of cores the
+    _**Application**_ will be using.
+  * `#provision_memory( Fixnum )` -- Specifies the maximum amount of RAM the
+    _**Application**_ will be using.
+  * `#provision_disk( Fixnum )` -- Specifies the maximum amount of disk space the
+    _**Application**_ will be using.
+  * `#handler_for( Symbol, Symbol )` -- Specifies methods to handle the following events:
+    * `:pause`
+    * `:resume`
+    * `:abort`
+    * `:suspend`
+    * `:restore`
+  * `instance_service_for( Symbol, Class )` -- Adds a custom _**Instance**_ RPC API.
+  * `rest_service_for( Symbol, Module )` -- Hooks-up to the _**REST**_ service to provide a custom REST API.
+  * `dispatcher_service_for( Symbol, Class )` -- Hooks-up to the _**Dispatcher**_ to provide a custom RPC API.
+  * `serialize_with( Module )` -- A serializer to be used for:
+    * `#options`
+    * `Report#data`
+    * `Runtime` `Snapshot`
 
-## Installation
+Access is also available to:
 
-Add this line to your application's Gemfile:
+  * `#options` -- Passed options.
+  * `#runtime` -- The _**Application**_'s `Runtime` environment, as a way to
+    store and access _state_ and _data_. Upon receiving a _suspend_ event, the
+    `Runtime` will be stored to disk as a `Snapshot` for later restoration.
+    * `Runtime#state` -- State accessor.
+    * `Runtime#data` -- Data accessor.
+  * `#report( data )` -- Stores given `data`, to be included in a later generated
+    `Report` and accessed via `Report#data`.
 
-```ruby
-gem 'cuboid'
-```
+### Instance
 
-And then execute:
+An _**Instance**_ is a process container for a **Cuboid** _**Application**_;
+**Cuboid** is application-centric and follows the one process-per-application principle.
 
-    $ bundle install
+This is in order to enforce isolation (_state_, _data_, _fault_) between
+_**Applications**_, take advantage of _OS_ task management and generally keep
+things simple.
 
-Or install it yourself as:
+### Dispatcher
 
-    $ gem install cuboid
+A _**Dispatcher**_ is a server which awaits for _**Instance**_ spawn requests
+(`dispatch` calls) upon which it spawns and passes the _**Instance**_'s
+connection info to the client.
 
-## Usage
+The client can then proceed to use the _**Instance**_ to run and generally manage
+the contained _**Application**_.
 
-TODO: Write usage instructions here
+#### Grid
 
-## Development
+A _**Dispatcher**_ _**Grid**_ is a software mesh network of _**Dispatcher**_
+servers, aimed towards providing automated _load-balancing_ based on available
+system resources and each _**Application**_'s provisioning configuration.
 
-After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+No _topology_ needs to be specified, the only configuration necessary is
+providing any existing _**Grid**_ member upon start-up and the rest will be
+sorted out automatically.
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+The network is _self-healing_ and will monitor _node_ connectivity, taking steps
+to ensure that neither server nor network conditions will disrupt dispatching.
 
-## Contributing
+##### Scalability
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/cuboid.
+_**Dispatchers**_ can be easily _plugged_ to or _unplugged_ from the _**Grid**_
+to scale up or down as necessary.
+
+_Plugging_ happens at boot-time and _unplugging_ can take place via the available
+_APIs_.
+
+### Scheduler
+
+The _**Scheduler**_ is a server which:
+
+  1. Accepts _**Application**_ options. 
+  2. Stores them in a queue. 
+  3. Pops options and passes them to spawned _**Instances**_.
+  4. Monitors _**Instance**_ progress.
+  5. Upon _**Application**_ completion stores report to disk.
+  6. Shuts down the _**Instance**_.
+
+#### Dispatcher
+
+The _**Scheduler**_ can be configured with a _**Dispatcher**_, upon which case,
+it will use it to spawn _**Instances**_.
+
+If the _**Dispatcher**_ is a _**Grid**_ member then the _**Scheduler**_ will
+also enjoy _load-balancing_ features. 
+
+## APIs
+
+### Local
+
+Local access can call upon via the `Cuboid::Application` API and the API defined by the 
+_**Application**_ itself.
+
+### RPC
+
+A simple RPC is employed, specs for 3rd party implementations can be found at:
+
+https://github.com/Arachni/arachni-rpc/wiki
+
+Each _**Application**_ can extend upon this and expose an API via its _**Instance**_'s 
+RPC interface.
+
+### REST
+
+A REST API is also available, taking advantage of HTTP sessions to make progress
+tracking easier.
+
+The REST interface is basically a web _**Dispatcher**_ and centralised point of
+management for the rest of the entities.
+
+Each _**Application**_ can extend upon this and expose an API via its _**REST**_ 
+service's interface.
+
+## Examples
+
+See `examples/`.
+
+### MyApp
+
+Tutorial application going over different APIs and **Cuboid** _**Application**_
+options and specification.
+
+
+## License
+
+Please see the _LICENSE.md_ file.

data/Rakefile

@@ -1,4 +1,57 @@
-# frozen_string_literal: true
+=begin
+    Copyright 2020 Alex Douckas <alexdouckas@gmail.com>, Tasos Laskos <tasos.laskos@gmail.com>
 
-require "bundler/gem_tasks"
-task default: %i[]
+    This file is part of the Engine Framework project and is subject to
+    redistribution and commercial restrictions. Please see the Engine Framework
+    web site for more information on licensing and terms of use.
+=end
+
+require File.expand_path(File.dirname(__FILE__)) + '/lib/cuboid'
+
+begin
+    require 'rspec'
+    require 'rspec/core/rake_task'
+
+    namespace :spec do
+
+        desc 'Run core library tests.'
+        RSpec::Core::RakeTask.new( :core ) do |t|
+            t.pattern = FileList[ 'spec/cuboid/**/*_spec.rb' ]
+        end
+
+        desc 'Run plugin tests.'
+        RSpec::Core::RakeTask.new( :plugins ) do |t|
+            t.pattern = FileList[ 'spec/components/plugins/**/*_spec.rb' ]
+        end
+    end
+
+    RSpec::Core::RakeTask.new
+rescue LoadError
+    puts 'If you want to run the tests please install rspec first:'
+    puts '  gem install rspec'
+end
+
+desc 'Generate docs.'
+task :docs do
+    outdir = "../cuboid-docs"
+    sh "rm -rf #{outdir}"
+    sh "mkdir -p #{outdir}"
+
+    sh "yardoc -o #{outdir}"
+
+    sh "rm -rf .yardoc"
+end
+
+desc 'Remove reporter and log files.'
+task :clean do
+    files = %w(error.log *.crf *.csf *.yaml *.json *.marshal *.gem pkg/*.gem
+        reports/*.crf snapshots/*.csf logs/*.log spec/support/logs/*.log
+        spec/support/reports/*.crf spec/support/snapshots/*.csf
+    ).map { |file| Dir.glob( file ) }.flatten
+
+    next if files.empty?
+
+    puts 'Removing:'
+    files.each { |file| puts "  * #{file}" }
+    FileUtils.rm files
+end

data/config/paths.yml

@@ -0,0 +1,15 @@
+# Sets default locations for writing different kinds of files.
+#
+# * '~' will be expanded to $HOME.
+# * Directories will be created if they don't already exist.
+
+# Error and RPC logs.
+logs:
+# Default directory for scan snapshots generated by RPC Instances.
+snapshots:
+# Default report location for the RPC Scheduler.
+reports:
+# Directory for temporary files -- like for excess workload that's been
+# offloaded to disk etc.
+# Will default to the OS temporary directory.
+tmpdir:

data/cuboid.gemspec

@@ -1,31 +1,69 @@
-# frozen_string_literal: true
+# coding: utf-8
 
-require_relative "lib/cuboid/version"
+Gem::Specification.new do |s|
+    require_relative File.expand_path( File.dirname( __FILE__ ) ) + '/lib/cuboid/version'
 
-Gem::Specification.new do |spec|
-  spec.name          = "cuboid"
-  spec.version       = Cuboid::VERSION
-  spec.authors       = ["Tasos Laskos"]
-  spec.email         = ["tasos.laskos@gmail.com"]
+    s.name              = 'cuboid'
+    s.version           = Cuboid::VERSION
+    s.date              = Time.now.strftime( '%Y-%m-%d' )
+    s.summary           = 'An application-centric, decentralised and distributed computing solution. '
 
-  spec.summary       = "Placeholder"
-  spec.homepage      = "http://placeholder.com"
-  spec.required_ruby_version = Gem::Requirement.new(">= 2.4.0")
+    s.homepage          = 'https://github.com/qadron/cuboid'
+    s.email             = 'tasos.laskos@gmail.com'
+    s.authors           = [ 'Tasos Laskos' ]
+    s.licenses          = ['All rights reserved.']
 
-  spec.metadata["homepage_uri"] = spec.homepage
+    s.files            += Dir.glob( 'config/**/**' )
+    s.files            += Dir.glob( 'lib/**/**' )
+    s.files            += Dir.glob( 'logs/**/**' )
+    s.files            += Dir.glob( 'components/**/**' )
+    s.files            += Dir.glob( 'spec/**/**' )
+    s.files            += %w(Gemfile Rakefile cuboid.gemspec)
+    s.test_files        = Dir.glob( 'spec/**/**' )
 
-  # Specify which files should be added to the gem when it is released.
-  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
-  spec.files = Dir.chdir(File.expand_path(__dir__)) do
-    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{\A(?:test|spec|features)/}) }
-  end
-  spec.bindir        = "exe"
-  spec.executables   = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
-  spec.require_paths = ["lib"]
+    s.extra_rdoc_files  = %w(README.md LICENSE.md CHANGELOG.md)
 
-  # Uncomment to register a new dependency of your gem
-  # spec.add_dependency "example-gem", "~> 1.0"
+    s.rdoc_options      = [ '--charset=UTF-8' ]
+
+    s.add_dependency 'awesome_print',       '1.6.1'
+
+    # Don't specify version, messes with the packages since they always grab the
+    # latest one.
+    s.add_dependency 'bundler'
+
+    s.add_dependency 'concurrent-ruby',     '1.1.8'
+    s.add_dependency 'concurrent-ruby-ext', '1.1.8'
+
+    # For compressing/decompressing system state archives.
+    s.add_dependency 'rubyzip',             '1.2.2'
+
+    s.add_dependency 'childprocess',        '0.8.0'
+
+    # RPC serialization.
+    s.add_dependency 'msgpack',             '1.1.0'
+
+    # Optimized JSON.
+    s.add_dependency 'oj',                  '3.11.5'
+    s.add_dependency 'oj_mimic_json',       '1.0.1'
+
+    # Web server
+    s.add_dependency 'puma',                '3.10.0'
+
+    s.add_dependency 'rack',                '2.2.3'
+    s.add_dependency 'rack-test'
+
+    # REST API
+    s.add_dependency 'sinatra',             '2.1.0'
+    s.add_dependency 'sinatra-contrib',     '2.1.0'
+
+    # RPC client/server implementation.
+    s.add_dependency 'arachni-rpc',         '~> 0.2.1.4'
+
+    s.add_dependency 'vmstat',              '2.3.0'
+    s.add_dependency 'sys-proctable',       '1.1.5'
+
+    s.description = <<DESCRIPTION
+An application-centric, decentralised and distributed computing solution.
+DESCRIPTION
 
-  # For more information and examples about making a new gem, checkout our
-  # guide at: https://bundler.io/guides/creating_gem.html
 end

data/lib/cuboid.rb

@@ -1,8 +1,100 @@
-# frozen_string_literal: true
+require 'rubygems'
+require 'bundler/setup'
+require 'tmpdir'
 
-require_relative "cuboid/version"
+require 'oj'
+require 'oj_mimic_json'
+
+require_relative 'cuboid/version'
+
+require 'concurrent'
+require 'pp'
+require 'ap'
+
+def ap( obj )
+    super obj, raw: true
+end
 
 module Cuboid
-  class Error < StandardError; end
-  # Your code goes here...
+
+    class <<self
+
+        # Runs a minor GC to collect young, short-lived objects.
+        #
+        # Generally called after analysis operations that generate a lot of
+        # new temporary objects.
+        def collect_young_objects
+            # GC.start( full_mark: false )
+        end
+
+        def null_device
+            Gem.win_platform? ? 'NUL' : '/dev/null'
+        end
+
+        # @return   [Bool]
+        def windows?
+            Gem.win_platform?
+        end
+
+        # @return   [Bool]
+        def linux?
+            @is_linux ||= RbConfig::CONFIG['host_os'] =~ /linux/
+        end
+
+        # @return   [Bool]
+        def mac?
+            @is_mac ||= RbConfig::CONFIG['host_os'] =~ /darwin|mac os/i
+        end
+
+        # @return   [Bool]
+        #   `true` if the `CUBOID_PROFILE` env variable is set,
+        #   `false` otherwise.
+        def profile?
+            !!ENV['CUBOID_PROFILE']
+        end
+
+        if Cuboid.windows?
+            require 'find'
+            require 'fileutils'
+            require 'Win32API'
+            require 'win32ole'
+
+            def get_long_win32_filename( short_name )
+                short_name = short_name.dup
+                max_path   = 1024
+                long_name  = ' ' * max_path
+
+                lfn_size = Win32API.new(
+                    "kernel32", 
+                    "GetLongPathName",
+                    ['P','P','L'],
+                    'L'
+                ).call( short_name, long_name, max_path )
+
+                (1..max_path).include?( lfn_size ) ? 
+                    long_name[0..lfn_size-1] : short_name
+            end 
+        else
+            def get_long_win32_filename( short_name )
+                short_name
+            end
+        end
+    end
+
 end
+
+require_relative 'cuboid/banner'
+require_relative 'cuboid/ui/output_interface'
+
+# If there's no UI driving us then there's no output interface.
+# Chances are that someone is using Engine as a Ruby lib so there's no
+# need for a functional output interface, so provide non-functional one.
+#
+# However, functional or not, the system does depend on one being available.
+if !Cuboid::UI.constants.include?(:Output)
+    require_relative 'cuboid/ui/output'
+end
+
+require_relative 'cuboid/application'
+
+Cuboid::UI::OutputInterface.initialize