kamerling 0.0.1

data/Rakefile

@@ -0,0 +1,18 @@
+require 'rake/testtask'
+require 'reek/rake/task'
+require 'rubocop/rake_task'
+
+task default: %i[spec rubocop reek]
+
+Rake::TestTask.new :spec do |task|
+  task.test_files = FileList['spec/**/*_spec.rb']
+  task.warning    = true
+end
+
+Reek::Rake::Task.new do |task|
+  task.config_files  = 'config/reek.yml'
+  task.fail_on_error = false
+  task.reek_opts     = '--quiet'
+end
+
+Rubocop::RakeTask.new

data/bin/kamerling

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+
+require_relative '../lib/kamerling'
+
+Thread.abort_on_exception = true
+
+Kamerling::Logging.log_to Logger.new $stdout
+
+Kamerling::ServerRunner.new(ARGV).start.join

data/config/reek.yml

@@ -0,0 +1,18 @@
+DuplicateMethodCall:
+  exclude:
+  - Kamerling#self.class_definition_from
+  - Kamerling::Handler#handle
+
+IrresponsibleModule:
+  enabled: false
+
+NestedIterators:
+  max_allowed_nesting: 2
+
+UncommunicativeMethodName:
+  exclude:
+  - Kamerling#self.UUIDObject
+
+UnusedParameters:
+  exclude:
+  - Kamerling::Server::HTTP#initialize

data/kamerling.gemspec

@@ -0,0 +1,30 @@
+Gem::Specification.new do |gem|
+  gem.author      = 'Piotr Szotkowski'
+  gem.description = 'A network server for distributing computations across different clients speaking TCP or UDP.'
+  gem.email       = 'p.szotkowski@tele.pw.edu.pl'
+  gem.homepage    = 'https://github.com/chastell/kamerling'
+  gem.license     = 'AGPL-3.0'
+  gem.name        = 'kamerling'
+  gem.summary     = 'Kamerling: a computation network server'
+  gem.version     = '0.0.1'
+
+  gem.files       = `git ls-files -z`.split "\0"
+  gem.executables = gem.files.grep(%r{^bin/}).map { |path| File.basename path }
+  gem.test_files  = gem.files.grep %r{^spec/.*\.rb$}
+
+  gem.add_dependency 'after_do', '~> 0.3.0'
+  gem.add_dependency 'sequel',   '~> 4.4'
+  gem.add_dependency 'sinatra',  '~> 1.4'
+  gem.add_dependency 'slim',     '~> 2.0'
+  gem.add_dependency 'sqlite3',  '~> 1.3'
+
+  gem.add_development_dependency 'bogus',          '~> 0.1.3'
+  gem.add_development_dependency 'minitest',       '~> 5.0'
+  gem.add_development_dependency 'minitest-focus', '~> 1.1'
+  gem.add_development_dependency 'nokogiri',       '~> 1.6'
+  gem.add_development_dependency 'rack-test',      '~> 0.6.2'
+  gem.add_development_dependency 'rake',           '~> 10.1'
+  gem.add_development_dependency 'reek',           '~> 1.3'
+  gem.add_development_dependency 'rerun',          '~> 0.8.2'
+  gem.add_development_dependency 'rubocop',        '~> 0.18.0'
+end

data/lib/kamerling/addr.rb

@@ -0,0 +1,18 @@
+module Kamerling
+  Addr = Struct.new :host, :port, :prot do
+    def connectable?
+      TCPSocket.open(*self).close
+      true
+    rescue Errno::ECONNREFUSED
+      false
+    end
+
+    def to_a
+      [host, port]
+    end
+
+    def to_s
+      "#{host}:#{port} (#{prot})"
+    end
+  end
+end

data/lib/kamerling/client.rb

@@ -0,0 +1,2 @@
+module Kamerling class Client < UUIDObject :addr, busy: false
+end end

data/lib/kamerling/core_extensions/main.rb

@@ -0,0 +1,17 @@
+module Kamerling module CoreExtensions module Main
+  module_function
+
+  def req param
+    method   = caller.first[/`(.*)'$/, 1]
+    callsite = Class === self ? "#{name}.#{method}" : "#{self.class}##{method}"
+    raise "#{callsite}: param #{param} is required"
+  end
+
+  def warn_off
+    verbose  = $VERBOSE
+    $VERBOSE = false
+    yield
+  ensure
+    $VERBOSE = verbose
+  end
+end end end

data/lib/kamerling/core_extensions.rb

@@ -0,0 +1 @@
+require_relative 'core_extensions/main'

data/lib/kamerling/handler.rb

@@ -0,0 +1,24 @@
+module Kamerling class Handler
+  UnknownInput = Class.new RuntimeError
+
+  def initialize receiver: Receiver.new, registrar: Registrar.new
+    @receiver, @registrar = receiver, registrar
+  end
+
+  def handle input, addr
+    message = Message.new input
+    case message.type
+    when :RGST
+      registrar.register addr: addr, client_uuid: message.client_uuid,
+        project_uuid: message.project_uuid
+    when :RSLT
+      receiver.receive addr: addr, client_uuid: message.client_uuid,
+        data: message.payload, task_uuid: message.task_uuid
+    end
+  rescue Message::UnknownType => exception
+    raise UnknownInput, exception.message
+  end
+
+  attr_reader :receiver, :registrar
+  private     :receiver, :registrar
+end end

data/lib/kamerling/http_api.rb

@@ -0,0 +1,38 @@
+require 'forwardable'
+require 'sinatra/base'
+require 'slim'
+
+module Kamerling class HTTPAPI < Sinatra::Base
+  extend Forwardable
+
+  configure { set repos: Repos }
+
+  get '/' do
+    slim :root
+  end
+
+  get '/clients' do
+    warn_off { slim :clients, locals: { clients: repos.clients } }
+  end
+
+  get '/projects' do
+    warn_off { slim :projects, locals: { projects: repos.projects } }
+  end
+
+  get '/projects/:project_uuid' do
+    project = repos.project params['project_uuid']
+    clients = repos.clients_for project
+    tasks   = repos.tasks_for project
+    warn_off { slim :project, locals: { clients: clients, tasks: tasks } }
+  end
+
+  post '/projects' do
+    uuid = params.fetch('uuid') { UUID.new }
+    repos << Project.new(name: params['name'], uuid: uuid)
+    redirect '/projects'
+  end
+
+  private
+
+  delegate repos: :settings
+end end

data/lib/kamerling/logging.rb

@@ -0,0 +1,20 @@
+require 'after_do'
+require 'logger'
+
+module Kamerling module Logging
+  module_function
+
+  def log_to logger: Logger.new($stdout)
+    Server::Sock.extend AfterDo
+    Server::Sock.before :start do |*, server|
+      logger.info "start #{server.addr}"
+    end
+    Server::Sock.before :handle do |input, client_addr|
+      logger.info "connect #{client_addr}"
+      logger.debug "received #{client_addr} #{input}"
+    end
+    Server::Sock.after :stop do |*, server|
+      logger.info "stop #{server.addr}"
+    end
+  end
+end end

data/lib/kamerling/message.rb

@@ -0,0 +1,47 @@
+module Kamerling class Message
+  KnownTypes  = %w[DATA PING RGST RSLT]
+  UnknownType = Class.new RuntimeError
+
+  def self.[](client: req(:client), payload: req(:payload),
+              project: req(:project), task: req(:task), type: req(:type))
+    new "#{type}\0\0\0\0\0\0\0\0\0\0\0\0" + UUID.bin(client.uuid) +
+      UUID.bin(project.uuid) + UUID.bin(task.uuid) + payload
+  end
+
+  def initialize raw
+    @raw = raw
+    type = raw[0..3]
+    raise UnknownType, type unless KnownTypes.include? type or type.empty?
+  end
+
+  def == other
+    raw == other.raw
+  end
+
+  def client_uuid
+    UUID[raw[16..31]]
+  end
+
+  def payload
+    raw[64..-1]
+  end
+
+  def project_uuid
+    UUID[raw[32..47]]
+  end
+
+  def task_uuid
+    UUID[raw[48..63]]
+  end
+
+  def to_s
+    raw
+  end
+
+  def type
+    raw[0..3].to_sym
+  end
+
+  attr_reader :raw
+  private     :raw
+end end

data/lib/kamerling/migrations/1_basic_schema.rb

@@ -0,0 +1,45 @@
+Sequel.migration do
+  change do
+    create_table :clients do
+      uuid    :uuid, primary_key: true
+      boolean :busy, null: false
+      inet    :host, null: false
+      integer :port, null: false
+      string  :prot, null: false
+    end
+
+    create_table :projects do
+      uuid   :uuid, primary_key: true
+      string :name, null: false
+    end
+
+    create_table :registrations do
+      uuid    :uuid, primary_key: true
+      inet    :host, null: false
+      integer :port, null: false
+      string  :prot, null: false
+      foreign_key :client_uuid,  :clients,  index: true, null: false,
+        type: :uuid
+      foreign_key :project_uuid, :projects, index: true, null: false,
+        type: :uuid
+    end
+
+    create_table :results do
+      uuid    :uuid, primary_key: true
+      bytea   :data, null: false
+      inet    :host, null: false
+      integer :port, null: false
+      string  :prot, null: false
+      foreign_key :client_uuid, :clients, index: true, null: false, type: :uuid
+      foreign_key :task_uuid,   :tasks,   index: true, null: false, type: :uuid
+    end
+
+    create_table :tasks do
+      uuid    :uuid, primary_key: true
+      boolean :done, null: false
+      bytea   :data, null: false
+      foreign_key :project_uuid, :projects, index: true, null: false,
+        type: :uuid
+    end
+  end
+end

data/lib/kamerling/net_dispatcher.rb

@@ -0,0 +1,8 @@
+module Kamerling class NetDispatcher
+  def dispatch addr, bytes
+    case addr.prot
+    when :TCP then TCPSocket.open(*addr) { |socket| socket << bytes }
+    when :UDP then UDPSocket.new.send bytes, 0, *addr
+    end
+  end
+end end

data/lib/kamerling/project.rb

@@ -0,0 +1,2 @@
+module Kamerling class Project < UUIDObject :name
+end end

data/lib/kamerling/receiver.rb

@@ -0,0 +1,11 @@
+module Kamerling class Receiver
+  def receive addr: req(:addr), client_uuid: req(:client_uuid),
+              data: req(:data), repos: Repos, task_uuid: req(:task_uuid)
+    client = repos[Client][client_uuid]
+    task   = repos[Task][task_uuid]
+    result = Result.new addr: addr, client: client, data: data, task: task
+    client.busy = false
+    task.done   = true
+    repos << result << client << task
+  end
+end end

data/lib/kamerling/registrar.rb

@@ -0,0 +1,9 @@
+module Kamerling class Registrar
+  def register addr: req(:addr), client_uuid: req(:client_uuid),
+               project_uuid: req(:project_uuid), repos: Repos
+    client  = repos[Client][client_uuid]
+    project = repos[Project][project_uuid]
+    reg     = Registration.new addr: addr, client: client, project: project
+    repos[Registration] << reg
+  end
+end end

data/lib/kamerling/registration.rb

@@ -0,0 +1,2 @@
+module Kamerling class Registration < UUIDObject :addr, :client, :project
+end end

data/lib/kamerling/repo.rb

@@ -0,0 +1,32 @@
+module Kamerling class Repo
+  NotFound = Class.new RuntimeError
+
+  def initialize klass, source
+    @klass, @source = klass, source
+  end
+
+  def << object
+    hash = object.to_h
+    warn_off { source << hash }
+  rescue Sequel::UniqueConstraintViolation
+    warn_off { source.where(uuid: object.uuid).update hash }
+  end
+
+  def [] uuid
+    hash = warn_off { source[uuid: uuid] }
+    raise NotFound, "#{klass} with UUID #{uuid}" unless hash
+    klass.from_h hash
+  end
+
+  def all
+    source.all.map { |hash| klass.from_h hash }
+  end
+
+  def related_to object
+    key = "#{object.class.name.split('::').last.downcase}_uuid".to_sym
+    source.where(key => object.uuid).map { |hash| klass.from_h hash }
+  end
+
+  attr_reader :klass, :source
+  private     :klass, :source
+end end

data/lib/kamerling/repos.rb

@@ -0,0 +1,65 @@
+warn_off { require 'sequel' }
+
+Sequel.extension :migration
+
+module Kamerling class Repos
+  class << self
+    attr_writer :repos
+
+    def << object
+      repos[object.class] << object
+      self
+    end
+
+    def [] klass
+      repos[klass]
+    end
+
+    def clients
+      repos[Client].all
+    end
+
+    def clients_for project
+      repos[Registration].related_to(project).map(&:client)
+    end
+
+    def db= db
+      warn_off { Sequel::Migrator.run db, "#{__dir__}/migrations" }
+      @repos = nil
+      @db    = db
+    end
+
+    def free_clients_for project
+      clients_for(project).reject(&:busy)
+    end
+
+    def next_task_for project
+      repos[Task].related_to(project).reject(&:done).first
+    end
+
+    def project project_uuid
+      repos[Project][project_uuid]
+    end
+
+    def projects
+      repos[Project].all
+    end
+
+    def tasks_for project
+      repos[Task].related_to project
+    end
+
+    private
+
+    def db
+      @db ||= self.db = Sequel.sqlite
+    end
+
+    def repos
+      @repos ||= Hash.new do |repos, klass|
+        table = "#{klass.name.split('::').last.downcase}s".to_sym
+        repos[klass] = Repo.new klass, warn_off { db[table] }
+      end
+    end
+  end
+end end

data/lib/kamerling/result.rb

@@ -0,0 +1,2 @@
+module Kamerling class Result < UUIDObject :addr, :client, :data, :task
+end end

data/lib/kamerling/server/http.rb

@@ -0,0 +1,26 @@
+module Kamerling module Server class HTTP
+  attr_reader :addr
+
+  def initialize addr: req(:addr)
+    @addr = addr
+  end
+
+  def join
+    thread.join
+  end
+
+  def start
+    @thread = Thread.new do
+      Rack::Handler::WEBrick.run HTTPAPI, Host: addr.host, Port: addr.port
+    end
+    loop { break if addr.connectable? }
+    self
+  end
+
+  def stop
+    thread.exit.join
+  end
+
+  attr_reader :thread
+  private     :thread
+end end end

data/lib/kamerling/server/sock.rb

@@ -0,0 +1,32 @@
+module Kamerling module Server class Sock
+  attr_reader :addr
+
+  def initialize addr: req(:addr), handler: Handler.new
+    @addr    = addr
+    @handler = handler
+  end
+
+  def join
+    thread.join
+  end
+
+  def start
+    @thread = Thread.new { run_loop }
+    wait_till_started
+    self
+  end
+
+  def stop
+    thread.exit.join
+  end
+
+  attr_reader :handler, :thread
+  private     :handler, :thread
+
+  private
+
+  def handle input, client_addr
+    handler.handle input, client_addr
+  rescue Handler::UnknownInput
+  end
+end end end

data/lib/kamerling/server/tcp.rb

@@ -0,0 +1,19 @@
+module Kamerling module Server class TCP < Sock
+  private
+
+  def handle_connection socket
+    client_addr = Addr[*socket.remote_address.ip_unpack, :TCP]
+    input       = socket.read
+    handle input, client_addr
+  ensure
+    socket.close
+  end
+
+  def run_loop
+    Socket.tcp_server_loop(*addr) { |socket| handle_connection socket }
+  end
+
+  def wait_till_started
+    loop { break if addr.connectable? }
+  end
+end end end

data/lib/kamerling/server/udp.rb

@@ -0,0 +1,20 @@
+module Kamerling module Server class UDP < Sock
+  private
+
+  def handle_connection socket
+    input, conn = socket.recvfrom 2**16
+    client_addr = Addr[conn[3], conn[1], :UDP]
+    handle input, client_addr
+  end
+
+  def run_loop
+    socket = UDPSocket.new.tap { |server| server.bind(*addr) }
+    loop { handle_connection socket if IO.select [socket] }
+  ensure
+    socket.close if socket
+  end
+
+  def wait_till_started
+    200.times { thread.run }
+  end
+end end end

data/lib/kamerling/server_runner.rb

@@ -0,0 +1,50 @@
+require 'optparse'
+
+module Kamerling class ServerRunner
+  Settings = Struct.new(*%i[db host http tcp udp])
+
+  def initialize args, classes: def_classes, orm: Sequel, repos: Repos
+    @args    = args
+    repos.db = orm.connect settings.db
+    @servers = { http: :TCP, tcp: :TCP, udp: :UDP }.map do |type, prot|
+      addr = Addr[settings.host, settings.send(type), prot]
+      classes[type].new addr: addr if addr.port
+    end.compact
+  end
+
+  def join
+    servers.each(&:join)
+  end
+
+  def start
+    servers.each(&:start)
+    self
+  end
+
+  attr_reader :args, :servers
+  private     :args, :servers
+
+  private
+
+  def def_classes
+    { http: Server::HTTP, tcp: Server::TCP, udp: Server::UDP }
+  end
+
+  def settings
+    @settings ||= Settings.new.tap do |sets|
+      sets.db   = 'sqlite::memory:'
+      sets.host = '127.0.0.1'
+      OptionParser.new do |opts|
+        opts.on("--db #{sets.db}", String, 'database') do |db|
+          sets.db = db
+        end
+        opts.on("--host #{sets.host}", String, 'server host') do |host|
+          sets.host = host
+        end
+        opts.on('--http 0', Integer, 'HTTP port') { |http| sets.http = http }
+        opts.on('--tcp 0',  Integer, 'TCP port')  { |tcp|  sets.tcp  = tcp  }
+        opts.on('--udp 0',  Integer, 'UDP port')  { |udp|  sets.udp  = udp  }
+      end.parse! args
+    end
+  end
+end end

data/lib/kamerling/task.rb

@@ -0,0 +1,2 @@
+module Kamerling class Task < UUIDObject :data, :project, done: false
+end end

data/lib/kamerling/task_dispatcher.rb

@@ -0,0 +1,29 @@
+module Kamerling class TaskDispatcher
+  def initialize net_dispatcher: NetDispatcher.new, repos: Repos
+    @net_dispatcher = net_dispatcher
+    @repos          = repos
+  end
+
+  def dispatch
+    repos.projects.each do |project|
+      repos.free_clients_for(project).each do |client|
+        task = repos.next_task_for(project)
+        dispatch_task client: client, project: project, task: task if task
+      end
+    end
+  end
+
+  attr_reader :net_dispatcher, :repos
+  private     :net_dispatcher, :repos
+
+  private
+
+  def dispatch_task client: req(:client), project: req(:project),
+                    task: req(:task)
+    message = Message[client: client, payload: task.data, project: project,
+      task: task, type: :DATA]
+    net_dispatcher.dispatch client.addr, message.to_s
+    client.busy = true
+    repos << client
+  end
+end end

data/lib/kamerling/uuid.rb

@@ -0,0 +1,17 @@
+require 'securerandom'
+
+module Kamerling module UUID
+  module_function
+
+  def [] bin
+    bin.unpack('H8H4H4H4H12').join '-'
+  end
+
+  def bin uuid
+    [uuid.tr('-', '')].pack 'H*'
+  end
+
+  def new
+    SecureRandom.uuid
+  end
+end end