stapel 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 7aa52485f766d6fa3f1753c76edd0fe1c7d4fd22
+  data.tar.gz: c76735760f6cc822ccaf9b1381f76b12d3a649ed
+SHA512:
+  metadata.gz: 823f20bcc19751e612e77a95614cc26bf79567353bf4c120e5f5bb748f9c3cd5a2bc8046d7f48b62d1dfa461878ca71d6661e25e07513943624d9e43a091fa47
+  data.tar.gz: b5431fa08b220b00b2c8f39de551f1317ede0b644eea03348685c91e899613eebae575e5897322028c9326c0c04c9d7b71b6dd8b9c8f5cd585c770a16a58ee5a

data/bin/stapel

@@ -0,0 +1,51 @@
+#!/usr/bin/env ruby
+require_relative '../lib/pool.rb'
+require_relative '../lib/machine.rb'
+require_relative '../lib/worker.rb'
+# project layout:
+# inputs
+#   job1 ... any filename
+#   job2
+# output
+#   job1 #this will be copied from ~user/output
+#     files written by job1
+#   job2.cpy #unfinished copy of job, will be retried when master is restarted
+# data
+#   some_file #additional files that your runner needs (libs, binaries whatever
+# config #contains a ssh config for client list
+# run #the script that is executed on the worker for each input
+# init #the script that is executed on the worker once to setup the worker
+# cleanup #the script that is executed on the worker after a worker is finished
+# aggregate #the script that will be executed on the master to aggregate all results after all jobs finished
+
+def aggregate(path_to_env,output)
+  aggregate_path = File.join(path_to_env, "aggregate")
+  system(aggregate_path, output) if File.exists?(aggregate_path)
+end
+
+def run(path_to_env)
+  pool = WorkPool.new(path_to_env)
+  Machine.get_all(path_to_env).each do |machine|
+    machine.num_workers.times do
+      Thread.new { run_worker_on_machine(path_to_env, machine, pool) }
+    end
+  end
+  Thread.list.each{|t| t.join unless t == Thread.current}
+  aggregate(path_to_env, pool.output)
+end
+
+def run_worker_on_machine(path_to_env, machine, pool)
+  worker = Worker.new(path_to_env, machine, pool)
+  worker.upload_environment
+  while (input = pool.take)
+    worker.run_input(input)
+    worker.download_result(input)
+  end
+ensure
+  worker.cleanup if worker
+end
+
+
+Thread.abort_on_exception = true
+fail 'usage master.rb path/to/project' unless ARGV.length == 1
+run(File.expand_path(ARGV[0]))

data/lib/machine.rb

@@ -0,0 +1,44 @@
+require 'yaml'
+# A class that handles the ssh conection and parameters to a single machine
+class Machine
+  attr_accessor :host, :user, :keys, :num_workers, :port
+
+  def initialize(params)
+    @host, @user, @keys = params['host'], params['user'], params['keys']
+    @num_workers, @port = params['num_workers'], params['port']
+  end
+
+  def self.get_all(path_to_env)
+    config = YAML.load_file(File.join(path_to_env, 'config.yaml'))
+    default = config['default']
+    machines = config['clients'].map { |machine| default.merge(machine) }
+    machines.map { |m|  Machine.new(m) }
+  end
+
+  def to_ssh_args
+    args = { user: @user, port: @port, keys: @keys, keys_only: true }
+    args[:number_of_password_prompts] = 0
+    args
+  end
+
+  def ssh(cmd)
+    Net::SSH.start(@host, @user, to_ssh_args) do |ssh|
+      res = OpenStruct.new(stdout: '', stderr: '')
+      ssh.exec!(cmd) { |_channel, stream, data| res[stream] << data }
+      return res
+    end
+  end
+
+  def scp(local_path, remote_path)
+    raise "file not found: #{local_path}" unless File.exists?(local_path)
+    Net::SCP.upload!(@host, @user, local_path, remote_path, ssh: to_ssh_args, recursive: true) #doesn't change the name of the last folder in local path to remote_path...
+  end
+
+  def scp_r(remote_path, local_path)
+    Net::SCP.download!(@host, @user, remote_path, local_path, ssh: to_ssh_args, recursive: true)
+  end
+
+  def log(tag, text)
+    puts "[#@host,#{Time.now},#{tag}]: #{text}"
+  end
+end

data/lib/pool.rb

@@ -0,0 +1,27 @@
+require 'fileutils'
+require 'pathname'
+
+# a class that handles the pool of inputs
+class WorkPool
+  attr_accessor :output
+  def initialize(path_to_env)
+    input_prefix = Pathname.new(File.join(path_to_env, 'inputs'))
+    output_prefix = Pathname.new(File.join(path_to_env, 'output'))
+    Dir.glob("#{output_prefix}/**/*.cpy").each { |copy| FileUtils.rm_r(copy) }
+    inputs = inputs_under(input_prefix)
+    done =  inputs_under(output_prefix)
+    @aktive = inputs - done
+    @mut = Mutex.new
+    @output = File.join(path_to_env, "outputs_#{Time.now.to_i}")
+    FileUtils.mkdir( output )
+  end
+
+  def inputs_under(prefix)
+    glob = prefix.to_s + '/**/*'
+    Dir.glob(glob).map { |p| Pathname.new(p).relative_path_from(prefix).to_s }
+  end
+
+  def take
+    @mut.synchronize { return @aktive.pop }
+  end
+end

data/lib/worker.rb

@@ -0,0 +1,45 @@
+require 'ostruct'
+require 'net/ssh'
+require 'net/scp'
+
+# class used to represent a single worker on one machine
+class Worker
+  def initialize(path_to_env, machine, pool)
+    @path_to_env, @machine, @pool = path_to_env, machine, pool
+    @target_dir = "/home/#{machine.user}/"+@machine.ssh('mktemp -d stapel_tempdir_XXXXXX').stdout.strip
+    raise "failed to create temp dir" if @target_dir!~ /stapel_tempdir/
+    @path_to_output = pool.output
+  end
+
+  def upload(*paths)
+    paths.each{|p| @machine.scp(File.join(@path_to_env,p), @target_dir)}
+  end
+
+  def upload_environment
+    upload("run", "init", "inputs", "data", "cleanup")
+    @machine.log("upload","done")
+    @machine.log("init",@machine.ssh("cd #{@target_dir}; ./init"))
+    @machine.log("init","done")
+  end
+
+  def run_input(input)
+    input_path = File.join(@target_dir, 'inputs', input) #this should include the project file name...
+    @machine.ssh("cd #{@target_dir}; rm -rf output; mkdir output")
+    @machine.log("run", "on #{input}")
+    @machine.log("run", @machine.ssh("cd #{@target_dir}; ./run #{input_path}"))
+  end
+
+  def download_result(input)
+    local_cpy_path = File.join(@path_to_output, "#{input}.cpy")
+    remote_output_path = File.join(@target_dir, 'output')
+    local_output_path = File.join(@path_to_output, input)
+    @machine.scp_r(remote_output_path, local_cpy_path)
+    FileUtils.mv(File.join(local_cpy_path,"output"), local_output_path)
+    FileUtils.rmdir(local_cpy_path)
+    @machine.log("download","done")
+  end
+
+  def cleanup
+    @machine.log("cleanup",@machine.ssh("cd #{@target_dir}; ./cleanup")) if @target_dir
+  end
+end

metadata

@@ -0,0 +1,82 @@
+--- !ruby/object:Gem::Specification
+name: stapel
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Cornelius Aschermann
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-08-03 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: net-scp
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.1
+- !ruby/object:Gem::Dependency
+  name: net-ssh
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+description: A tool for dead simple batch processing on multiple machines
+email: coco@hexgolems.com
+executables:
+- stapel
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/stapel
+- lib/machine.rb
+- lib/pool.rb
+- lib/worker.rb
+homepage: http://github.com/eqv/stapel
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Batch processing made easy
+test_files: []