xpflow 0.1b

data/lib/xpflow/templates/utils/batch

@@ -0,0 +1,4 @@
+#!/bin/bash -eu
+# Runs SSH so that it will hold connection for the given amount of seconds.
+
+exec <%= path %>/ssh -f sleep ${1}

data/lib/xpflow/templates/utils/bootstrap

@@ -0,0 +1,12 @@
+#!/bin/bash -eu
+#
+#  We boostrap here and return YAML
+#
+#  The needed variables are:
+#     - tmpdir - tmpdir created on a remote site
+#
+
+tmpdir=$(<%= path %>/ssh "mktemp -d")
+
+echo "---"
+echo ":tmpdir: ${tmpdir}"

data/lib/xpflow/templates/utils/hostname

@@ -0,0 +1,3 @@
+#!/bin/bash -eu
+
+exec <%= path %>/ssh hostname

data/lib/xpflow/templates/utils/ping

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+exec <%= path %>/ssh true

data/lib/xpflow/templates/utils/rsync

@@ -0,0 +1,12 @@
+#!/bin/bash -eux
+
+if [ $# -ne 2 ]; then
+    echo "Give src dir and dst dir."
+    exit 1
+fi
+
+src="$1"
+dst="$2"
+
+cmd="scp -S <%= path %>/ssh $src :$dst"
+exec "$cmd"

data/lib/xpflow/templates/utils/scp

@@ -0,0 +1,17 @@
+#!/bin/bash -eu
+
+if [ $# -ne 2 ]; then
+    echo "Give src dir and dst dir."
+    exit 1
+fi
+
+src="$1"
+dst="$2"
+
+switch=""
+
+if [ -d "$src" ]; then
+    switch="-r"
+fi
+
+exec scp -F <%= path %>/ssh-config ${switch} ${src} this:${dst}

data/lib/xpflow/templates/utils/scp_many

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -eux
+
+dest_dir=${1}
+shift
+
+exec scp -F <%= path %>/ssh-config -- "$@" this:${dest_dir}

data/lib/xpflow/templates/utils/ssh

@@ -0,0 +1,3 @@
+#!/bin/bash -eu
+
+exec ssh -F <%= path %>/ssh-config this "$@"

data/lib/xpflow/templates/utils/ssh-interactive

@@ -0,0 +1,4 @@
+#!/bin/bash -eu
+#
+
+exec <%= path %>/ssh -t

data/lib/xpflow/templates/utils/taktuk

@@ -0,0 +1,19 @@
+#!/bin/bash -eu
+# 
+# this assumes that taktuk is installed on nodes
+
+if [ $# -eq 0 ]; then
+    echo "No file given."
+    exit 1
+fi
+
+list_local=${1}
+shift
+
+tmpdir=$(<%= path %>/ssh mktemp -d)
+list_remote=${tmpdir}/hostfile
+
+# <%= path %>/batch 3  # pipe connection for 3 seconds
+<%= path %>/scp ${list_local} ${list_remote}
+<%= path %>/ssh taktuk -f ${list_remote} "$@"
+# <%= path %>/ssh rm -rf ${tmpdir}

data/lib/xpflow/threads.rb

@@ -0,0 +1,187 @@
+
+require 'thread'
+
+module XPFlow
+
+    $xpflow_global_mutex = Mutex.new
+
+    def self.global_synchronize(&block)
+        return $xpflow_global_mutex.synchronize(&block)
+    end
+
+    $xpflow_global_counters = {}
+
+    def self.global_counter(label = :global)
+        value = global_synchronize do
+            h = $xpflow_global_counters
+            if h.key?(label)
+                h[label] += 1
+            else
+                h[label] = 0
+            end
+            h[label]
+        end
+        return value
+    end
+
+    # Implements a pool of threads.
+    # The threads can execute blocks of code
+
+    class ThreadPool
+
+        def initialize(size)
+            @size = size
+            @queue = Queue.new
+            @threads = size.times.map do
+                Thread.new do
+                    worker
+                end
+            end
+        end
+
+        def worker
+            while true
+                el = @queue.pop
+                break if el.nil?
+                block, args = el
+                block.call(*args)
+            end
+        end
+
+        def execute(*args, &block)
+            @queue.push([ block, args ])
+        end
+
+        def join(yes_or_no = true)
+            @size.times { @queue.push(nil) }
+            @threads.each { |t| t.join } if yes_or_no
+        end
+    end
+
+    class OrderedArray
+
+        def initialize
+            @queue = []
+            @lock = Mutex.new
+            @cond = ConditionVariable.new
+        end
+
+        def give(i, x)
+            @lock.synchronize do
+                @queue.push([i, x])
+                @cond.broadcast
+            end
+        end
+
+        def take(n)
+            @lock.synchronize do
+                while @queue.length < n
+                    @cond.wait(@lock)
+                end
+                arr = @queue.sort { |x, y| x.first <=> y.first }.map { |i, x| x }
+                arr
+            end
+        end
+
+    end
+
+    # Let's one thread wait for a value
+    # Do not call 'take' multiple times.
+
+    class Meeting
+
+        def initialize(run)
+            @lock = Mutex.new
+            @cond = ConditionVariable.new
+            @queue = []
+            @run = run
+        end
+
+        def give(x)
+            @lock.synchronize do
+                @queue.push(x)
+                @cond.broadcast
+            end
+        end
+
+        def _take(n)
+            flatten = (n < 0)
+            n = n.abs
+            @lock.synchronize do
+                while @queue.length < n
+                    @cond.wait(@lock)
+                end
+                arr = @queue.shift(n)
+                arr = arr.first if flatten
+                arr
+            end
+        end
+
+        def take(n)
+            begin
+                return _take(n)
+            rescue Interrupt => e
+                raise RunMsgError.new(@run, "Thread rendez-vous has been interrupted.")
+            end
+        end
+
+    end
+
+    # Collects exceptions (using 'push' method)
+    # and rethrows them collectively as RunError.
+
+    class ExceptionBag
+
+        def initialize
+            @lock = Mutex.new
+            @bag = []
+        end
+
+        def push(x)
+            @lock.synchronize do
+                @bag.push(x)
+            end
+        end
+
+        def raise_if_needed(run)
+            @lock.synchronize do
+                copy = @bag.map.to_a
+                raise RunError.new(run, copy) if copy.length > 0
+            end
+        end
+
+    end
+
+    class Threads
+        
+        def self.defaults
+            return { :join => true }
+        end
+
+        def self.merge_opts(x, y)
+            return x.merge(y) { |k, old, new|
+                (new.nil? ? old : new)
+            }
+        end
+
+        def self.run(run, list, opts = {}, &block)
+            opts = merge_opts(defaults(), opts)
+            bag = ExceptionBag.new
+            size = [ list.length, opts[:pool] ].min
+            pool = ThreadPool.new(size)
+            list.each_with_index do |el, i|
+                pool.execute(el, i) do |x, it|
+                    begin
+                        block.call(x, it)
+                    rescue RunError => e
+                        bag.push(e)
+                    end
+                end
+            end
+            pool.join(opts[:join])
+            bag.raise_if_needed(run)
+        end
+
+    end
+
+end