zeus-justinf 0.13.5

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    YzA2MDdhY2JjNGYwMjBkNjRmNDI1MzJkNDcwNTU1MzcwNTE2NjdiYw==
+  data.tar.gz: !binary |-
+    YjAzZWU2OGFiYmVlYmY1OTRjNDUyZTM3YjhkYmZmYWZhZmQ2NjBlNw==
+SHA512:
+  metadata.gz: !binary |-
+    NDgyNmQ4ZmE0NWUyNzZkZjZjNDViNzlhZDMyZTczYmJjYTQ0ZTA2NTViM2Jl
+    NjY1NjFhOWU2NGMyMzc3NDBiOGQ1M2FiYzU4NjlmNDBjODFlZTNjYzhiNWM2
+    OTc0ZTJlYTgzYjUyYTIxZDE4MTFlYjYyZGU3ZWEzNTMyMmJlOWI=
+  data.tar.gz: !binary |-
+    MjZiMGNkNTkwNzE2ZGJhYTU5MjAzMTEwYWFjNzA3MDU0OWVhOWMxNTNmNzk2
+    NDBiNzAyNGMwYzExMDE1YzI4ZGI3YWEzZWVlZDI1ZTdjNGM5MDA1N2QzMjM5
+    MWI2ZTgwMzI5YzJjMWUxZWM0NWYxNDY4Yjc4OTM1MzI5YzMzYzA=

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in zeus.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,36 @@
+PATH
+  remote: .
+  specs:
+    zeus-justinf (0.13.5)
+      method_source (>= 0.6.7)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    diff-lcs (1.1.3)
+    hpricot (0.8.6)
+    method_source (0.8.2)
+    mustache (0.99.5)
+    rake (10.2.2)
+    rdiscount (2.1.7)
+    ronn (0.7.3)
+      hpricot (>= 0.8.2)
+      mustache (>= 0.7.0)
+      rdiscount (>= 1.5.8)
+    rspec (2.12.0)
+      rspec-core (~> 2.12.0)
+      rspec-expectations (~> 2.12.0)
+      rspec-mocks (~> 2.12.0)
+    rspec-core (2.12.2)
+    rspec-expectations (2.12.1)
+      diff-lcs (~> 1.1.3)
+    rspec-mocks (2.12.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  rake
+  ronn (>= 0.7.0)
+  rspec (~> 2.12.0)
+  zeus-justinf!

data/MIT-LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Burke Libbey
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,22 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+require 'fileutils'
+require 'pathname'
+
+ROOT_PATH = Pathname.new(File.expand_path("../../", __FILE__))
+RUBYGEM_PATH = Pathname.new(File.expand_path("../", __FILE__))
+
+task build: [:manifest]
+task default: :build
+
+task :manifest do
+  files = `find . -type f | sed 's|^\./||'`.lines.map(&:chomp)
+  exceptions = [
+    /.gitignore$/,
+    /^MANIFEST$/,
+    /^pkg\//,
+  ]
+  files.reject! { |f| exceptions.any? {|ex| f =~ ex }}
+  File.open('MANIFEST', 'w') {|f| f.puts files.join("\n") }
+end
+

data/bin/zeus

@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+platform = `uname -sm`
+
+exe = case platform
+      when /^Darwin/    ; "zeus-darwin-amd64"
+      when /^Linux.*64/ ; "zeus-linux-amd64"
+      when /^Linux.*/   ; "zeus-linux-386"
+      else
+        puts "Zeus is not supported on your platform."
+        puts "It's not likely to ever be possible on Windows."
+        puts "If you're using another platform that you think should work easily, open an issue at:"
+        puts "https://github.com/burke/zeus/issues"
+        exit 1
+      end
+
+zeusgemdir = File.expand_path("../../", __FILE__)
+exec "#{zeusgemdir}/build/#{exe}", *ARGV

data/ext/inotify-wrapper/extconf.rb

@@ -0,0 +1,24 @@
+if /linux/ =~ RUBY_PLATFORM
+  open("Makefile", "wb") do |f|
+    f.write <<-EOF
+CXX = g++
+CXXFLAGS = -O3 -g -Wall
+
+inotify-wrapper: inotify-wrapper.o
+	$(CXX) $(CXXFLAGS) $< -o $@
+
+%.o: %.cpp
+	$(CXX) $(CXXFLAGS) -c $< -o $@
+
+install:
+	# do nothing
+    EOF
+  end
+else
+  open("Makefile", "wb") do |f|
+    f.write <<-EOF
+install:
+	# do nothing
+    EOF
+  end
+end

data/ext/inotify-wrapper/inotify-wrapper.cpp

@@ -0,0 +1,116 @@
+#include <map>
+#include <string>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/inotify.h>
+
+#include <errno.h>
+
+#define EVENT_SIZE (sizeof (struct inotify_event))
+#define EVENT_BUF_LEN (1024 * (EVENT_SIZE + 16))
+
+using namespace std;
+
+static int _inotify_fd;
+static map<int, string> _WatchedFiles;
+static map<string, bool> _FileIsWatched;
+
+// static int inotifyFlags = IN_ATTRIB | IN_MODIFY | IN_MOVE_SELF | IN_DELETE_SELF;
+static int inotifyFlags = IN_ATTRIB | IN_MODIFY | IN_MOVE_SELF | IN_DELETE_SELF;
+
+void maybeAddFileToWatchList(string file)
+{
+  if (_FileIsWatched[file]) return;
+
+  int wd = inotify_add_watch(_inotify_fd, file.c_str(), inotifyFlags);
+  int attempts = 0;
+  // Files are momentarily inaccessible when they are rewritten. I couldn't
+  // find a good way to deal with this, so we poll 'deleted' files for 0.25s or so
+  // to see if they reappear.
+  while (wd == -1 && errno == ENOENT) {
+    usleep(10000);
+    wd = inotify_add_watch(_inotify_fd, file.c_str(), inotifyFlags);
+    if (attempts++ == 25) break; // try for at most about a quarter of a second
+  }
+  if (wd != -1) {
+    _WatchedFiles[wd] = file;
+    _FileIsWatched[file] = true;
+  }
+}
+
+// This essentially removes a file from the watchlist then
+// immediately re-adds it. This is because when a file is rewritten,
+// as so many editors love to do, the watchdescriptor no longer refers to
+// the file, so re must re-watch the path.
+void replaceFileInWatchList(int wd, string file)
+{
+  _FileIsWatched.erase(file);
+  _WatchedFiles.erase(wd);
+  inotify_rm_watch(_inotify_fd, wd);
+  maybeAddFileToWatchList(file);
+}
+
+void handleStdin()
+{
+  char line[2048];
+  if (fgets(line, sizeof(line), stdin) == NULL) return;
+  line[strlen(line)-1] = 0;
+
+  maybeAddFileToWatchList(string(line));
+}
+
+void handleInotify()
+{
+  int length;
+  int i = 0;
+  char buffer[EVENT_BUF_LEN];
+  string filename;
+
+  length = read(_inotify_fd, buffer, EVENT_BUF_LEN);
+  if (length < 0) return;
+
+  while (i < length) {
+    struct inotify_event *event = (struct inotify_event *) &buffer[i];
+    string file = _WatchedFiles[event->wd];
+    if (file != "") {
+      printf("%s\n", file.c_str());
+      fflush(stdout);
+      replaceFileInWatchList(event->wd, file);
+    }
+
+    i += EVENT_SIZE + event->len;
+  }
+}
+
+void go()
+{
+  fd_set rfds;
+  int retval;
+
+  for (;;) {
+    FD_ZERO(&rfds);
+    FD_SET(0, &rfds);
+    FD_SET(_inotify_fd, &rfds);
+
+    retval = select(_inotify_fd+1, &rfds, NULL, NULL, NULL);
+
+    if (retval == -1) {
+      // perror("select");
+    } else if (retval) {
+      if (FD_ISSET(0, &rfds))           handleStdin();
+      if (FD_ISSET(_inotify_fd, &rfds)) handleInotify();
+    }
+  }
+}
+
+
+int main(int argc, const char *argv[])
+{
+  _inotify_fd = inotify_init();
+  go();
+}

data/lib/zeus.rb

@@ -0,0 +1,226 @@
+# encoding: utf-8
+require 'socket'
+
+# load exact json version from Gemfile.lock to avoid conflicts
+gemfile = "#{ENV["BUNDLE_GEMFILE"] || "Gemfile"}.lock"
+if File.exist?(gemfile) && version = File.read(gemfile)[/^    json \((.*)\)/, 1]
+  gem 'json', version
+end
+require 'json'
+require 'pty'
+require 'set'
+
+require 'zeus/load_tracking'
+require 'zeus/plan'
+require 'zeus/version'
+
+module Zeus
+  class << self
+    attr_accessor :plan, :dummy_tty, :master_socket
+
+    # this is totally asinine, but readline gets super confused when it's
+    # required at a time when stdin or stdout is not connected to a TTY,
+    # no matter what we do to tell it otherwise later. So we create a dummy
+    # TTY in case readline is required.
+    #
+    # Yup.
+    def setup_dummy_tty!
+      return if self.dummy_tty
+      master, self.dummy_tty = PTY.send(:open)
+      Thread.new {
+        loop { master.read(1024) }
+      }
+      STDIN.reopen(dummy_tty)
+      STDOUT.reopen(dummy_tty)
+    end
+
+    def setup_master_socket!
+      return master_socket if master_socket
+
+      fd = ENV['ZEUS_MASTER_FD'].to_i
+      self.master_socket = UNIXSocket.for_fd(fd)
+    end
+
+    def go(identifier=:boot)
+      # Thanks to the magic of fork, this following line will return
+      # many times: Every time the parent step receives a request to
+      # run a command.
+      if run_command = boot_steps(identifier)
+        ident, local = run_command
+        return command(ident, local)
+      end
+    end
+
+    def boot_steps(identifier)
+      while true
+        boot_step = catch(:boot_step) do
+          $0 = "zeus slave: #{identifier}"
+
+          setup_dummy_tty!
+          master = setup_master_socket!
+          feature_pipe_r, feature_pipe_w = IO.pipe
+
+          # I need to give the master a way to talk to me exclusively
+          local, remote = UNIXSocket.pair(Socket::SOCK_STREAM)
+          master.send_io(remote)
+
+          # Now I need to tell the master about my PID and ID
+          local.write "P:#{Process.pid}:#{identifier}\0"
+          local.send_io(feature_pipe_r)
+          feature_pipe_r.close
+
+          # Now we run the action and report its success/fail status to the master.
+          features = Zeus::LoadTracking.features_loaded_by {
+            run_action(local, identifier, feature_pipe_w)
+          }
+
+          # the master wants to know about the files that running the action caused us to load.
+          Thread.new { notify_features(feature_pipe_w, features) }
+
+          # We are now 'connected'. From this point, we may receive requests to fork.
+          children = Set.new
+          while true
+            messages = local.recv(2**16)
+
+            # Reap any child runners or slaves that might have exited in
+            # the meantime. Note that reaping them like this can leave <=1
+            # zombie process per slave around while the slave waits for a
+            # new command.
+            children.each do |pid|
+              children.delete(pid) if Process.waitpid(pid, Process::WNOHANG)
+            end
+
+            messages.split("\0").each do |new_identifier|
+              new_identifier =~ /^(.):(.*)/
+              code, ident = $1, $2
+              pid = fork
+              if pid
+                # We're in the parent. Record the child:
+                children << pid
+              elsif code == "S"
+                # Child, supposed to start another step:
+                throw(:boot_step, ident.to_sym)
+              else
+                # Child, supposed to run a command:
+                return [ident.to_sym, local]
+              end
+            end
+          end
+        end
+        identifier = boot_step
+      end
+    end
+
+    private
+
+    def command(identifier, sock)
+      $0 = "zeus runner: #{identifier}"
+      Process.setsid
+
+      local, remote = UNIXSocket.pair(:DGRAM)
+      sock.send_io(remote)
+      remote.close
+      sock.close
+
+      pid_and_argument_and_environment_count = local.recv(2**16).chomp("\0")
+      client_pid, argument_count, environment_count = pid_and_argument_and_environment_count.split(":")
+
+      argument_count = argument_count.to_i
+      environment_count = environment_count.to_i
+
+      arg_io = local.recv_io
+      transmitted = arg_io.read.chomp("\0").split("\0")
+
+      if transmitted.length != argument_count + environment_count
+        raise "Transmitted count mismatch: Expected #{argument_count} (args) + #{environment_count} (env), got #{transmitted.length}"
+      end
+
+      arguments = transmitted.slice(0, argument_count)
+      environment = transmitted.slice(argument_count, environment_count)
+
+      pid = fork {
+        $0 = "zeus command: #{identifier}"
+
+        environment.each do |env_var|
+          key, val = env_var.split("=")
+          ENV[key] = val
+        end
+
+        plan.after_fork
+        client_terminal = local.recv_io
+
+        local.write "P:#{Process.pid}:\0"
+        local.close
+
+        $stdin.reopen(client_terminal)
+        $stdout.reopen(client_terminal)
+        $stderr.reopen(client_terminal)
+        ARGV.replace(arguments)
+
+        plan.send(identifier)
+      }
+
+      kill_command_if_client_quits!(pid, client_pid)
+
+      Process.wait(pid)
+      code = $?.exitstatus || 0
+
+      local.write "#{code}\0"
+
+      local.close
+    rescue Exception
+      # If anything at all went wrong, kill the client - if anything
+      # went wrong before the runner can clean up, it might hang
+      # around forever.
+      Process.kill(:TERM, client_pid)
+    end
+
+    def kill_command_if_client_quits!(command_pid, client_pid)
+      Thread.new {
+        loop {
+          begin
+            Process.kill(0, client_pid)
+          rescue Errno::ESRCH
+            Process.kill(9, command_pid)
+            exit 0
+          end
+          sleep 1
+        }
+      }
+    end
+
+    def notify_features(pipe, features)
+      features.each do |t|
+        pipe.puts t
+      end
+    end
+
+    def report_error_to_master(local, error)
+      str = "R:"
+      str << "#{error.backtrace[0]}: #{error.message} (#{error.class})\n"
+      error.backtrace[1..-1].each do |line|
+        str << "\tfrom #{line}\n"
+      end
+      str << "\0"
+      local.write str
+    end
+
+    def run_action(socket, identifier, feature_pipe_w)
+      loaded = false
+      begin
+        plan.after_fork unless identifier == :boot
+        plan.send(identifier)
+        loaded = true
+        socket.write "R:OK\0"
+      rescue Exception => e
+        report_error_to_master(socket, e)
+
+        # Report any setup-time failures back to the Zeus master:
+        unless loaded
+          notify_features(feature_pipe_w, Zeus::LoadTracking.all_features)
+        end
+        feature_pipe_w.close
+      end
+    end
+  end
+end