zeus 0.0.1 → 0.1.0

data/README.md

@@ -1,24 +1,55 @@
 # Zeus
 
-TODO: Write a gem description
+## What?
 
-## Installation
+Zeus preloads your app so that your normal development tasks such as `console`, `server`, `generate`, and tests are faster.
+
+## Why?
+
+Because waiting 25 seconds sucks, but waiting 0.4 seconds doesn't.
 
-Add this line to your application's Gemfile:
+## When?
 
-    gem 'zeus'
+Not yet. Zeus is nowhere near production-ready yet. Use only if you really like broken things.
 
-And then execute:
+## Ugly bits
+
+* Probably crashes a lot
+* Creates a bunch of sockets
+* Uses an obscene number of file descriptors
+
+## Installation
 
-    $ bundle
+Install the gem.
 
-Or install it yourself as:
+    gem install zeus
 
-    $ gem install zeus
+Copy `examples/rails.rb` to `{your app}/.zeus.rb`
 
 ## Usage
 
-TODO: Write usage instructions here
+Start the server:
+
+    zeus start
+
+Run some commands:
+
+    zeus console
+    zeus server
+    zeus testrb -Itest -I. test/unit/omg_test.rb
+    zeus generate model omg
+    zeus rake -T
+    zeus runner omg.rb
+
+## TODO (roughly prioritized)
+
+* Kill process when files are detected to have changed
+* Handle client/server without requiring a unix socket for each acceptor (1 shared socket)
+* Make the code less terrible
+* Figure out how to run full test suites without multiple env loads
+* Support other frameworks?
+* Use fsevents instead of kqueue to reduce the obscene number of file descriptors.
+* Support epoll on linux
 
 ## Contributing
 

data/bin/zeus

@@ -0,0 +1,13 @@
+#!/usr/bin/env ruby
+
+zeus = File.expand_path("../../lib/", __FILE__)
+$:.unshift(zeus) unless $:.include?(zeus)
+
+if ARGV[0] == "start"
+  require 'zeus/server'
+  require './.zeus.rb'
+  Zeus::Server.run
+else
+  require 'zeus/client'
+  Zeus::Client.run
+end

data/examples/rails.rb

@@ -0,0 +1,78 @@
+require 'socket'
+
+Zeus::Server.define! do
+  stage :boot do
+
+    action do
+      ENV_PATH  = File.expand_path('../config/environment',  __FILE__)
+      BOOT_PATH = File.expand_path('../config/boot',  __FILE__)
+      APP_PATH  = File.expand_path('../config/application',  __FILE__)
+      ROOT_PATH = File.expand_path('..',  __FILE__)
+
+      require BOOT_PATH
+      require 'rails/all'
+    end
+
+    stage :default_bundle do
+      action { Bundler.require(:default) }
+
+      stage :dev do
+        action do
+          Bundler.require(:development)
+          ENV['RAILS_ENV'] = "development"
+          require APP_PATH
+          Rails.application.require_environment!
+        end
+
+        acceptor :generate, ".zeus.dev_generate.sock" do
+          require 'rails/commands/generate'
+        end
+        acceptor :runner, ".zeus.dev_runner.sock" do
+          require 'rails/commands/runner'
+        end
+        acceptor :console, ".zeus.dev_console.sock" do
+          require 'rails/commands/console'
+          Rails::Console.start(Rails.application)
+        end
+
+        acceptor :server, ".zeus.dev_server.sock" do
+          require 'rails/commands/server'
+          server = Rails::Server.new
+          Dir.chdir(Rails.application.root)
+          server.start
+        end
+
+        stage :prerake do
+          action do
+            require 'rake'
+            load 'Rakefile'
+          end
+
+          acceptor :rake, ".zeus.dev_rake.sock" do
+            Rake.application.run
+          end
+
+        end
+      end
+
+      stage :test do
+        action do
+          ENV['RAILS_ENV'] = "test"
+          Bundler.require(:test)
+          require APP_PATH
+          Rails.application.require_environment!
+        end
+
+        acceptor :testrb, ".zeus.test_testrb.sock" do
+          forkpoint testrb: acceptor(".zeus.test_testrb.sock") {
+            (r = Test::Unit::AutoRunner.new(true)).process_args(ARGV) or
+              abort r.options.banner + " tests..."
+            exit r.run
+          }
+        end
+
+      end
+
+    end
+  end
+end

data/lib/zeus/client.rb

@@ -0,0 +1,78 @@
+require "io/console"
+require "json"
+require "pty"
+require "socket"
+
+module Zeus
+  class Client
+
+    SIGNALS = {
+      "\x03" => "TERM",
+      "\x1C" => "QUIT"
+    }
+    SIGNAL_REGEX = Regexp.union(SIGNALS.keys)
+
+    def self.maybe_raw(&b)
+      if $stdout.tty?
+        $stdout.raw(&b)
+      else
+        b.call
+      end
+    end
+
+    def self.run
+      maybe_raw do
+        PTY.open do |master, slave|
+          $stdout.tty? and master.winsize = $stdout.winsize
+          winch, winch_ = IO.pipe
+          trap("WINCH") { winch_ << "\0" }
+
+          case ARGV.shift
+          when 'testrb', 't'
+            socket = UNIXSocket.new(".zeus.test_testrb.sock")
+          when 'console', 'c'
+            socket = UNIXSocket.new(".zeus.dev_console.sock")
+          when 'server', 's'
+            socket = UNIXSocket.new(".zeus.dev_server.sock")
+          when 'rake'
+            socket = UNIXSocket.new(".zeus.dev_rake.sock")
+          when 'runner', 'r'
+            socket = UNIXSocket.new(".zeus.dev_runner.sock")
+          when 'generate', 'g'
+            socket = UNIXSocket.new(".zeus.dev_generate.sock")
+          end
+          socket.send_io(slave)
+          socket << ARGV.to_json << "\n"
+          slave.close
+
+          pid = socket.gets.strip.to_i
+
+          begin
+            buffer = ""
+
+            while ready = select([winch, master, $stdin])[0]
+              if ready.include?(winch)
+                winch.read(1)
+                $stdout.tty? and master.winsize = $stdout.winsize
+                Process.kill("WINCH", pid)
+              end
+
+              if ready.include?($stdin)
+                input = $stdin.readpartial(4096, buffer)
+                input.scan(SIGNAL_REGEX).each { |signal|
+                  Process.kill(SIGNALS[signal], pid)
+                }
+                master << input
+              end
+
+              if ready.include?(master)
+                $stdout << master.readpartial(4096, buffer)
+              end
+            end
+          rescue EOFError
+          end
+        end
+      end
+    end
+  end
+end

data/lib/zeus/process.rb

@@ -0,0 +1,20 @@
+module Process
+
+  def self.killall_descendants(sig, base=Process.pid)
+    descendants(base).each do |pid|
+      begin
+        Process.kill(sig, pid)
+      rescue Errno::ESRCH
+      end
+    end
+  end
+
+  def self.descendants(base=Process.pid)
+    descendants = Hash.new{|ht,k| ht[k]=[k]}
+    Hash[*`ps -eo pid,ppid`.scan(/\d+/).map{|x|x.to_i}].each{|pid,ppid|
+      descendants[ppid] << descendants[pid]
+    }
+    descendants[base].flatten - [base]
+  end
+end
+

data/lib/zeus/server.rb

@@ -0,0 +1,298 @@
+require 'json'
+require 'socket'
+
+require 'rb-kqueue'
+require 'zeus/process'
+
+module Zeus
+  module Server
+    def self.define!(&b)
+      @@root = Stage.new("(root)")
+      @@root.instance_eval(&b)
+      @@files = {}
+    end
+
+    def self.pid_has_file(pid, file)
+      @@files[file] ||= []
+      @@files[file] << pid
+    end
+
+    def self.killall_with_file(file)
+      pids = @@files[file]
+      @@process_tree.kill_nodes_with_feature(file)
+    end
+
+    TARGET_FD_LIMIT = 8192
+
+    def self.configure_number_of_file_descriptors
+      limit = Process.getrlimit(Process::RLIMIT_NOFILE)
+      if limit[0] < TARGET_FD_LIMIT && limit[1] >= TARGET_FD_LIMIT
+        Process.setrlimit(Process::RLIMIT_NOFILE, TARGET_FD_LIMIT)
+      else
+        puts "\x1b[33m[zeus] Warning: increase the max number of file descriptors. If you have a large project, this max cause a crash in about 10 seconds.\x1b[0m"
+      end
+    end
+
+    def self.notify(event)
+      if event.flags.include?(:delete)
+        # file was deleted, so we need to close and reopen it.
+        event.watcher.disable!
+        begin
+          @@queue.watch_file(event.watcher.path, :write, :extend, :rename, :delete, &method(:notify))
+        rescue Errno::ENOENT
+          lost_files << event.watcher.path
+        end
+      end
+      puts "\x1b[37m[zeus] dependency change: #{event.watcher.path}\x1b[0m"
+      killall_with_file(event.watcher.path)
+    end
+
+    def self.run
+      $0 = "zeus master"
+      configure_number_of_file_descriptors
+      trap("INT") { exit 0 }
+      at_exit { Process.killall_descendants(9) }
+
+      $r_features, $w_features = IO.pipe
+      $w_features.sync = true
+
+      $r_pids, $w_pids = IO.pipe
+      $w_pids.sync = true
+
+      @@process_tree = ProcessTree.new
+      @@root_stage_pid = @@root.run
+
+      @@queue = KQueue::Queue.new
+
+      lost_files = []
+
+      @@file_watchers = {}
+      loop do
+        @@queue.poll
+
+        # TODO: It would be really nice if we could put the queue poller in the select somehow.
+        #   --investigate kqueue. Is this possible?
+        rs, _, _ = IO.select([$r_features, $r_pids], [], [], 1)
+        rs.each do |r|
+          case r
+          when $r_pids     ; handle_pid_message(r.readline)
+          when $r_features ; handle_feature_message(r.readline)
+          end
+        end if rs
+      end
+
+    end
+
+    class ProcessTree
+      class Node
+        attr_accessor :pid, :children, :features
+        def initialize(pid)
+          @pid, @children, @features = pid, [], {}
+        end
+
+        def add_child(node)
+          self.children << node
+        end
+
+        def add_feature(feature)
+          self.features[feature] = true
+        end
+
+        def has_feature?(feature)
+          self.features[feature] == true
+        end
+
+        def inspect
+          "(#{pid}:#{features.size}:[#{children.map(&:inspect).join(",")}])"
+        end
+
+      end
+
+      def inspect
+        @root.inspect
+      end
+
+      def initialize
+        @root = Node.new(Process.pid)
+        @nodes_by_pid = {Process.pid => @root}
+      end
+
+      def node_for_pid(pid)
+        @nodes_by_pid[pid.to_i] ||= Node.new(pid.to_i)
+      end
+
+      def process_has_parent(pid, ppid)
+        curr = node_for_pid(pid)
+        base = node_for_pid(ppid)
+        base.add_child(curr)
+      end
+
+      def process_has_feature(pid, feature)
+        node = node_for_pid(pid)
+        node.add_feature(feature)
+      end
+
+      def kill_node(node)
+        @nodes_by_pid.delete(node.pid)
+        # recall that this process explicitly traps INT -> exit 0
+        Process.kill("INT", node.pid)
+      end
+
+      def kill_nodes_with_feature(file, base = @root)
+        if base.has_feature?(file)
+          if base == @root.children[0] || base == @root
+            puts "\x1b[31mOne of zeus's dependencies changed. Not killing zeus. You may have to restart the server.\x1b[0m"
+            return false
+          end
+          kill_node(base)
+          return true
+        else
+          base.children.dup.each do |node|
+            if kill_nodes_with_feature(file, node)
+              base.children.delete(node)
+            end
+          end
+          return false
+        end
+      end
+
+    end
+
+    def self.handle_pid_message(data)
+      data =~ /(\d+):(\d+)/
+      pid, ppid = $1.to_i, $2.to_i
+      @@process_tree.process_has_parent(pid, ppid)
+    end
+
+    def self.handle_feature_message(data)
+      data =~ /(\d+):(.*)/
+      pid, file = $1.to_i, $2
+      @@process_tree.process_has_feature(pid, file)
+      return if @@file_watchers[file]
+      begin
+        @@file_watchers[file] = true
+        @@queue.watch_file(file.chomp, :write, :extend, :rename, :delete, &method(:notify))
+      # rescue Errno::EMFILE
+      #   exit 1
+      rescue Errno::ENOENT
+        puts "No file found at #{file.chomp}"
+      end
+    end
+
+    class Stage
+      attr_reader :pid
+      def initialize(name)
+        @name = name
+        @stages, @actions = [], []
+      end
+
+      def action(&b)
+        @actions << b
+      end
+
+      def stage(name, &b)
+        @stages << Stage.new(name).tap { |s| s.instance_eval(&b) }
+      end
+
+      def acceptor(name, socket, &b)
+        @stages << Acceptor.new(name, socket, &b)
+      end
+
+      # There are a few things we want to accomplish:
+      # 1. Running all the actions (each time this stage is killed and restarted)
+      # 2. Starting all the substages (and restarting them when necessary)
+      # 3. Starting all the acceptors (and restarting them when necessary)
+      def run
+        @pid = fork {
+          $0 = "zeus spawner: #{@name}"
+          pid = Process.pid
+          $w_pids.puts "#{pid}:#{Process.ppid}\n"
+          puts "\x1b[35m[zeus] starting spawner `#{@name}`\x1b[0m"
+          trap("INT") {
+            puts "\x1b[35m[zeus] killing spawner `#{@name}`\x1b[0m"
+            exit 0
+          }
+
+          @actions.each(&:call)
+
+          $LOADED_FEATURES.each do |f|
+            $w_features.puts "#{pid}:#{f}\n"
+          end
+
+          pids = {}
+          @stages.each do |stage|
+            pids[stage.run] = stage
+          end
+
+          loop do
+            begin
+              pid = Process.wait
+            rescue Errno::ECHILD
+              raise "Stage `#{@name}` has no children. All terminal nodes must be acceptors"
+            end
+            if (status = $?.exitstatus) > 0
+              exit status
+            else # restart the stage that died.
+              stage = pids[pid]
+              pids[stage.run] = stage
+            end
+          end
+
+        }
+      end
+
+    end
+
+    class Acceptor
+      attr_reader :pid
+      def initialize(name, socket, &b)
+        @name = name
+        @socket = socket
+        @action = b
+      end
+
+      def run
+        @pid = fork {
+          $0 = "zeus acceptor: #{@name}"
+          pid = Process.pid
+          $w_pids.puts "#{pid}:#{Process.ppid}\n"
+          $LOADED_FEATURES.each do |f|
+            $w_features.puts "#{pid}:#{f}\n"
+          end
+          puts "\x1b[35m[zeus] starting acceptor `#{@name}`\x1b[0m"
+          trap("INT") {
+            puts "\x1b[35m[zeus] killing acceptor `#{@name}`\x1b[0m"
+            exit 0
+          }
+
+          File.unlink(@socket) rescue nil
+          server = UNIXServer.new(@socket)
+          loop do
+            ActiveRecord::Base.clear_all_connections! # TODO : refactor
+            client = server.accept
+            child = fork do
+              ActiveRecord::Base.establish_connection # TODO :refactor
+              ActiveSupport::DescendantsTracker.clear
+              ActiveSupport::Dependencies.clear
+
+              terminal = client.recv_io
+              arguments = JSON.load(client.gets.strip)
+
+              client << $$ << "\n"
+              $stdin.reopen(terminal)
+              $stdout.reopen(terminal)
+              $stderr.reopen(terminal)
+              ARGV.replace(arguments)
+
+              @action.call
+            end
+            Process.detach(child)
+            client.close
+          end
+        }
+      end
+
+    end
+
+  end
+end

data/lib/zeus/version.rb

@@ -1,3 +1,3 @@
 module Zeus
-  VERSION = "0.0.1"
+  VERSION = "0.1.0"
 end

data/zeus.gemspec

@@ -4,8 +4,8 @@ require File.expand_path('../lib/zeus/version', __FILE__)
 Gem::Specification.new do |gem|
   gem.authors       = ["Burke Libbey"]
   gem.email         = ["burke@libbey.me"]
-  gem.description   = %q{OMGZEUS: Write a gem description}
-  gem.summary       = %q{OMGZEUS: Write a gem summary}
+  gem.description   = %q{Zeus preloads pretty much everything you'll ever want to use in development.}
+  gem.summary       = %q{Zeus is an alpha-quality application preloader with terrible documentation.}
   gem.homepage      = ""
 
   gem.files         = `git ls-files`.split($\)
@@ -14,4 +14,6 @@ Gem::Specification.new do |gem|
   gem.name          = "zeus"
   gem.require_paths = ["lib"]
   gem.version       = Zeus::VERSION
+
+  gem.add_dependency "rb-kqueue-burke", "~> 0.1.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: zeus
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.1.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,12 +9,24 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-07-25 00:00:00.000000000 Z
-dependencies: []
-description: ! 'OMGZEUS: Write a gem description'
+date: 2012-07-27 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rb-kqueue-burke
+  requirement: &70349438039760 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: *70349438039760
+description: Zeus preloads pretty much everything you'll ever want to use in development.
 email:
 - burke@libbey.me
-executables: []
+executables:
+- zeus
 extensions: []
 extra_rdoc_files: []
 files:
@@ -23,7 +35,12 @@ files:
 - LICENSE
 - README.md
 - Rakefile
+- bin/zeus
+- examples/rails.rb
 - lib/zeus.rb
+- lib/zeus/client.rb
+- lib/zeus/process.rb
+- lib/zeus/server.rb
 - lib/zeus/version.rb
 - zeus.gemspec
 homepage: ''
@@ -49,5 +66,5 @@ rubyforge_project:
 rubygems_version: 1.8.11
 signing_key: 
 specification_version: 3
-summary: ! 'OMGZEUS: Write a gem summary'
+summary: Zeus is an alpha-quality application preloader with terrible documentation.
 test_files: []