tork 15.0.0

data/bin/tork-herald

@@ -0,0 +1,47 @@
+#!/usr/bin/env ruby
+=begin
+
+TORK-HERALD 1 2012-01-23 15.0.0
+==============================================================================
+
+NAME
+------------------------------------------------------------------------------
+
+tork-herald - reports modified files
+
+SYNOPSIS
+------------------------------------------------------------------------------
+
+`tork-herald` [*OPTION*]...
+
+DESCRIPTION
+------------------------------------------------------------------------------
+
+This program monitors the current working directory and prints relative paths
+of modified files, one per line, to the standard output stream.
+
+OPTIONS
+------------------------------------------------------------------------------
+
+`-h`, `--help`
+  Display this help manual using man(1).
+
+SEE ALSO
+------------------------------------------------------------------------------
+
+tork(1), tork-driver(1), tork-master(1), tork-herald(1)
+
+=end =========================================================================
+
+$0 = File.basename(__FILE__) # for easier indentification in ps(1) output
+
+require 'binman'
+BinMan.help
+
+require 'guard'
+require 'guard/listener'
+
+listener = Guard::Listener.select_and_init(:watch_all_modifications => true)
+listener.on_change {|files| puts files }
+STDOUT.sync = true # don't buffer puts()
+listener.start

data/bin/tork-master

@@ -0,0 +1,87 @@
+#!/usr/bin/env ruby
+=begin
+
+TORK-MASTER 1 2012-01-23 15.0.0
+==============================================================================
+
+NAME
+------------------------------------------------------------------------------
+
+tork-master - absorbs overhead and runs tests
+
+SYNOPSIS
+------------------------------------------------------------------------------
+
+`tork-master` [*OPTION*]...
+
+DESCRIPTION
+------------------------------------------------------------------------------
+
+This program reads the following single-line commands (JSON arrays) from its
+standard input stream and performs the respective actions as described below.
+
+`["load",` *paths*`,` *files*`]`
+  Adds the given array of *paths* to Ruby's $LOAD_PATH, loads the given array
+  of *files* after removing their ".rb" file extension if present, and prints
+  the given command line to the standard output stream.
+
+`["test",` *test_file*`,` *test_names*`]`
+  Runs the given *test_file* in a forked child process while instructing your
+  chosen unit testing framework (loaded by your test execution overhead) to
+  only run those tests that are named in the given array of *test_names*.
+
+  Prints the given command line to the standard output stream immediately
+  after forking the child process.
+
+  Prints the given command line, modified with `"pass"` (if the test passed)
+  or `"fail"` (if the test failed) in place of `"test"`, to the standard
+  output stream after the forked child process finishes.
+
+  The standard output and error streams of the forked child process are
+  redirected to a file whose path and name are the same as that of the test
+  file being run by the forked child process but with ".log" appended.
+
+`["stop"]`
+  Stops all tests that are currently running and prints the given command line
+  to the standard output stream.
+
+`["quit"]`
+  Stops all tests that are currently running and exits.
+
+OPTIONS
+------------------------------------------------------------------------------
+
+`-h`, `--help`
+  Display this help manual using man(1).
+
+FILES
+------------------------------------------------------------------------------
+
+*.tork.rb*
+  Optional Ruby script for configuring tork(1).
+
+ENVIRONMENT
+------------------------------------------------------------------------------
+
+`TORK_CONFIGS`
+  A single-line JSON array containing paths to actual files or names of
+  helper libraries in the tork/config/ namespace of Ruby's load path.
+  These configuration files are loaded just before *.tork.rb* is loaded.
+
+SEE ALSO
+------------------------------------------------------------------------------
+
+tork(1), tork-driver(1), tork-master(1), tork-herald(1)
+
+=end =========================================================================
+
+$0 = File.basename(__FILE__) # for easier indentification in ps(1) output
+
+require 'binman'
+BinMan.help
+
+require 'tork/master'
+Tork::Master.loop
+
+Process.waitall
+raise SystemExit # prevent empty test suite from running in the master process

data/lib/tork/client.rb

@@ -0,0 +1,34 @@
+module Tork
+module Client
+
+  class Receiver < Thread
+    def initialize *popen_args
+      (@io = IO.popen(*popen_args)).sync = true
+      super() { loop { yield @io.gets } }
+    end
+
+    def quit
+      kill # stop receive loop
+      Process.kill :SIGTERM, @io.pid
+      Process.wait @io.pid # reap zombie
+      @io.close # prevent further I/O
+    end
+  end
+
+  class Transceiver < Receiver
+    def initialize *popen_args
+      @io_write_lock = Mutex.new
+      popen_args[1] = 'w+'
+      super
+    end
+
+    def send command
+      @io_write_lock.synchronize do
+        warn "#{caller[2]} SEND #{command.inspect}" if $DEBUG
+        @io.puts JSON.dump(command)
+      end
+    end
+  end
+
+end
+end

data/lib/tork/config.rb

@@ -0,0 +1,83 @@
+require 'ostruct'
+
+module Tork
+  _user_config_file = '.tork.rb'
+
+  Config = OpenStruct.new
+
+  #---------------------------------------------------------------------------
+  # defaults
+  #---------------------------------------------------------------------------
+
+  Config.max_forked_workers = [
+    # http://stackoverflow.com/questions/891537#6420817
+    'fgrep -c processor /proc/cpuinfo', # Linux
+    'sysctl -n hw.ncpu',                # BSD
+    'hwprefs cpu_count',                # Darwin 9
+    'hwprefs thread_count',             # Darwin 10
+  ].map {|cmd| `#{cmd} 2>/dev/null`.to_i }.push(1).max
+
+  Config.overhead_load_paths = ['lib', 'test', 'spec']
+
+  Config.overhead_file_globs = ['{test,spec}/{test,spec}_helper.rb']
+
+  Config.reabsorb_file_greps = [/^#{Regexp.quote(_user_config_file)}$/,
+                                %r<(test|spec)/\1_helper\.rb>]
+
+  Config.all_test_file_globs = ['{test,spec}/**/*_{test,spec}.rb']
+
+  Config.test_file_globbers = {
+    # source files that correspond to test files
+    %r<^lib/.+\.rb$> => lambda do |path|
+      base = File.basename(path, '.rb')
+      "{test,spec}/**/#{base}_{test,spec}.rb"
+    end,
+
+    # the actual test files themselves
+    %r<^(test|spec)/.+_\1\.rb$> => lambda {|path| path }
+  }
+
+  Config.test_name_extractor = lambda do |line|
+    case line
+    when /^\s*def\s+test_(\w+)/ then $1
+    when /^\s*(test|context|should|describe|it)\b.+?(['"])(.*?)\2/ then $3
+    end
+  end
+
+  Config.before_fork_hooks = []
+
+  Config.after_fork_hooks = [
+    # tell testing framework to only run the named tests inside the test file
+    lambda do |worker_number, log_file, test_file, test_names|
+      unless test_names.empty?
+        case File.basename(test_file)
+        when /(\b|_)test(\b|_)/ # Test::Unit
+          ARGV.push '--name', "/(?i:#{
+            test_names.map do |name|
+              # elide string interpolation and invalid method name characters
+              name.gsub(/\#\{.*?\}/, ' ').strip.gsub(/\W+/, '.*')
+            end.join('|')
+          })/"
+        when /(\b|_)spec(\b|_)/ # RSpec
+          test_names.each {|name| ARGV.push '--example', name }
+        end
+      end
+    end
+  ]
+
+  #---------------------------------------------------------------------------
+  # overrides
+  #---------------------------------------------------------------------------
+
+  if ENV.key? 'TORK_CONFIGS'
+    JSON.load(ENV['TORK_CONFIGS']).each do |config|
+      if File.exist? config
+        load File.expand_path(config)
+      else
+        require "tork/config/#{config}"
+      end
+    end
+  end
+
+  load _user_config_file if File.exist? _user_config_file
+end

data/lib/tork/config/parallel_tests.rb

@@ -0,0 +1,7 @@
+require 'tork/config'
+
+Tork::Config.after_fork_hooks << proc do |worker_number|
+  # for compatitibilty with parallel_tests gem,
+  # store numbers as strings: "", "2", "3", "4"
+  ENV['TEST_ENV_NUMBER'] = (worker_number + 1).to_s if worker_number > 0
+end

data/lib/tork/config/rails.rb

@@ -0,0 +1,43 @@
+require 'tork/config'
+require 'active_support/inflector'
+
+Tork::Config.reabsorb_file_greps.push(
+  %r<^config/.+\.(rb|yml)$>,
+  %r<^db/schema\.rb$>,
+  %r<^Gemfile\.lock$>
+)
+
+Tork::Config.test_file_globbers[%r<^(app|lib|test|spec)/.+\.rb$>] =
+  lambda do |path|
+    base = File.basename(path, '.rb')
+    poly = ActiveSupport::Inflector.pluralize(base)
+    "{test,spec}/**/{#{base},#{poly}_*}_{test,spec}.rb"
+  end
+
+Tork::Config.test_file_globbers[%r<^(test|spec)/factories/.+_factory\.rb$>] =
+  lambda do |path|
+    base = File.basename(path, '_factory.rb')
+    poly = ActiveSupport::Inflector.pluralize(base)
+    "{test,spec}/**/{#{base},#{poly}_*}_{test,spec}.rb"
+  end
+
+Tork::Config.after_fork_hooks << proc do
+  unless ActiveRecord::Base.connection_config[:database] == ':memory:'
+    ActiveRecord::Base.connection.reconnect!
+  end
+end
+
+begin
+  require 'rails/railtie'
+  Class.new Rails::Railtie do
+    config.before_initialize do |app|
+      if app.config.cache_classes
+        warn "tork/config/rails: Setting #{app.class}.config.cache_classes = false"
+        app.config.cache_classes = false
+      end
+    end
+  end
+rescue LoadError
+  warn "tork/config/rails: Railtie not available; please manually set:\n\t"\
+       "config.cache_classes = false"
+end

data/lib/tork/driver.rb

@@ -0,0 +1,136 @@
+require 'json'
+require 'diff/lcs'
+require 'tork/client'
+require 'tork/server'
+require 'tork/config'
+
+module Tork
+module Driver
+
+  extend Server
+  extend self
+
+  def run_all_test_files
+    run_test_files Dir[*Config.all_test_file_globs]
+  end
+
+  def stop_running_test_files
+    @master.send [:stop]
+    @running_test_files.clear
+  end
+
+  def rerun_passed_test_files
+    run_test_files @passed_test_files
+  end
+
+  def rerun_failed_test_files
+    run_test_files @failed_test_files
+  end
+
+  def reabsorb_overhead_files
+    @master.quit if defined? @master
+
+    @master = Client::Transceiver.new('tork-master') do |line|
+      event, file, tests = JSON.load(line)
+
+      case event.to_sym
+      when :test
+        @waiting_test_files.delete file
+        @running_test_files.push file
+
+      when :pass
+        @running_test_files.delete file
+        @failed_test_files.delete file
+        if tests.empty? and not @passed_test_files.include? file
+          @passed_test_files.push file
+        end
+
+      when :fail
+        @running_test_files.delete file
+        @passed_test_files.delete file
+        @failed_test_files.push file unless @failed_test_files.include? file
+      end
+
+      @client.print line
+    end
+
+    @master.send [:load, Config.overhead_load_paths,
+                  Dir[*Config.overhead_file_globs]]
+
+    rerun_running_test_files
+  end
+
+  def loop
+    reabsorb_overhead_files
+
+    @herald = Client::Receiver.new('tork-herald') do |line|
+      changed_file = line.chomp
+      warn "tork-driver: herald: #{changed_file}" if $DEBUG
+
+      # find and run the tests that correspond to the changed file
+      Config.test_file_globbers.each do |regexp, globber|
+        if regexp =~ changed_file and glob = globber.call(changed_file)
+          run_test_files Dir[glob]
+        end
+      end
+
+      # reabsorb text execution overhead if overhead files changed
+      if Config.reabsorb_file_greps.any? {|r| r =~ changed_file }
+        @client.puts JSON.dump([:over, changed_file])
+        # NOTE: new thread because reabsorb_overhead_files will kill this one
+        Thread.new { reabsorb_overhead_files }.join
+      end
+    end
+
+    super
+
+    @herald.quit
+    @master.quit
+  end
+
+private
+
+  @waiting_test_files = []
+  @running_test_files = []
+  @passed_test_files = []
+  @failed_test_files = []
+
+  def rerun_running_test_files
+    run_test_files @running_test_files
+  end
+
+  def run_test_files files
+    files.each {|f| run_test_file f }
+  end
+
+  def run_test_file file
+    if File.exist? file and not @waiting_test_files.include? file
+      @waiting_test_files.push file
+      @master.send [:test, file, find_changed_test_names(file)]
+    end
+  end
+
+  @lines_by_file = {}
+
+  def find_changed_test_names test_file
+    # cache the contents of the test file for diffing below
+    new_lines = File.readlines(test_file)
+    old_lines = @lines_by_file[test_file] || new_lines
+    @lines_by_file[test_file] = new_lines
+
+    # find which tests have changed inside the given test file
+    Diff::LCS.diff(old_lines, new_lines).flatten.map do |change|
+      catch :found do
+        # search backwards from the line that changed up to
+        # the first line in the file for test definitions
+        change.position.downto(0) do |i|
+          if test_name = Config.test_name_extractor.call(new_lines[i])
+            throw :found, test_name
+          end
+        end; nil # prevent unsuccessful search from returning an integer
+      end
+    end.compact.uniq
+  end
+
+end
+end

data/lib/tork/master.rb

@@ -0,0 +1,99 @@
+require 'json'
+require 'tork/server'
+require 'tork/config'
+
+module Tork
+module Master
+
+  extend Server
+  extend self
+
+  def load paths, files
+    $LOAD_PATH.unshift(*paths)
+
+    files.each do |file|
+      branch, leaf = File.split(file)
+      file = leaf if paths.include? branch
+      require file.sub(/\.rb$/, '')
+    end
+
+    @client.print @command_line
+  end
+
+  def test test_file, test_names
+    # throttle forking rate to meet the maximum concurrent workers limit
+    sleep 1 until @command_by_worker_pid.size < Config.max_forked_workers
+
+    log_file = test_file + '.log'
+    worker_number = @worker_number_pool.shift
+
+    Config.before_fork_hooks.each do |hook|
+      hook.call worker_number, log_file, test_file, test_names
+    end
+
+    worker_pid = fork do
+      # make the process title Test::Unit friendly and ps(1) searchable
+      $0 = "tork-worker[#{worker_number}] #{test_file}"
+
+      # detach worker process from master process' group for kill -pgrp
+      Process.setsid
+
+      # detach worker process from master process' standard input stream
+      STDIN.reopen IO.pipe.first
+
+      # capture test output in log file because tests are run in parallel
+      # which makes it difficult to understand interleaved output thereof
+      STDERR.reopen(STDOUT.reopen(log_file, 'w')).sync = true
+
+      Config.after_fork_hooks.each do |hook|
+        hook.call worker_number, log_file, test_file, test_names
+      end
+
+      # after loading the user's test file, the at_exit() hook of the user's
+      # testing framework will take care of running the tests and reflecting
+      # any failures in the worker process' exit status, which will then be
+      # handled by the SIGCHLD trap registered in the master process (below)
+      Kernel.load test_file
+    end
+
+    @command_by_worker_pid[worker_pid] = @command.push(worker_number)
+    @client.print @command_line
+  end
+
+  def stop
+    # NOTE: the SIGCHLD handler will reap these killed worker processes
+    Process.kill :SIGTERM, *@command_by_worker_pid.keys.map {|pid| -pid }
+  rescue ArgumentError, SystemCallError
+    # some workers might have already exited before we sent them the signal
+  end
+
+  def loop
+    super
+    stop
+  end
+
+private
+
+  @worker_number_pool = (0 ... Config.max_forked_workers).to_a
+  @command_by_worker_pid = {}
+
+  # process exited child processes and report finished workers to client
+  trap :SIGCHLD do
+    begin
+      while wait2_array = Process.wait2(-1, Process::WNOHANG)
+        child_pid, child_status = wait2_array
+        if command = @command_by_worker_pid.delete(child_pid)
+          @worker_number_pool.push command.pop
+          command[0] = child_status.success? ? 'pass' : 'fail'
+          @client.puts JSON.dump(command.push(child_status))
+        else
+          warn "tork-master: unknown child exited: #{wait2_array.inspect}"
+        end
+      end
+    rescue SystemCallError
+      # raised by wait2() when there are currently no child processes
+    end
+  end
+
+end
+end