foreman 0.47.0 → 0.50.0

data/lib/foreman/engine.rb

@@ -1,8 +1,7 @@
 require "foreman"
-require "foreman/color"
+require "foreman/env"
 require "foreman/process"
 require "foreman/procfile"
-require "foreman/utils"
 require "tempfile"
 require "timeout"
 require "fileutils"
@@ -10,219 +9,305 @@ require "thread"
 
 class Foreman::Engine
 
-  attr_reader :environment
-  attr_reader :procfile
-  attr_reader :directory
+  attr_reader :env
   attr_reader :options
-
-  COLORS = %w( cyan yellow green magenta red blue intense_cyan intense_yellow
-               intense_green intense_magenta intense_red, intense_blue )
-
-  Foreman::Color.enable($stdout)
-
-  def initialize(procfile, options={})
-    @procfile  = Foreman::Procfile.new(procfile) if File.exists?(procfile)
-    @directory = options[:app_root] || File.expand_path(File.dirname(procfile))
+  attr_reader :processes
+
+  # Create an +Engine+ for running processes
+  #
+  # @param [Hash] options
+  #
+  # @option options [String] :formation (all=1)    The process formation to use
+  # @option options [Fixnum] :port      (5000)     The base port to assign to processes
+  # @option options [String] :root      (Dir.pwd)  The root directory from which to run processes
+  #
+  def initialize(options={})
     @options = options.dup
-    @output_mutex = Mutex.new
 
-    @options[:env] ||= default_env
-    @environment = read_environment_files(@options[:env])
+    @options[:formation] ||= (options[:concurrency] || "all=1")
+
+    @env       = {}
+    @mutex     = Mutex.new
+    @names     = {}
+    @processes = []
+    @running   = {}
+    @readers   = {}
   end
 
+  # Start the processes registered to this +Engine+
+  #
   def start
-    proctitle "ruby: foreman master"
-    termtitle "#{File.basename(@directory)} - foreman"
-
     trap("TERM") { puts "SIGTERM received"; terminate_gracefully }
     trap("INT")  { puts "SIGINT received";  terminate_gracefully }
-    trap("HUP")  { puts "SIGHUP received";  terminate_gracefully }
+    trap("HUP")  { puts "SIGHUP received";  terminate_gracefully } if ::Signal.list.keys.include? 'HUP'
 
-    assign_colors
+    startup
     spawn_processes
     watch_for_output
-    watch_for_termination
+    sleep 0.1
+    watch_for_termination { terminate_gracefully }
+    shutdown
   end
 
-  def port_for(process, num, base_port=nil)
-    base_port ||= 5000
-    offset = procfile.process_names.index(process.name) * 100
-    base_port.to_i + offset + num - 1
+  # Register a process to be run by this +Engine+
+  #
+  # @param [String] name     A name for this process
+  # @param [String] command  The command to run
+  # @param [Hash]   options
+  #
+  # @option options [Hash] :env  A custom environment for this process
+  #
+  def register(name, command, options={})
+    options[:env] ||= env
+    options[:cwd] ||= File.dirname(command.split(" ").first)
+    process = Foreman::Process.new(command, options)
+    @names[process] = name
+    @processes << process
   end
 
-  def apply_environment!
-    environment.each { |k,v| ENV[k] = v }
+  # Clear the processes registered to this +Engine+
+  #
+  def clear
+    @names     = {}
+    @processes = []
   end
 
-  def self.read_environment(filename)
-    return {} unless File.exists?(filename)
+  # Register processes by reading a Procfile
+  #
+  # @param [String] filename  A Procfile from which to read processes to register
+  #
+  def load_procfile(filename)
+    options[:root] ||= File.dirname(filename)
+    Foreman::Procfile.new(filename).entries do |name, command|
+      register name, command, :cwd => options[:root]
+    end
+    self
+  end
+
+  # Load a .env file into the +env+ for this +Engine+
+  #
+  # @param [String] filename  A .env file to load into the environment
+  #
+  def load_env(filename)
+    Foreman::Env.new(filename).entries do |name, value|
+      @env[name] = value
+    end
+  end
 
-    File.read(filename).split("\n").inject({}) do |hash, line|
-      if line =~ /\A([A-Za-z_0-9]+)=(.*)\z/
-        key, val = [$1, $2]
-        case val
-          when /\A'(.*)'\z/ then hash[key] = $1
-          when /\A"(.*)"\z/ then hash[key] = $1.gsub(/\\(.)/, '\1')
-          else hash[key] = val
+  # Send a signal to all processesstarted by this +Engine+
+  #
+  # @param [String] signal  The signal to send to each process
+  #
+  def killall(signal="SIGTERM")
+    if Foreman.windows?
+      @running.each do |pid, (process, index)|
+        system "sending #{signal} to #{name_for(pid)} at pid #{pid}"
+        begin
+          Process.kill(signal, pid)
+        rescue Errno::ESRCH, Errno::EPERM
         end
       end
-      hash
+    else
+      begin
+        Process.kill "-#{signal}", Process.pid
+      rescue Errno::ESRCH, Errno::EPERM
+      end
     end
   end
 
-private ######################################################################
-
-  def spawn_processes
-    concurrency = Foreman::Utils.parse_concurrency(@options[:concurrency])
+  # Get the process formation
+  #
+  # @returns [Fixnum]  The formation count for the specified process
+  #
+  def formation
+    @formation ||= parse_formation(options[:formation])
+  end
 
-    procfile.entries.each do |entry|
-      reader, writer = (IO.method(:pipe).arity == 0 ? IO.pipe : IO.pipe("BINARY"))
-      entry.spawn(concurrency[entry.name], writer, @directory, @environment, port_for(entry, 1, base_port)).each do |process|
-        running_processes[process.pid] = process
-        readers[process] = reader
-      end
-    end
+  # List the available process names
+  #
+  # @returns [Array]  A list of process names
+  #
+  def process_names
+    @processes.map { |p| @names[p] }
   end
 
-  def base_port
-    options[:port] || environment["PORT"] || ENV["PORT"] || 5000
+  # Get the +Process+ for a specifid name
+  #
+  # @param [String] name  The process name
+  #
+  # @returns [Foreman::Process]  The +Process+ for the specified name
+  #
+  def process(name)
+    @names.invert[name]
   end
 
-  def kill_all(signal="SIGTERM")
-    running_processes.each do |pid, process|
-      info "sending #{signal} to pid #{pid}"
-      process.kill signal
+  # Yield each +Process+ in order
+  #
+  def each_process
+    process_names.each do |name|
+      yield name, process(name)
     end
   end
 
-  def terminate_gracefully
-    return if @terminating
-    @terminating = true
-    info "sending SIGTERM to all processes"
-    kill_all "SIGTERM"
-    Timeout.timeout(5) do
-      while running_processes.length > 0
-        pid, status = Process.wait2
-        process = running_processes.delete(pid)
-        info "process terminated", process.name
-      end
-    end
-  rescue Timeout::Error
-    info "sending SIGKILL to all processes"
-    kill_all "SIGKILL"
-  end
-
-  def poll_readers
-    rs, ws = IO.select(readers.values, [], [], 1)
-    (rs || []).each do |r|
-      data = r.gets
-      next unless data
-      data.force_encoding("BINARY") if data.respond_to?(:force_encoding)
-      ps, message = data.split(",", 2)
-      color = colors[ps.split(".").first]
-      info message, ps, color
-    end
+  # Get the root directory for this +Engine+
+  #
+  # @returns [String]  The root directory
+  #
+  def root
+    File.expand_path(options[:root] || Dir.pwd)
   end
 
-  def watch_for_output
-    Thread.new do
-      require "win32console" if Foreman.windows?
-      begin
-        loop do
-          poll_readers
-        end
-      rescue Exception => ex
-        puts ex.message
-        puts ex.backtrace
-      end
+  # Get the port for a given process and offset
+  #
+  # @param [Foreman::Process] process   A +Process+ associated with this engine
+  # @param [Fixnum]           instance  The instance of the process
+  #
+  # @returns [Fixnum] port  The port to use for this instance of this process
+  #
+  def port_for(process, instance, base=nil)
+    if base
+      base + (@processes.index(process.process) * 100) + (instance - 1)
+    else
+      base_port + (@processes.index(process) * 100) + (instance - 1)
     end
   end
 
-  def watch_for_termination
-    pid, status = Process.wait2
-    process = running_processes.delete(pid)
-    info "process terminated", process.name
-    terminate_gracefully
-  rescue Errno::ECHILD
+  # Get the base port for this foreman instance
+  #
+  # @returns [Fixnum] port  The base port
+  #
+  def base_port
+    (options[:port] || env["PORT"] || ENV["PORT"] || 5000).to_i
   end
 
-  def info(message, name="system", color=:white)
-    output  = ""
-    output += $stdout.color(color)
-    output += "#{Time.now.strftime("%H:%M:%S")} #{pad_process_name(name)} | "
-    output += $stdout.color(:reset)
-    output += message.chomp
-    puts output
+  # deprecated
+  def environment
+    env
   end
 
-  def print(message=nil)
-    @output_mutex.synchronize do
-      $stdout.print message
-    end
-  end
+private
 
-  def puts(message=nil)
-    @output_mutex.synchronize do
-      $stdout.puts message
-    end
+### Engine API ######################################################
+
+  def startup
+    raise TypeError, "must use a subclass of Foreman::Engine"
   end
 
-  def longest_process_name
-    @longest_process_name ||= begin
-      longest = procfile.process_names.map { |name| name.length }.sort.last
-      longest = 6 if longest < 6 # system
-      longest
-    end
+  def output(name, data)
+    raise TypeError, "must use a subclass of Foreman::Engine"
   end
 
-  def pad_process_name(name="system")
-    name.to_s.ljust(longest_process_name + 3) # add 3 for process number padding
+  def shutdown
+    raise TypeError, "must use a subclass of Foreman::Engine"
   end
 
-  def proctitle(title)
-    $0 = title
+## Helpers ##########################################################
+
+  def create_pipe
+    IO.method(:pipe).arity.zero? ? IO.pipe : IO.pipe("BINARY")
   end
 
-  def termtitle(title)
-    printf("\033]0;#{title}\007") unless Foreman.windows?
+  def name_for(pid)
+    process, index = @running[pid]
+    [ @names[process], index.to_s ].compact.join(".")
   end
 
-  def running_processes
-    @running_processes ||= {}
+  def parse_formation(formation)
+    pairs = formation.to_s.gsub(/\s/, "").split(",")
+
+    pairs.inject(Hash.new(0)) do |ax, pair|
+      process, amount = pair.split("=")
+      process == "all" ? ax.default = amount.to_i : ax[process] = amount.to_i
+      ax
+    end
   end
 
-  def readers
-    @readers ||= {}
+  def output_with_mutex(name, message)
+    @mutex.synchronize do
+      output name, message
+    end
   end
 
-  def colors
-    @colors ||= {}
+  def system(message)
+    output_with_mutex "system", message
   end
 
-  def assign_colors
-    procfile.entries.each_with_index do |entry, idx|
-      colors[entry.name] = COLORS[idx % COLORS.length]
+  def termination_message_for(status)
+    if status.exited?
+      "exited with code #{status.exitstatus}"
+    elsif status.signaled?
+      "terminated by SIG#{Signal.list.invert[status.termsig]}"
+    else
+      "died a mysterious death"
     end
   end
 
-  def process_by_reader(reader)
-    readers.invert[reader]
+  def flush_reader(reader)
+    until reader.eof?
+      data = reader.gets
+      output_with_mutex name_for(@readers.key(reader)), data
+    end
   end
 
-  def read_environment_files(filenames)
-    environment = {}
+## Engine ###########################################################
 
-    (filenames || "").split(",").map(&:strip).each do |filename|
-      error "No such file: #{filename}" unless File.exists?(filename)
-      environment.merge!(Foreman::Engine.read_environment(filename))
+  def spawn_processes
+    @processes.each do |process|
+      1.upto(formation[@names[process]]) do |n|
+        reader, writer = create_pipe
+        begin
+          pid = process.run(:output => writer, :env => { "PORT" => port_for(process, n).to_s })
+          writer.puts "started with pid #{pid}"
+        rescue Errno::ENOENT
+          writer.puts "unknown command: #{process.command}"
+        end
+        @running[pid] = [process, n]
+        @readers[pid] = reader
+      end
     end
+  end
 
-    environment
+  def watch_for_output
+    Thread.new do
+      begin
+        loop do
+          (IO.select(@readers.values).first || []).each do |reader|
+            data = reader.gets
+            output_with_mutex name_for(@readers.invert[reader]), data
+          end
+        end
+      rescue Exception => ex
+        puts ex.message
+        puts ex.backtrace
+      end
+    end
+  end
+
+  def watch_for_termination
+    pid, status = Process.wait2
+    output_with_mutex name_for(pid), termination_message_for(status)
+    @running.delete(pid)
+    yield if block_given?
+    pid
+  rescue Errno::ECHILD
   end
 
-  def default_env
-    env = File.join(directory, ".env")
-    File.exists?(env) ? env : ""
+  def terminate_gracefully
+    return if @terminating
+    @terminating = true
+    if Foreman.windows?
+      system  "sending SIGKILL to all processes"
+      killall "SIGKILL"
+    else
+      system  "sending SIGTERM to all processes"
+      killall "SIGTERM"
+    end
+    Timeout.timeout(5) do
+      watch_for_termination while @running.length > 0
+    end
+  rescue Timeout::Error
+    system  "sending SIGKILL to all processes"
+    killall "SIGKILL"
   end
 
 end

data/lib/foreman/env.rb

@@ -0,0 +1,27 @@
+require "foreman"
+
+class Foreman::Env
+
+  attr_reader :entries
+
+  def initialize(filename)
+    @entries = File.read(filename).split("\n").inject({}) do |ax, line|
+      if line =~ /\A([A-Za-z_0-9]+)=(.*)\z/
+        key = $1
+        case val = $2
+          when /\A'(.*)'\z/ then ax[key] = $1
+          when /\A"(.*)"\z/ then ax[key] = $1.gsub(/\\(.)/, '\1')
+          else ax[key] = val
+        end
+      end
+      ax
+    end
+  end
+
+  def entries
+    @entries.each do |key, value|
+      yield key, value
+    end
+  end
+
+end

data/lib/foreman/export/base.rb

@@ -1,27 +1,81 @@
 require "foreman/export"
-require "foreman/utils"
+require "ostruct"
+require "pathname"
+require "shellwords"
 
 class Foreman::Export::Base
 
-  attr_reader :location, :engine, :app, :log, :port, :user, :template, :concurrency
+  attr_reader :location
+  attr_reader :engine
+  attr_reader :options
+  attr_reader :formation
+
+  # deprecated
+  attr_reader :port
 
   def initialize(location, engine, options={})
-    @location    = location
-    @engine      = engine
-    @app         = options[:app]
-    @log         = options[:log]
-    @port        = options[:port]
-    @user        = options[:user]
-    @template    = options[:template]
-    @concurrency = Foreman::Utils.parse_concurrency(options[:concurrency])
+    @location  = location
+    @engine    = engine
+    @options   = options.dup
+    @formation = engine.formation
+
+    # deprecated
+    def port
+      Foreman::Export::Base.warn_deprecation!
+      engine.base_port
+    end
+
+    # deprecated
+    def template
+      Foreman::Export::Base.warn_deprecation!
+      options[:template]
+    end
+
+    # deprecated
+    def @engine.procfile
+      Foreman::Export::Base.warn_deprecation!
+      @processes.map do |process|
+        OpenStruct.new(
+          :name => @names[process],
+          :process => process
+        )
+      end
+    end
   end
 
   def export
-    raise "export method must be overridden"
+    error("Must specify a location") unless location
+    FileUtils.mkdir_p(location) rescue error("Could not create: #{location}")
+    FileUtils.mkdir_p(log) rescue error("Could not create: #{log}")
+    FileUtils.chown(user, nil, log) rescue error("Could not chown #{log} to #{user}")
+  end
+
+  def app
+    options[:app] || "app"
+  end
+
+  def log
+    options[:log] || "/var/log/#{app}"
+  end
+
+  def user
+    options[:user] || app
   end
 
 private ######################################################################
 
+  def self.warn_deprecation!
+    @@deprecation_warned ||= false
+    return if @@deprecation_warned
+    puts "WARNING: Using deprecated exporter interface. Please update your exporter"
+    puts "the interface shown in the upstart exporter:"
+    puts
+    puts "https://github.com/ddollar/foreman/blob/master/lib/foreman/export/upstart.rb"
+    puts "https://github.com/ddollar/foreman/blob/master/data/export/upstart/process.conf.erb"
+    puts
+    @@deprecation_warned = true
+  end
+
   def error(message)
     raise Foreman::Export::Exception.new(message)
   end
@@ -29,8 +83,19 @@ private ######################################################################
   def say(message)
     puts "[foreman export] %s" % message
   end
+  
+  def clean(filename)
+    return unless File.exists?(filename)
+    say "cleaning up: #{filename}"
+    FileUtils.rm(filename)
+  end
 
-  def export_template(exporter, file, template_root)
+  def shell_quote(value)
+    '"' + Shellwords.escape(value) + '"'
+  end
+
+  # deprecated
+  def old_export_template(exporter, file, template_root)
     if template_root && File.exist?(file_path = File.join(template_root, file))
       File.read(file_path)
     elsif File.exist?(file_path = File.expand_path(File.join("~/.foreman/templates", file)))
@@ -40,27 +105,42 @@ private ######################################################################
     end
   end
 
+  def export_template(name, file=nil, template_root=nil)
+    if file && template_root
+      old_export_template name, file, template_root
+    else
+      name_without_first = name.split("/")[1..-1].join("/")
+      matchers = []
+      matchers << File.join(options[:template], name_without_first) if options[:template]
+      matchers << File.expand_path("~/.foreman/templates/#{name}")
+      matchers << File.expand_path("../../../../data/export/#{name}", __FILE__)
+      File.read(matchers.detect { |m| File.exists?(m) })
+    end
+  end
+
+  def write_template(name, target, binding)
+    compiled = ERB.new(export_template(name)).result(binding)
+    write_file target, compiled
+  end
+
+  def chmod(mode, file)
+    say "setting #{file} to mode #{mode}"
+    FileUtils.chmod mode, File.join(location, file)
+  end
+
+  def create_directory(dir)
+    say "creating: #{dir}"
+    FileUtils.mkdir_p(File.join(location, dir))
+  end
+
   def write_file(filename, contents)
     say "writing: #{filename}"
 
+    filename = File.join(location, filename) unless Pathname.new(filename).absolute?
+
     File.open(filename, "w") do |file|
       file.puts contents
     end
   end
 
-  # Quote a string to be used on the command line. Backslashes are escapde to \\ and quotes 
-  # escaped to \"
-  #
-  #   str - string to be quoted
-  #
-  # Examples
-  # 
-  #  shell_quote("FB|123\"\\1")
-  #  # => "\"FB|123\"\\"\\\\1\""
-  #
-  # Returns the the escaped string surrounded by quotes 
-  def shell_quote(str)
-    "\"#{str.gsub(/\\/){ '\\\\' }.gsub(/["]/){ "\\\"" }}\""
-  end
-
 end

data/lib/foreman/export/bluepill.rb

@@ -4,23 +4,9 @@ require "foreman/export"
 class Foreman::Export::Bluepill < Foreman::Export::Base
 
   def export
-    error("Must specify a location") unless location
-
-    FileUtils.mkdir_p location
-
-    app = self.app || File.basename(engine.directory)
-    user = self.user || app
-    log_root = self.log || "/var/log/#{app}"
-    template_root = self.template
-
-    Dir["#{location}/#{app}.pill"].each do |file|
-      say "cleaning up: #{file}"
-      FileUtils.rm(file)
-    end
-
-    master_template = export_template("bluepill", "master.pill.erb", template_root)
-    master_config   = ERB.new(master_template).result(binding)
-    write_file "#{location}/#{app}.pill", master_config
+    super
+    clean "#{location}/#{app}.pill"
+    write_template "bluepill/master.pill.erb", "#{app}.pill", binding
   end
 
 end