vcap_common 1.0.10

data/lib/vcap/subprocess.rb

@@ -0,0 +1,211 @@
+require 'posix-spawn'
+
+module VCAP
+  class SubprocessError < StandardError; end
+
+  # Command exited with unexpected status code
+  class SubprocessStatusError < SubprocessError
+    attr_reader :command, :status, :stdout, :stderr
+
+    def initialize(command, stdout, stderr, status)
+      @command     = command
+      @status      = status
+      @stdout      = stdout
+      @stderr      = stderr
+    end
+
+    def to_s
+      "ERROR: Command '#{@command}' exited with status '#{status.exitstatus}'"
+    end
+  end
+
+  # Command ran longer than allowed
+  class SubprocessTimeoutError < SubprocessError
+    attr_reader :command, :timeout, :stdout, :stderr
+
+    def initialize(timeout, command, stdout, stderr)
+      @command = command
+      @timeout = timeout
+      @stdout  = stdout
+      @stderr  = stderr
+    end
+
+    def to_s
+      "ERROR: Command '#{@command}' timed out"
+    end
+  end
+
+  # Failure reading from stdin/stdout
+  class SubprocessReadError < SubprocessError
+    attr_reader :command, :stdout, :stderr
+
+    def initialize(failed_iostr, command, stdout, stderr)
+      @failed_iostr = failed_iostr
+      @command = command
+      @stdout  = stdout
+      @stderr  = stderr
+    end
+
+    def to_s
+      "ERROR: Failed reading from #{@failed_iostr} while executing '#{@command}'"
+    end
+  end
+
+  # Utility class providing:
+  #   - Ability to capture stdout/stderr of a command
+  #   - Exceptions when commands fail (useful for running a chain of commands)
+  #   - Easier integration with unit tests.
+  class Subprocess
+    READ_SIZE = 4096
+
+    def self.run(*args)
+      VCAP::Subprocess.new.run(*args)
+    end
+
+    # Runs the supplied command in a subshell.
+    #
+    # @param  command               String   The command to be run
+    # @param  expected_exit_status  Integer  The expected exit status of the command in [0, 255]
+    # @param  timeout               Integer  How long the command should be allowed to run for
+    #                                        nil indicates no timeout
+    # @param  options               Hash     Options to be passed to Posix::Spawn
+    #                                        See https://github.com/rtomayko/posix-spawn
+    # @param  env                   Hash     Environment to be passed to Posix::Spawn
+    #                                        See https://github.com/rtomayko/posix-spawn
+    #
+    # @raise  VCAP::SubprocessStatusError    Thrown if the exit status does not match the expected
+    #                                        exit status.
+    # @raise  VCAP::SubprocessTimeoutError   Thrown if a timeout occurs.
+    # @raise  VCAP::SubprocessReadError      Thrown if there is an error reading from any of the pipes
+    #                                        to the child.
+    #
+    # @return Array                          An array of [stdout, stderr, status]. Note that status
+    #                                        is an instance of Process::Status.
+    #
+    def run(command, expected_exit_status=0, timeout=nil, options={}, env={})
+      # We use a pipe to ourself to time out long running commands (if desired) as follows:
+      #   1. Set up a pipe to ourselves
+      #   2. Install a signal handler that writes to one end of our pipe on SIGCHLD
+      #   3. Select on the read end of our pipe and check if our process exited
+      sigchld_r, sigchld_w = IO.pipe
+      prev_sigchld_handler = install_sigchld_handler(sigchld_w)
+
+      start = Time.now.to_i
+      child_pid, stdin, stdout, stderr = POSIX::Spawn.popen4(env, command, options)
+      stdin.close
+
+      # Used to look up the name of an io object when an errors occurs while
+      # reading from it, as well as to look up the corresponding buffer to
+      # append to.
+      io_map = {
+        stderr    => { :name => 'stderr',    :buf => '' },
+        stdout    => { :name => 'stdout',    :buf => '' },
+        sigchld_r => { :name => 'sigchld_r', :buf => '' },
+        sigchld_w => { :name => 'sigchld_w', :buf => '' },
+      }
+
+      status = nil
+      time_left   = timeout
+      read_cands  = [stdout, stderr, sigchld_r]
+      error_cands = read_cands.dup
+
+      begin
+        while read_cands.length > 0
+          active_ios = IO.select(read_cands, nil, error_cands, time_left)
+
+          # Check if timeout was hit
+          if timeout
+            time_left  = timeout - (Time.now.to_i - start)
+            unless active_ios && (time_left > 0)
+              raise VCAP::SubprocessTimeoutError.new(timeout,
+                                                     command,
+                                                     io_map[stdout][:buf],
+                                                     io_map[stderr][:buf])
+            end
+          end
+
+          # Read as much as we can from the readable ios before blocking
+          for io in active_ios[0]
+            begin
+              io_map[io][:buf] << io.read_nonblock(READ_SIZE)
+            rescue IO::WaitReadable
+              # Reading would block, so put ourselves back on the loop
+            rescue EOFError
+              # Pipe has no more data, remove it from the readable/error set
+              # NB: We cannot break from the loop here, as the other pipes may have data to be read
+              read_cands.delete(io)
+              error_cands.delete(io)
+            end
+
+            # Our signal handler notified us that >= 1 children have exited;
+            # check if our child has exited.
+            if (io == sigchld_r) && Process.waitpid(child_pid, Process::WNOHANG)
+              status = $?
+              read_cands.delete(sigchld_r)
+              error_cands.delete(sigchld_r)
+            end
+          end
+
+          # Error reading from one or more pipes.
+          unless active_ios[2].empty?
+            io_names = active_ios[2].map {|io| io_map[io][:name] }
+            raise SubprocessReadError.new(io_names.join(', '),
+                                          command,
+                                          io_map[stdout][:buf],
+                                          io_map[stderr][:buf])
+          end
+        end
+
+      rescue
+        # A timeout or an error occurred while reading from one or more pipes.
+        # Kill the process if we haven't reaped its exit status already.
+        kill_pid(child_pid) unless status
+        raise
+
+      ensure
+        # Make sure we reap the child's exit status, close our fds, and restore
+        # the previous SIGCHLD handler
+        unless status
+          Process.waitpid(child_pid)
+          status = $?
+        end
+        io_map.each_key {|io| io.close unless io.closed? }
+        trap('CLD') { prev_sigchld_handler.call } if prev_sigchld_handler
+      end
+
+      unless status.exitstatus == expected_exit_status
+        raise SubprocessStatusError.new(command,
+                                        io_map[stdout][:buf],
+                                        io_map[stderr][:buf],
+                                        status)
+      end
+
+      [io_map[stdout][:buf], io_map[stderr][:buf], status]
+    end
+
+    private
+
+    def install_sigchld_handler(write_pipe)
+      prev_handler = trap('CLD') do
+        begin
+          # Notify select loop that a child exited. We use a nonblocking write
+          # to avoid writing more than PIPE_BUF bytes before we have the chance
+          # to drain the pipe. Note that we only need to write a single byte
+          # to detect if our child has exited.
+          write_pipe.write_nonblock('x') unless write_pipe.closed?
+        rescue IO::WaitWritable
+        end
+        prev_handler.call if prev_handler
+      end
+      prev_handler
+    end
+
+    def kill_pid(pid)
+      begin
+        Process.kill('KILL', pid)
+      rescue Errno::ESRCH
+      end
+    end
+  end
+
+end

data/lib/vcap/user_pools/user_ops.rb

@@ -0,0 +1,47 @@
+$:.unshift(File.join(File.dirname(__FILE__),'../../common/lib'))
+$:.unshift(File.join(File.dirname(__FILE__)))
+
+require 'logger'
+require 'vcap/subprocess'
+
+module VCAP
+  module UserOps
+    class << self
+      def run(cmd, expected_exit_status = 0)
+        result = VCAP::Subprocess.run(cmd, expected_exit_status)
+      end
+
+      def group_exists?(name)
+        found = false
+        Etc.group { |g| found = true if g.name == name}
+        Etc.endgrent
+        found
+      end
+
+      def install_group(group_name)
+        run("addgroup --system #{group_name}")
+      end
+
+      def remove_group(group_name)
+        run("delgroup --system #{group_name}")
+      end
+
+      def user_exists?(name)
+        found = false
+        Etc.passwd { |u| found = true if u.name == name}
+        Etc.endpwent
+        found
+      end
+
+      def install_user(user_name, group_name)
+        run("adduser --system --quiet --no-create-home --home '/nonexistent' #{user_name}")
+        run("usermod -g #{group_name} #{user_name}")
+      end
+
+      def remove_user(user_name)
+        run("deluser #{user_name}")
+      end
+
+    end
+  end
+end

data/lib/vcap/user_pools/user_pool.rb

@@ -0,0 +1,45 @@
+$:.unshift(File.join(File.dirname(__FILE__),'..'))
+$:.unshift(File.dirname(__FILE__))
+require 'user_pool_util'
+require 'user_ops'
+require 'subprocess'
+
+module VCAP
+  class UserPool
+    attr_accessor :free_users
+    attr_accessor :busy_users
+
+    def initialize(name, logger = nil)
+      @logger = logger ||  Logger.new(STDOUT)
+      UserPoolUtil.init
+      @free_users = UserPoolUtil.open_pool(name)
+      @busy_users = Hash.new
+      @logger.debug("Initialized user pool #{name} with #{@free_users.size} users.")
+    end
+
+    def alloc_user
+      user_name, user = @free_users.shift
+      if user_name != nil
+        @busy_users[user_name] = user
+      else
+        raise "out of users!!"
+      end
+      @logger.debug "alloc()'d user #{user_name}"
+      user
+    end
+
+    def free_user(user)
+      user_name = user[:user_name]
+      if @busy_users.has_key?(user_name)
+        VCAP::Subprocess.run("pkill -9 -u #{user_name}", 1)
+        @busy_users.delete(user_name)
+        @free_users[user_name] = user
+        @logger.debug "free()'d user #{user_name}"
+      else
+        raise "invalid free user: #{user_name}"
+      end
+    end
+  end
+end
+
+

data/lib/vcap/user_pools/user_pool_util.rb

@@ -0,0 +1,107 @@
+$:.unshift(File.dirname(__FILE__))
+
+require 'logger'
+require 'vcap/subprocess'
+require 'user_ops'
+
+module VCAP
+  module UserPoolUtil
+    class << self
+      def init(logger = nil)
+        @logger = logger ||  Logger.new(STDOUT)
+      end
+
+      def user_from_num(name, num)
+        "user-pool-#{name}-#{num}"
+      end
+
+      def group_from_name(name)
+        "user-pool-#{name}"
+      end
+
+      def kill_group_procs(group_name)
+        @logger.debug("killing all procs in group #{group_name}")
+        #XXX -- fixme VCAP::Subprocess.run("pkill -9 -G #{group_name}" , 0)
+      end
+
+      def install_pool(name, size)
+        raise ArgumentError("pool name must not contain dashes") if name =~ /-/
+        group_name = group_from_name(name)
+
+        @logger.info("Creating user pool #{name} with #{size} users.")
+        if VCAP::UserOps.group_exists?(group_name)
+          raise ArgumentError.new("group #{group_name} already exists")
+        end
+        VCAP::UserOps.install_group(group_name)
+        kill_group_procs(group_name)
+
+        begin
+          1.upto(size) do |number|
+            user_name = user_from_num(name, number)
+            if VCAP::UserOps.user_exists?(user_name)
+              VCAP::UserOps.remove_user(user_name)
+              @logger.warn("User reset occured for user #{user_name}!")
+            end
+            @logger.debug("installing user #{user_name}")
+            VCAP::UserOps.install_user(user_name, group_name)
+          end
+        rescue => e
+          @logger.error e.to_s
+          @logger.error("pool creation failed, cleaning up")
+          remove_pool(name)
+        end
+      end
+
+      def remove_pool(name)
+        @logger.info("Removing user pool #{name}.")
+        group_name = group_from_name(name)
+        kill_group_procs(group_name)
+
+        Etc.passwd { |u|
+          if u.name.split('-')[2] == name
+            @logger.debug "removed user #{u.name}"
+            VCAP::UserOps.remove_user(u.name)
+          end
+        }
+        Etc.endpwent
+
+        if VCAP::UserOps.group_exists?(group_name)
+          VCAP::UserOps.remove_group(group_name)
+        else
+          @logger.warn "Pool group #{group_name} missing!!"
+        end
+      end
+
+      def pool_exists?(name)
+        group_name = group_from_name(name)
+        VCAP::UserOps.group_exists?(group_name)
+      end
+
+      def open_pool(name)
+        group_name = group_from_name(name)
+        pool_users = Hash.new
+        unless VCAP::UserOps.group_exists?(group_name)
+          raise ArgumentError.new("no group named #{group_name} exists - can't open pool.")
+        end
+        Etc.passwd { |u|
+          if u.name.split('-')[2] == name
+            pool_users[u.name] = {:user_name => u.name, :uid => u.uid, :gid => u.gid}
+          end
+        }
+        pool_users
+      end
+
+      def pool_list
+        list = []
+        Etc.group { |g|
+          if ['user','pool'] == g.name.split('-')[0..1]
+            list.push(g.name.split('-')[2])
+          end
+        }
+        Etc.endgrent
+        list.map {|name| "#{name} #{open_pool(name).size}"}
+      end
+
+    end
+  end
+end

metadata

@@ -0,0 +1,166 @@
+--- !ruby/object:Gem::Specification
+name: vcap_common
+version: !ruby/object:Gem::Version
+  version: 1.0.10
+  prerelease: 
+platform: ruby
+authors:
+- Derek Collison
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2011-02-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: eventmachine
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.12.11.cloudfoundry.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.12.11.cloudfoundry.3
+- !ruby/object:Gem::Dependency
+  name: thin
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.3.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.3.1
+- !ruby/object:Gem::Dependency
+  name: yajl-ruby
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.8.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.8.3
+- !ruby/object:Gem::Dependency
+  name: nats
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.4.22.beta.8
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.4.22.beta.8
+- !ruby/object:Gem::Dependency
+  name: posix-spawn
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.3.6
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.3.6
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+description: common vcap classes/methods
+email:
+- derek.collison@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/json_message.rb
+- lib/json_schema.rb
+- lib/services/api/async_requests.rb
+- lib/services/api/clients/service_gateway_client.rb
+- lib/services/api/const.rb
+- lib/services/api/messages.rb
+- lib/services/api/util.rb
+- lib/services/api.rb
+- lib/vcap/common.rb
+- lib/vcap/component.rb
+- lib/vcap/config.rb
+- lib/vcap/fiber_tracing.rb
+- lib/vcap/json_schema.rb
+- lib/vcap/priority_queue.rb
+- lib/vcap/process_utils.rb
+- lib/vcap/quota.rb
+- lib/vcap/rolling_metric.rb
+- lib/vcap/spec/em.rb
+- lib/vcap/spec/forked_component/base.rb
+- lib/vcap/spec/forked_component/nats_server.rb
+- lib/vcap/spec/forked_component.rb
+- lib/vcap/subprocess.rb
+- lib/vcap/user_pools/user_ops.rb
+- lib/vcap/user_pools/user_pool.rb
+- lib/vcap/user_pools/user_pool_util.rb
+homepage: http://github.com/vmware-ac/core
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.22
+signing_key: 
+specification_version: 3
+summary: vcap common
+test_files: []