omnibus 1.0.0

data/lib/omnibus/fetchers/path_fetcher.rb

@@ -0,0 +1,65 @@
+#
+# Copyright:: Copyright (c) 2012 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module Omnibus
+
+  # Fetcher implementation for projects on the filesystem
+  class PathFetcher < Fetcher
+
+    name :path
+
+    def initialize(software)
+      @name = software.name
+      @source = software.source
+      @project_dir = software.project_dir
+      @version = software.version
+    end
+
+    def description
+      s=<<-E
+source path:    #{@source[:path]}
+local location: #{@project_dir}
+E
+    end
+    
+    def rsync
+      if OHAI.platform == "windows"
+        # Robocopy's return code is 1 if it succesfully copies over the
+        # files and 0 if the files are already existing at the destination
+        sync_cmd = "robocopy #{@source[:path]}\\ #{@project_dir}\\ /MIR /S"
+        shell = Mixlib::ShellOut.new(sync_cmd, :returns => [0, 1])
+      else
+        sync_cmd = "rsync --delete -a #{@source[:path]}/ #{@project_dir}/"
+        shell = Mixlib::ShellOut.new(sync_cmd)
+      end
+      shell.run_command
+    end
+
+    def clean
+      # Here, clean will do the same as fetch: reset source to pristine state
+      rsync
+    end
+
+    def fetch
+      rsync
+    end
+
+    def fetch_required?
+      true
+    end
+  end
+end

data/lib/omnibus/fetchers/s3_cache_fetcher.rb

@@ -0,0 +1,42 @@
+#
+# Copyright:: Copyright (c) 2012 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'omnibus/fetcher'
+require 'omnibus/s3_cacher'
+
+module Omnibus
+  class S3CacheFetcher < NetFetcher
+    include SoftwareS3URLs
+
+    name :s3cache
+
+    def initialize(software)
+      @software = software
+      super
+    end
+
+    def fetch
+      log "S3 Cache enabled, #{name} will be fetched from S3 cache"
+      super
+    end
+
+    def source_uri
+      URI.parse(url_for(@software))
+    end
+  end
+end
+

data/lib/omnibus/health_check.rb

@@ -0,0 +1,260 @@
+#
+# Copyright:: Copyright (c) 2012 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module Omnibus
+  class HealthCheck
+
+    WHITELIST_LIBS = [
+                      /ld-linux/,
+                      /libc\.so/,
+                      /libcrypt\.so/,
+                      /libdl/,
+                      /libfreebl\d\.so/,
+                      /libgcc_s\.so/,
+                      /libm\.so/,
+                      /libnsl\.so/,
+                      /libpthread/,
+                      /libresolv\.so/,
+                      /librt\.so/,
+                      /libstdc\+\+\.so/,
+                      /libutil\.so/,
+                      /linux-vdso.+/,
+                      /linux-gate\.so/,
+                     ]
+
+    SOLARIS_WHITELIST_LIBS = [
+                              /libaio\.so/,
+                              /libavl\.so/,
+                              /libcrypt_[di]\.so/,
+                              /libcrypto.so/,
+                              /libcurses\.so/,
+                              /libdoor\.so/,
+                              /libgen\.so/,
+                              /libmd5\.so/,
+                              /libmd\.so/,
+                              /libmp\.so/,
+                              /libscf\.so/,
+                              /libsec\.so/,
+                              /libsocket\.so/,
+                              /libssl.so/,
+                              /libthread.so/,
+                              /libuutil\.so/,
+                              /libz.so/,
+                              # solaris 11 libraries:
+                              /libc\.so\.1/,
+                              /libm\.so\.2/,
+                              /libdl\.so\.1/,
+                              /libnsl\.so\.1/,
+                              /libpthread\.so\.1/,
+                              /librt\.so\.1/,
+                              /libcrypt\.so\.1/,
+                              /libgdbm\.so\.3/,
+                              # solaris 9 libraries:
+                              /libm\.so\.1/,
+                              /libc_psr\.so\.1/,
+                              /s9_preload\.so\.1/,
+                             ]
+
+    MAC_WHITELIST_LIBS = [
+                          /libobjc\.A\.dylib/,
+                          /libSystem\.B\.dylib/,
+                          /CoreFoundation/,
+                          /Tcl$/,
+                          /Cocoa$/,
+                          /Carbon$/,
+                          /IOKit$/,
+                          /Tk$/,
+                          /libutil\.dylib/,
+                          /libffi\.dylib/,
+                          /libncurses\.5\.4\.dylib/,
+                          /libiconv/
+                         ]
+
+    WHITELIST_FILES = [
+                       /jre\/bin\/javaws/,
+                       /jre\/bin\/policytool/,
+                       /jre\/lib/,
+                       /jre\/plugin/,
+                      ]
+
+    def self.log(msg)
+      puts "[health_check] #{msg}"
+    end
+
+    def self.run(install_dir)
+      if OHAI.platform == "mac_os_x"
+        bad_libs = health_check_otool(install_dir)
+      else
+        bad_libs = health_check_ldd(install_dir)
+      end
+
+      unresolved = []
+      unreliable = []
+      detail = []
+
+      if bad_libs.keys.length > 0
+        bad_libs.each do |name, lib_hash|
+          lib_hash.each do |lib, linked_libs|
+            linked_libs.each do |linked, count|
+              if linked =~ /not found/
+                unresolved << lib unless unresolved.include? lib
+              else
+                unreliable << linked unless unreliable.include? linked
+              end
+              detail << "#{name}|#{lib}|#{linked}|#{count}"
+            end
+          end
+        end
+        log "*** Health Check Failed, Summary follows:"
+        bad_omnibus_libs, bad_omnibus_bins = bad_libs.keys.partition { |k| k.include? "embedded/lib" }
+        log "*** The following Omnibus-built libraries have unsafe or unmet dependencies:"
+        bad_omnibus_libs.each { |lib| log "    --> #{lib}" }
+        log "*** The following Omnibus-built binaries have unsafe or unmet dependencies:"
+        bad_omnibus_bins.each { |bin| log "    --> #{bin}" }
+        if unresolved.length > 0
+          log "*** The following requirements could not be resolved:"
+          unresolved.each { |lib| log "    --> #{lib}"}
+        end
+        if unreliable.length > 0
+          log "*** The following libraries cannot be guaranteed to be on target systems:"
+          unreliable.each { |lib| log "    --> #{lib}"}
+        end
+        log "*** The precise failures were:"
+        detail.each do |line|
+          item, dependency, location, count = line.split('|')
+          reason = location =~ /not found/ ? "Unresolved dependency" : "Unsafe dependency"
+          log "    --> #{item}"
+          log "    DEPENDS ON: #{dependency}"
+          log "      COUNT: #{count}"
+          log "      PROVIDED BY: #{location}"
+          log "      FAILED BECAUSE: #{reason}"
+        end
+	raise "Health Check Failed"
+      end
+    end
+
+    def self.health_check_otool(install_dir)
+      otool_cmd = "find #{install_dir}/ -type f | egrep '\.(dylib|bundle)$' | xargs otool -L > otool.out 2>/dev/null"
+      log "Executing `#{otool_cmd}`"
+      shell = Mixlib::ShellOut.new(otool_cmd, :timeout => 3600)
+      shell.run_command
+
+      otool_output = File.read('otool.out')
+
+      current_library = nil
+      bad_libs = {}
+
+      otool_output.each_line do |line|
+        case line
+        when /^(.+):$/
+          current_library = $1
+        when /^\s+(.+) \(.+\)$/
+          linked = $1
+          name = File.basename(linked)
+          bad_libs = check_for_bad_library(install_dir, bad_libs, current_library, name, linked)
+        end
+      end
+
+      File.delete('otool.out')
+
+      bad_libs
+    end
+
+    def self.check_for_bad_library(install_dir, bad_libs, current_library, name, linked)
+      safe = nil
+
+      whitelist_libs = case OHAI.platform
+                       when 'mac_os_x'
+                         MAC_WHITELIST_LIBS
+                       when 'solaris2'
+                         SOLARIS_WHITELIST_LIBS
+                       else
+                         WHITELIST_LIBS
+                       end
+      whitelist_libs.each do |reg|
+        safe ||= true if reg.match(name)
+      end
+      WHITELIST_FILES.each do |reg|
+        safe ||= true if reg.match(current_library)
+      end
+
+      log "  --> Dependency: #{name}" if ARGV[0] == "verbose"
+      log "  --> Provided by: #{linked}" if ARGV[0] == "verbose"
+
+      if !safe && linked !~ Regexp.new(install_dir)
+        log "    -> FAILED: #{current_library} has unsafe dependencies" if ARGV[0] == "verbose"
+        bad_libs[current_library] ||= {}
+        bad_libs[current_library][name] ||= {}
+        if bad_libs[current_library][name].has_key?(linked)
+          bad_libs[current_library][name][linked] += 1
+        else
+          bad_libs[current_library][name][linked] = 1
+        end
+      else
+        log "    -> PASSED: #{name} is either whitelisted or safely provided." if ARGV[0] == "verbose"
+      end
+
+      bad_libs
+    end
+
+    def self.health_check_ldd(install_dir)
+      #
+      # ShellOut has GC turned off during execution, so when we're
+      # executing extremely long commands with lots of output, we
+      # should be mindful that the string concatentation for building
+      # #stdout will hurt memory usage drastically
+      #
+      ldd_cmd = "find #{install_dir}/ -type f | xargs ldd > ldd.out 2>/dev/null"
+      log "Executing `#{ldd_cmd}`"
+      shell = Mixlib::ShellOut.new(ldd_cmd, :timeout => 3600)
+      shell.run_command
+
+      ldd_output = File.read('ldd.out')
+
+      current_library = nil
+      bad_libs = {}
+
+      ldd_output.each_line do |line|
+        case line
+        when /^(.+):$/
+          current_library = $1
+	  log "*** Analysing dependencies for #{current_library}" if ARGV[0] == "verbose"
+        when /^\s+(.+) \=\>\s+(.+)( \(.+\))?$/
+          name = $1
+          linked = $2
+          bad_libs = check_for_bad_library(install_dir, bad_libs, current_library, name, linked)
+        when /^\s+(.+) \(.+\)$/
+          next
+        when /^\s+statically linked$/
+          next
+        when /^\s+libjvm.so/
+          next
+        when /^\s+libjava.so/
+          next
+        when /^\s+libmawt.so/
+          next
+        when /^\s+not a dynamic executable$/ # ignore non-executable files
+        else
+          log "*** Line did not match for #{current_library}\n#{line}"
+        end
+      end
+
+      File.delete('ldd.out')
+      bad_libs
+    end
+  end
+end

data/lib/omnibus/library.rb

@@ -0,0 +1,70 @@
+#
+# Copyright:: Copyright (c) 2012 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module Omnibus
+  #
+  # Used to generate the manifest of all software components with versions
+  class Library
+
+    def initialize
+      @projects = []
+      @components = []
+    end
+
+    def component_added(component)
+      @components << component
+    end
+
+    def version_map(project)
+      @components.select {|c| c.project == project}.inject({}) {|map, component|
+        map[component.name] = if component.given_version
+                                {:version       => component.version,
+                                 :given_version => component.given_version,
+                                 :overridden    => component.overridden?,
+                                 :version_guid  => component.version_guid}
+                              else
+                                ## Components without a version are
+                                ## pieces of the omnibus project
+                                ## itself, and so don't really fit
+                                ## with the concept of overrides
+                                v = {:version => project.build_version}
+                                if project.build_version.respond_to?(:git_sha)
+                                  v[:version_guid] = "git:#{project.build_version.git_sha}"
+                                end
+                                v
+                              end
+        map
+      }
+    end
+
+    def select(*args, &block)
+      @components.select(*args, &block)
+    end
+
+
+  end
+
+  def self.library
+    @library ||= Library.new
+  end
+
+  def self.component_added(*args)
+    library.component_added(*args)
+  end
+
+end
+

data/lib/omnibus/overrides.rb

@@ -0,0 +1,69 @@
+require 'pp'
+
+module Omnibus
+  module Overrides
+
+    DEFAULT_OVERRIDE_FILE_NAME = "omnibus.overrides"
+
+    # Parses a file of override information into a Hash.
+    #
+    # Each line of the file must be of the form
+    #
+    #
+    #     <package_name> <version>
+    #
+    # where the two pieces of data are separated by whitespace.
+    #
+    # @param file [String] the path to an overrides file
+    # @return [Hash, nil]
+    def self.parse_file(file)
+      if file
+        File.readlines(file).inject({}) do |acc, line|
+          info = line.split
+
+          unless info.count == 2
+            raise ArgumentError, "Invalid overrides line: '#{line.chomp}'"
+          end
+
+          package, version = info
+
+          if acc[package]
+            raise ArgumentError, "Multiple overrides present for '#{package}' in overrides file #{file}!"
+          end
+
+          acc[package] = version
+          acc
+        end
+      else
+        nil
+      end
+    end
+
+    # Return the full path to an overrides file, or +nil+ if no such
+    # file exists.
+    def self.resolve_override_file
+      file = ENV['OMNIBUS_OVERRIDE_FILE'] || DEFAULT_OVERRIDE_FILE_NAME
+      path = File.expand_path(file)
+      File.exist?(path) ? path : nil
+    end
+
+    # Return a hash of override information.  If no such information
+    # can be found, the hash will be empty
+    #
+    # @return [Hash]
+    def self.overrides
+      file = resolve_override_file
+      overrides = parse_file(file)
+
+      if overrides
+        puts "********************************************************************************"
+        puts "Using Overrides from #{Omnibus::Overrides.resolve_override_file}"
+        pp overrides
+        puts "********************************************************************************"
+      end
+
+      overrides || {}
+    end
+
+  end
+end