omnibus 1.0.0

data/lib/omnibus/reports.rb

@@ -0,0 +1,99 @@
+#
+# Copyright:: Copyright (c) 2012 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module Omnibus
+
+  module Reports
+    extend self
+
+
+    PADDING = 3
+ 
+    # Determine how wide a column should be, taking into account both
+    # the column name as well as all data in that column.  If no data
+    # will be stored in the column, the width is 0 (i.e., nothing
+    # should be printed, not even the column header)
+    def column_width(items, column_name)
+      widest_item = items.max{|a,b| a.size <=> b.size}
+      if widest_item
+        widest = (widest_item.size >= column_name.size) ? widest_item : column_name
+        widest.size + PADDING
+      else
+        0
+      end
+    end
+
+    def non_nil_values(hashes, selector_key)
+      hashes.map{|v| v[selector_key]}.compact
+    end
+
+    def pretty_version_map(project)
+      out = ""
+      version_map = Omnibus.library.version_map(project)
+
+      
+      # Pull out data to print out
+      versions = non_nil_values(version_map.values, :version)
+      guids = non_nil_values(version_map.values, :version_guid)
+
+      # We only want the versions that have truly been overridden;
+      # because we want to output a column only if something was
+      # overridden, but nothing if no packages were changed
+      overridden_versions = non_nil_values(version_map.values.select{|v| v[:overridden]},
+                                           :given_version)
+  
+
+      # Determine how wide the printed table columns need to be
+      name_width = column_width(version_map.keys, "Component")
+      version_width = column_width(versions, "Installed Version")
+      guid_width = column_width(guids, "Version GUID")
+      override_width = column_width(overridden_versions, "Overridden From")
+
+      total_width = name_width + version_width + guid_width + override_width
+      divider = "-" * total_width
+      
+      # Print out the column headers
+      out << "Component".ljust(name_width)
+      out << "Installed Version".ljust(version_width)
+      out << "Version GUID".ljust(guid_width)
+      # Only print out column if something was overridden
+      out << "Overridden From".ljust(override_width) if override_width > 0
+      out << "\n"
+      out << divider << "\n"
+
+      # Print out the table body
+      version_map.keys.sort.each do |name|
+        version = version_map[name][:version]
+        version_guid = version_map[name][:version_guid]
+
+        given_version = version_map[name][:given_version]
+        overridden = version_map[name][:overridden]
+
+        out << "#{name}".ljust(name_width)
+        out << version.to_s.ljust(version_width)
+        out << version_guid.to_s.ljust(guid_width) if version_guid
+        # Only print out column if something was overridden
+        out << given_version.ljust(override_width) if overridden
+        out << "\n"
+      end
+      out
+    end
+
+  end
+
+
+end

data/lib/omnibus/s3_cacher.rb

@@ -0,0 +1,136 @@
+#
+# Copyright:: Copyright (c) 2012 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'fileutils'
+require 'uber-s3'
+require 'omnibus/fetchers'
+
+module Omnibus
+
+
+  module SoftwareS3URLs
+
+    class InsufficientSpecification < ArgumentError
+    end
+
+    def config
+      Omnibus.config
+    end
+
+    def url_for(software)
+      "http://#{config.s3_bucket}.s3.amazonaws.com/#{key_for_package(software)}"
+    end
+
+    private
+
+    def key_for_package(package)
+      package.name     or raise InsufficientSpecification, "Software must have a name to cache it in S3 (#{package.inspect})"
+      package.version  or raise InsufficientSpecification, "Software must set a version to cache it in S3 (#{package.inspect})"
+      package.checksum or raise InsufficientSpecification, "Software must specify a checksum (md5) to cache it in S3 (#{package.inspect})"
+      "#{package.name}-#{package.version}-#{package.checksum}"
+    end
+
+  end
+
+  class S3Cache
+
+    include SoftwareS3URLs
+
+    def initialize
+      unless config.s3_bucket && config.s3_access_key && config.s3_secret_key
+        raise InvalidS3Configuration.new(config.s3_bucket, config.s3_access_key, config.s3_secret_key)
+      end
+      @client = UberS3.new(
+        :access_key         => config.s3_access_key,
+        :secret_access_key  => config.s3_secret_key,
+        :bucket             => config.s3_bucket,
+        :adaper             => :net_http
+      )
+    end
+
+    def log(msg)
+      puts "[S3 Cacher] #{msg}"
+    end
+
+    def config
+      Omnibus.config
+    end
+
+    def list
+      existing_keys = list_by_key
+      tarball_software.select {|s| existing_keys.include?(key_for_package(s))}
+    end
+
+    def list_by_key
+      bucket.objects('/').map(&:key)
+    end
+
+    def missing
+      already_cached = list_by_key
+      tarball_software.delete_if {|s| already_cached.include?(key_for_package(s))}
+    end
+
+    def tarball_software
+      Omnibus.library.select {|s| s.source && s.source.key?(:url)}
+    end
+
+    def populate
+      missing.each do |software|
+        fetch(software)
+
+        key = key_for_package(software)
+        content = IO.read(software.project_file)
+
+        log "Uploading #{software.project_file} as #{config.s3_bucket}/#{key}"
+        @client.store(key, content, :access => :public_read, :content_md5 => software.checksum)
+      end
+    end
+
+    def fetch_missing
+      missing.each do |software|
+        fetch(software)
+      end
+    end
+
+    private
+
+    def ensure_cache_dir
+      FileUtils.mkdir_p(config.cache_dir)
+    end
+
+    def fetch(software)
+      log "Fetching #{software.name}"
+      fetcher = Fetcher.without_caching_for(software)
+      if fetcher.fetch_required?
+        fetcher.download
+        fetcher.verify_checksum!
+      else
+        log "Cached copy up to date, skipping."
+      end
+    end
+
+    def bucket
+      @bucket ||= begin
+        b = UberS3::Bucket.new(@client, @client.bucket)
+        # creating the bucket is idempotent, make sure it's created:
+        @client.connection.put("/")
+        b
+      end
+    end
+
+  end
+end

data/lib/omnibus/software.rb

@@ -0,0 +1,430 @@
+#
+# Copyright:: Copyright (c) 2012 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'digest/md5'
+require 'mixlib/shellout'
+require 'net/ftp'
+require 'net/http'
+require 'net/https'
+require 'uri'
+
+require 'omnibus/fetcher'
+require 'omnibus/builder'
+require 'omnibus/config'
+
+require 'rake'
+
+module Omnibus
+
+  # Omnibus software DSL reader
+  class Software
+    include Rake::DSL
+
+    NULL_ARG = Object.new
+
+    # It appears that this is not used
+    attr_reader :builder
+
+    # @todo Why do we apparently use two different ways of
+    #   implementing what are effectively the same DSL methods?  Compare
+    #   with Omnibus::Project.
+    attr_reader :description
+
+    # @todo This doesn't appear to be used at all
+    attr_reader :fetcher
+
+    attr_reader :project
+
+    attr_reader :given_version
+    attr_reader :override_version
+
+    def self.load(filename, project, overrides={})
+      new(IO.read(filename), filename, project, overrides)
+    end
+
+    # @param io [String]
+    # @param filename [String]
+    # @param project [???] Is this a string or an Omnibus::Project?
+    # @param overrides [Hash]
+    #
+    # @see Omnibus::Overrides
+    #
+    # @todo See comment on {Omnibus::NullBuilder}
+    # @todo does `filename` need to be absolute, or does it matter?
+    # @ @todo Any reason to not have this just take a filename,
+    #   project, and override hash directly?  That is, why io AND a
+    #   filename, if the filename can always get you the contents you
+    #   need anyway?
+    def initialize(io, filename, project, overrides={})
+      @given_version    = nil
+      @override_version = nil
+      @name             = nil
+      @description      = nil
+      @source           = nil
+      @relative_path    = nil
+      @source_uri       = nil
+      @source_config    = filename
+      @project          = project
+      @always_build     = false
+
+      # Seems like this should just be Builder.new(self) instead
+      @builder = NullBuilder.new(self)
+
+      @dependencies = Array.new
+      instance_eval(io, filename, 0)
+
+      # Set override information after the DSL file has been consumed
+      @override_version = overrides[name]
+      
+      render_tasks
+    end
+
+    def name(val=NULL_ARG)
+      @name = val unless val.equal?(NULL_ARG)
+      @name
+    end
+
+    def description(val)
+      @description = val
+    end
+
+    # Add an Omnibus software dependency.
+    #
+    # @param val [String] the name of a Software dependency
+    # @return [void]
+    def dependency(val)
+      @dependencies << val
+    end
+
+    # Set or retrieve the list of software dependencies for this
+    # project.  As this is a DSL method, only pass the names of
+    # software components, not {Omnibus::Software} objects.
+    #
+    # These is the software that comprises your project, and is
+    # distinct from runtime dependencies.
+    #
+    # @note This will reinitialize the internal depdencies Array
+    #   and overwrite any dependencies that may have been set using
+    #   {#dependency}.
+    #
+    # @param val [Array<String>] a list of names of Software components
+    # @return [Array<String>]
+    def dependencies(val=NULL_ARG)
+      @dependencies = val unless val.equal?(NULL_ARG)
+      @dependencies
+    end
+
+    # Set or retrieve the source for the software
+    #
+    # @param val [Hash<Symbol, String>] a single key/pair that defines
+    #   the kind of source and a path specifier
+    # @option val [String] :git (nil) a Git URL
+    # @option val [String] :url (nil) a general URL
+    # @option val [String] :path (nil) a fully-qualified local file system path
+    #
+    # @todo Consider changing this to accept two arguments instead
+    # @todo This should throw an error if an invalid key is given, or
+    #   if more than one pair is given, or if no source value is ever
+    #   set.
+    def source(val=NULL_ARG)
+      @source = val unless val.equal?(NULL_ARG)
+      @source
+    end
+
+    # Set a version from a software descriptor file, or receive the
+    # effective version, taking into account any override information
+    # (if set)
+    def version(val=NULL_ARG)
+      @given_version = val unless val.equal?(NULL_ARG)
+      @override_version || @given_version
+    end
+
+    # Was this software version overridden externally, relative to the
+    # version declared within the software DSL file?
+    #
+    # @return [Boolean]
+    def overridden?
+      @override_version && (@override_version != @given_version)
+    end
+
+    # @todo see comments on {Omnibus::Fetcher#without_caching_for}
+    def version_guid
+      Fetcher.for(self).version_guid
+    end
+
+    # @todo Define as a delegator
+    def build_version
+      @project.build_version
+    end
+
+    # @todo Judging by existing usage, this should sensibly default to
+    #   the name of the software, since that's what it effectively does down in #project_dir
+    def relative_path(val)
+      @relative_path = val
+    end
+
+    # @todo Code smell... this only has meaning if the software was
+    #   defined with a :uri, and this is only used in
+    #   {Omnibus::NetFetcher}.  This responsibility is distributed
+    #   across two classes, one of which is a specific interface
+    #   implementation
+    # @todo Why the caching of the URI?
+    def source_uri
+      @source_uri ||= URI(@source[:url])
+    end
+
+    # @param val [Boolean]
+    # @return void
+    #
+    # @todo Doesn't necessarily need to be a Boolean if #always_build?
+    #   uses !! operator
+    def always_build(val)
+      @always_build = val
+    end
+
+    # @return [Boolean]
+    def always_build?
+      # Should do !!(@always_build)
+      @always_build
+    end
+
+    # @todo Code smell... this only has meaning if the software was
+    #   defined with a :uri, and this is only used in
+    #   {Omnibus::NetFetcher}.  This responsibility is distributed
+    #   across two classes, one of which is a specific interface
+    #   implementation
+    def checksum
+      @source[:md5]
+    end
+
+    # @todo Should this ever be legitimately used in the DSL?  It
+    #   seems that that facility shouldn't be provided, and thus this
+    #   should be made a private function (if it even really needs to
+    #   exist at all).
+    def config
+      Omnibus.config
+    end
+
+    # @!group Directory Accessors
+
+    def source_dir
+      config.source_dir
+    end
+
+    def cache_dir
+      config.cache_dir
+    end
+
+    # The directory that the software will be built in
+    #
+    # @return [String] an absolute filesystem path
+    def build_dir
+      "#{config.build_dir}/#{camel_case_path(install_dir)}"
+    end
+
+    # @todo Why the different name (i.e. *_dir instead of *_path, or
+    #   vice versa?)  Given the patterns that are being set up
+    #   elsewhere, this is just confusing inconsistency.
+    def install_dir
+      @project.install_path
+    end
+
+    # @!endgroup
+
+    # @todo It seems like this isn't used, and if it were, it should
+    # probably be part of Opscode::Builder instead
+    def max_build_jobs
+      if OHAI.cpu == nil
+        2
+      else
+        OHAI.cpu[:total] + 1
+      end
+    end
+
+    # @todo See comments for {#source_uri}... same applies here.  If
+    #   this is called in a non-source-software context, bad things will
+    #   happen.
+    def project_file
+      filename = source_uri.path.split('/').last
+      "#{cache_dir}/#{filename}"
+    end
+
+    # @todo this would be simplified and clarified if @relative_path
+    #   defaulted to @name... see the @todo tag for #relative_path
+    # @todo Move this up with the other *_dir methods for better
+    #   logical grouping
+    def project_dir
+      @relative_path ? "#{source_dir}/#{@relative_path}" : "#{source_dir}/#{@name}"
+    end
+
+    # @todo all the *_file methods should be next to each other for
+    #   better logical grouping
+    def manifest_file
+      manifest_file_from_name(@name)
+    end
+
+    # @todo Seems like this should be a private method, since it's
+    #   just used internally
+    def manifest_file_from_name(software_name)
+      "#{build_dir}/#{software_name}.manifest"
+    end
+
+    # The name of the sentinel file that marks the most recent fetch
+    # time of the software
+    #
+    # @return [String] an absolute path
+    #
+    # @see Omnibus::Fetcher
+    # @todo seems like this should be a private
+    #   method, since it's an implementation detail.
+    def fetch_file
+      "#{build_dir}/#{@name}.fetch"
+    end
+
+    # @todo This is actually "snake case", not camel case
+    # @todo this should be a private method
+    def camel_case_path(project_path)
+      path = project_path.dup
+      # split the path and remmove and empty strings
+      if platform == 'windows'
+        path.sub!(":", "")
+        parts = path.split("\\") - [""]
+        parts.join("_")
+      else
+        parts = path.split("/") - [""]
+        parts.join("_")
+      end
+    end
+
+    # Define a series of {Omnibus::Builder} DSL commands that are
+    # required to successfully build the software.
+    #
+    # @param block [block] a block of build commands
+    # @return void
+    #
+    # @see Omnibus::Builder
+    #
+    # @todo Not quite sure the proper way to document a "block"
+    #   parameter in Yard
+    # @todo Seems like this renders the setting of @builder in the
+    #   initializer moot
+    # @todo Rename this to something like "build_commands", since it
+    #   doesn't actually do any building
+    def build(&block)
+      @builder = Builder.new(self, &block)
+    end
+
+    # Returns the platform of the machine on which Omnibus is running,
+    # as determined by Ohai.
+    #
+    # @return [String]
+    def platform
+      OHAI.platform
+    end
+
+    # Return the architecture of the machine, as determined by Ohai.
+    # @return [String] Either "sparc" or "intel", as appropriate
+    # @todo Is this used?  Doesn't appear to be...
+    def architecture
+      OHAI.kernel['machine'] =~ /sun/ ? "sparc" : "intel"
+    end
+
+    private
+
+    # @todo What?!
+    # @todo It seems that this is not used... remove it
+    # @deprecated Use something else (?)
+    def command(*args)
+      raise "Method Moved."
+    end
+
+    def execute_build(fetcher)
+      fetcher.clean
+      @builder.build
+      touch manifest_file
+    end
+
+    def render_tasks
+      namespace "projects:#{@project.name}" do
+      namespace :software do
+        fetcher = Fetcher.for(self)
+
+        #
+        # set up inter-project dependencies
+        #
+        (@dependencies - [@name]).uniq.each do |dep|
+          task @name => dep
+          file manifest_file => manifest_file_from_name(dep)  
+        end
+
+        directory source_dir
+        directory cache_dir
+        directory build_dir
+        directory project_dir
+        namespace @name do
+          task :fetch => [ build_dir, source_dir, cache_dir, project_dir ] do
+            if !File.exists?(fetch_file) || fetcher.fetch_required?
+              # force build to run if we need to do an updated fetch
+              fetcher.fetch
+              touch fetch_file
+            end
+          end
+
+          task :build => :fetch do
+            if !always_build? && uptodate?(manifest_file, [fetch_file])
+              # if any direct deps have been built for any reason, we will need to
+              # clean/build ourselves
+              (@dependencies - [@name]).uniq.each do |dep|
+                unless uptodate?(manifest_file, [manifest_file_from_name(dep)])
+                  execute_build(fetcher)
+                  break
+                end
+              end
+
+            else 
+              # if fetch has occurred, or the component is configured to 
+              # always build, do a clean and build.
+              execute_build(fetcher)
+            end
+          end
+        end
+
+        #
+        # make the manifest file dependent on the latest file in the
+        # source tree in order to shrink the multi-thousand-node
+        # dependency graph that Rake was generating
+        #
+        latest_file = FileList["#{project_dir}/**/*"].sort { |a,b|
+          File.mtime(a) <=> File.mtime(b)
+        }.last
+
+        file manifest_file => (file latest_file)
+
+        file fetch_file => "#{name}:fetch" 
+        file manifest_file => "#{name}:build"
+
+        file fetch_file => (file @source_config)
+        file manifest_file => (file fetch_file)
+
+        desc "fetch and build #{@name} for #{@project.name}"
+        task @name => manifest_file
+      end
+      end
+    end
+
+  end
+end