wjordan-mixlib-install 0.8.0.alpha.6

data/lib/mixlib/install/backend/artifactory.rb

@@ -0,0 +1,214 @@
+#
+# Author:: Patrick Wright (<patrick@chef.io>)
+# Copyright:: Copyright (c) 2015 Chef, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "net/http"
+require "json"
+require "mixlib/install/artifact_info"
+require "artifactory"
+
+module Mixlib
+  class Install
+    class Backend
+      class Artifactory
+        class ConnectionError < StandardError; end
+        class AuthenticationError < StandardError; end
+        class NoArtifactsError < StandardError; end
+
+        ENDPOINT = "http://artifactory.chef.co".freeze
+
+        attr_accessor :options
+
+        def initialize(options)
+          @options = options
+        end
+
+        # Create filtered list of artifacts
+        #
+        # @return [Array<ArtifactInfo>] list of artifacts for the configured
+        # channel, product name, and product version.
+        # @return [ArtifactInfo] arifact info for the configured
+        # channel, product name, product version and platform info
+        #
+        def info
+          artifacts = if options.latest_version?
+            artifactory_latest
+          else
+            artifactory_artifacts(options.product_version)
+          end
+
+          if options.platform
+            artifacts.select! do |a|
+              a.platform == options.platform &&
+                a.platform_version == options.platform_version &&
+                a.architecture == options.architecture
+            end
+          end
+
+          artifacts.length == 1 ? artifacts.first : artifacts
+        end
+
+        #
+        # Get artifacts for the latest version, channel and product_name
+        #
+        # @return [Array<ArtifactInfo>] Array of info about found artifacts
+        def artifactory_latest
+          # Get the list of builds from the REST api.
+          # We do this because a user in the readers group does not have
+          # permissions to run aql against builds.
+          builds = client.get("/api/build/#{options.product_name}")
+
+          if builds.nil?
+            raise NoArtifactsError, <<-MSG
+Can not find any builds for #{options.product_name} in #{::Artifactory.endpoint}.
+            MSG
+          end
+
+          # Output we get is something like:
+          # {
+          #   "buildsNumbers": [
+          #     {"uri"=>"/12.5.1+20151213083009", "started"=>"2015-12-13T08:40:19.238+0000"},
+          #     {"uri"=>"/12.6.0+20160111212038", "started"=>"2016-01-12T00:25:35.762+0000"},
+          #     ...
+          #   ]
+          # }
+          # First we sort based on started
+          builds["buildsNumbers"].sort_by! { |b| b["started"] }.reverse!
+
+          # Now check if the build is in the requested channel or not
+          # Note that if you do this for any channel other than :unstable
+          # it will run a high number of queries but it is fine because we
+          # are using artifactory only for :unstable channel
+          builds["buildsNumbers"].each do |build|
+            version = build["uri"].gsub("/", "")
+            artifacts = artifactory_artifacts(version)
+
+            return artifacts unless artifacts.empty?
+          end
+
+          # we could not find any matching artifacts
+          []
+        end
+
+        #
+        # Get artifacts for a given version, channel and product_name
+        #
+        # @return [Array<ArtifactInfo>] Array of info about found artifacts
+        def artifactory_artifacts(version)
+          results = artifactory_query(<<-QUERY)
+items.find(
+  {"repo": "omnibus-#{options.channel}-local"},
+  {"@omnibus.project": "#{options.product_name}"},
+  {"@omnibus.version": "#{version}"}
+).include("repo", "path", "name", "property")
+          QUERY
+
+          # Merge artifactory properties and downloadUri to a flat Hash
+          results.collect! do |result|
+            { "downloadUri" => generate_download_uri(result) }.merge(
+              map_properties(result["properties"])
+            )
+          end
+
+          # Convert results to build records
+          results.map { |a| create_artifact(a) }
+        end
+
+        #
+        # Run an artifactory query for the given query.
+        #
+        # @return [Array<Hash>] Array of results returned from artifactory
+        def artifactory_query(query)
+          results = artifactory_request do
+            client.post("/api/search/aql", query, "Content-Type" => "text/plain")
+          end
+
+          results["results"]
+        end
+
+        def create_artifact(artifact_map)
+          ArtifactInfo.new(
+            md5:              artifact_map["omnibus.md5"],
+            sha256:           artifact_map["omnibus.sha256"],
+            version:          artifact_map["omnibus.version"],
+            platform:         artifact_map["omnibus.platform"],
+            platform_version: artifact_map["omnibus.platform_version"],
+            architecture:     artifact_map["omnibus.architecture"],
+            url:              artifact_map["downloadUri"]
+          )
+        end
+
+        private
+
+        # Converts Array<Hash> where the Hash is a key pair and
+        # value pair to a simplifed key/pair Hash
+        #
+        def map_properties(properties)
+          return {} if properties.nil?
+          properties.each_with_object({}) do |prop, h|
+            h[prop["key"]] = prop["value"]
+          end
+        end
+
+        # Construct the downloadUri from raw artifactory data
+        #
+        def generate_download_uri(result)
+          uri = []
+          uri << endpoint.sub(/\/$/, "")
+          uri << result["repo"]
+          uri << result["path"]
+          uri << result["name"]
+          uri.join("/")
+        end
+
+        def client
+          @client ||= ::Artifactory::Client.new(
+            endpoint: endpoint,
+            username: ENV["ARTIFACTORY_USERNAME"],
+            password: ENV["ARTIFACTORY_PASSWORD"]
+          )
+        end
+
+        def endpoint
+          @endpoint ||= ENV.fetch("ARTIFACTORY_ENDPOINT", ENDPOINT)
+        end
+
+        def artifactory_request
+          begin
+            results = yield
+          rescue Errno::ETIMEDOUT, ::Artifactory::Error::ConnectionError
+            raise ConnectionError, <<-EOS
+Artifactory endpoint '#{::Artifactory.endpoint}' is unreachable. Check that
+the endpoint is correct and there is an open connection to Chef's private network.
+            EOS
+          rescue ::Artifactory::Error::HTTPError => e
+            if e.code == 401 && e.message =~ /Bad credentials/
+              raise AuthenticationError, <<-EOS
+Artifactory server denied credentials. Verify ARTIFACTORY_USERNAME and
+ARTIFACTORY_PASSWORD environment variables are configured properly.
+              EOS
+            else
+              raise e
+            end
+          end
+
+          results
+        end
+      end
+    end
+  end
+end

data/lib/mixlib/install/backend/omnitruck.rb

@@ -0,0 +1,76 @@
+#
+# Author:: Patrick Wright (<patrick@chef.io>)
+# Copyright:: Copyright (c) 2015 Chef, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "net/http"
+require "json"
+require "mixlib/install/artifact_info"
+
+module Mixlib
+  class Install
+    class Backend
+      class Omnitruck
+        ENDPOINT = "https://omnitruck.chef.io/".freeze
+
+        attr_accessor :options
+
+        def initialize(options)
+          @options = options
+        end
+
+        def info
+          # If we are querying a single platform we need to call metadata
+          # endpoint otherwise we need to call versions endpoint in omnitruck
+          if options.platform
+            build = omnitruck_get("metadata", p: options.platform,
+                                              pv: options.platform_version,
+                                              m: options.architecture,
+                                              v: options.product_version
+                                 )
+            ArtifactInfo.from_json(build,
+                                   platform: options.platform,
+                                   platform_version: options.platform_version,
+                                   architecture: options.architecture
+            )
+          else
+            builds = omnitruck_get("versions", v: options.product_version)
+            ArtifactInfo.from_metadata_map(builds)
+          end
+        end
+
+        private
+
+        def omnitruck_get(resource, parameters)
+          uri = URI.parse(ENDPOINT)
+          http = Net::HTTP.new(uri.host, uri.port)
+          http.use_ssl = (uri.scheme == "https")
+
+          path = "/#{options.channel}/#{options.product_name}/#{resource}"
+          full_path = [path, URI.encode_www_form(parameters)].join("?")
+          request = Net::HTTP::Get.new(full_path)
+          request["Accept"] = "application/json"
+
+          res = http.request(request)
+
+          # Raise if response is not 2XX
+          res.value
+          res.body
+        end
+      end
+    end
+  end
+end

data/lib/mixlib/install/backend.rb

@@ -0,0 +1,36 @@
+#
+# Author:: Patrick Wright (<patrick@chef.io>)
+# Copyright:: Copyright (c) 2015 Chef, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "mixlib/install/backend/omnitruck"
+require "mixlib/install/backend/artifactory"
+
+module Mixlib
+  class Install
+    class Backend
+      def self.info(options)
+        backend = if options.for_omnitruck?
+          Backend::Omnitruck.new(options)
+        elsif options.for_artifactory?
+          Backend::Artifactory.new(options)
+        end
+
+        backend.info
+      end
+    end
+  end
+end

data/lib/mixlib/install/generator/base.rb

@@ -0,0 +1,65 @@
+#
+# Copyright:: Copyright (c) 2015 Chef, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "erb"
+require "ostruct"
+require "mixlib/install/backend/omnitruck"
+
+module Mixlib
+  class Install
+    class Generator
+      class Base
+        attr_reader :options
+
+        def initialize(options)
+          @options = options
+        end
+
+        #
+        # Returns the base path where the script fragments are located for
+        # the generator as a String.
+        #
+        def self.script_base_path
+          raise "You must define a script_base_path for your Generator::Base class."
+        end
+
+        #
+        # Gets the contents of the given script.
+        #
+        def self.get_script(name, context = {})
+          script_path = File.join(script_base_path, name)
+
+          # If there is an erb template we render it, otherwise we just read
+          # and returnt the contents of the script
+          if File.exist? "#{script_path}.erb"
+            # Default values to use incase they are not set in the context
+            context[:base_url] ||= Mixlib::Install::Backend::Omnitruck::ENDPOINT
+
+            context_object = OpenStruct.new(context).instance_eval { binding }
+            ERB.new(File.read("#{script_path}.erb")).result(context_object)
+          else
+            File.read(script_path)
+          end
+        end
+
+        def get_script(name, context = {})
+          self.class.get_script(name, context)
+        end
+      end
+    end
+  end
+end

data/lib/mixlib/install/generator/bourne/scripts/artifactory_urls.sh.erb

@@ -0,0 +1,31 @@
+# artifactory_urls.sh
+############
+# This section sets up the information for a build to be installed from artifactory
+#  then uses $platform, $platform_version and $machine to select the correct
+#  version.
+#
+# Inputs:
+# $platform:
+# $platform_version:
+# $machine:
+#
+# Outputs:
+# $download_url:
+# $sha256:
+# $md5:
+############
+
+<% artifacts.each do |artifact| %>
+  artifact_info_dir="$tmp_dir/artifact_info/<%= File.join(artifact.platform, artifact.platform_version, artifact.architecture)%>"
+  mkdir -p $artifact_info_dir
+  touch "$artifact_info_dir/artifact_info"
+  echo "url <%= artifact.url%>" >> "$artifact_info_dir/artifact_info"
+  echo "md5 <%= artifact.md5%>" >> "$artifact_info_dir/artifact_info"
+  echo "sha256 <%= artifact.sha256%>" >> "$artifact_info_dir/artifact_info"
+<% end %>
+
+artifact_info_for_platform="$tmp_dir/artifact_info/$platform/$platform_version/$machine/artifact_info"
+
+download_url=`awk '$1 == "url" { print $2 }' "$artifact_info_for_platform"`
+sha256=`awk '$1 == "sha256" { print $2 }' "$artifact_info_for_platform"`
+md5=`awk '$1 == "md5" { print $2 }' "$artifact_info_for_platform"`

data/lib/mixlib/install/generator/bourne/scripts/fetch_metadata.sh.erb

@@ -0,0 +1,48 @@
+# fetch_metadata.sh
+############
+# This section calls omnitruck to get the information about the build to be
+#   installed.
+#
+# Inputs:
+# $channel:
+# $project:
+# $version:
+# $platform:
+# $platform_version:
+# $machine:
+# $tmp_dir:
+#
+# Outputs:
+# $download_url:
+# $sha256:
+# $md5:
+############
+
+echo "Getting information for $project $channel $version for $platform..."
+
+metadata_filename="$tmp_dir/metadata.txt"
+metadata_url="<%= base_url %>$channel/$project/metadata?v=$version&p=$platform&pv=$platform_version&m=$machine"
+
+do_download "$metadata_url"  "$metadata_filename"
+
+cat "$metadata_filename"
+
+echo ""
+# check that all the mandatory fields in the downloaded metadata are there
+if grep '^url' $metadata_filename > /dev/null && grep '^sha256' $metadata_filename > /dev/null && grep '^md5' $metadata_filename > /dev/null; then
+  echo "downloaded metadata file looks valid..."
+else
+  echo "downloaded metadata file is corrupted or an uncaught error was encountered in downloading the file..."
+  # this generally means one of the download methods downloaded a 404 or something like that and then reported a successful exit code,
+  # and this should be fixed in the function that was doing the download.
+  report_bug
+  exit 1
+fi
+
+download_url=`awk '$1 == "url" { print $2 }' "$metadata_filename"`
+sha256=`awk '$1 == "sha256" { print $2 }' "$metadata_filename"`
+md5=`awk '$1 == "md5" { print $2 }' "$metadata_filename"`
+
+############
+# end of fetch_metadata.sh
+############

data/lib/mixlib/install/generator/bourne/scripts/fetch_package.sh

@@ -0,0 +1,53 @@
+# fetch_package.sh
+############
+# This section fetchs a package from $download_url and verifies its metadata.
+#
+# Inputs:
+# $download_url:
+# $tmp_dir:
+# Optional Inputs:
+# $cmdline_filename: Name of the package downloaded on local disk.
+# $cmdline_dl_dir: Name of the directory downloaded package will be saved to on local disk.
+#
+# Outputs:
+# $download_filename: Name of the downloaded file on local disk.
+# $filetype: Type of the file downloaded.
+############
+
+filename=`echo $download_url | sed -e 's/^.*\///'`
+filetype=`echo $filename | sed -e 's/^.*\.//'`
+
+# use either $tmp_dir, the provided directory (-d) or the provided filename (-f)
+if test "x$cmdline_filename" != "x"; then
+  download_filename="$cmdline_filename"
+elif test "x$cmdline_dl_dir" != "x"; then
+  download_filename="$cmdline_dl_dir/$filename"
+else
+  download_filename="$tmp_dir/$filename"
+fi
+
+# ensure the parent directory where to download the installer always exists
+download_dir=`dirname $download_filename`
+(umask 077 && mkdir -p $download_dir) || exit 1
+
+# check if we have that file locally available and if so verify the checksum
+cached_file_available="false"
+if test -f $download_filename; then
+  echo "$download_filename already exists, verifiying checksum..."
+  if do_checksum "$download_filename" "$sha256" "$md5"; then
+    echo "checksum compare succeeded, using existing file!"
+    cached_file_available="true"
+  else
+    echo "checksum mismatch, downloading latest version of the file"
+  fi
+fi
+
+# download if no local version of the file available
+if test "x$cached_file_available" != "xtrue"; then
+  do_download "$download_url"  "$download_filename"
+  do_checksum "$download_filename" "$sha256" "$md5" || checksum_mismatch
+fi
+
+############
+# end of fetch_package.sh
+############