vara 0.17.1

data/lib/vara/product_artifact_zipper.rb

@@ -0,0 +1,110 @@
+require 'tmpdir'
+require 'vara/log'
+require 'vara/product_artifact_validator'
+
+module Vara
+  class ProductArtifactZipper
+    include Loggable
+
+    def initialize(artifact_path, product_contents)
+      @artifact_path    = artifact_path
+      @product_contents = product_contents
+    end
+
+    def zip!
+      remove_previous_artifacts
+
+      in_tmp_dir do
+        copy_dirs_and_files
+        create_zip!
+      end
+    end
+
+    private
+
+    attr_reader :artifact_path, :product_contents
+
+    def remove_previous_artifacts
+      FileUtils.rm(artifact_path) if File.exist?(artifact_path)
+    end
+
+    def in_tmp_dir
+      Dir.mktmpdir do |temp_dir|
+        FileUtils.cd(temp_dir) do
+          yield
+        end
+      end
+    end
+
+    def copy_dirs_and_files
+      product_metadata = ProductMetadata.from_file(product_contents.metadata_path)
+
+      copy_and_validate_releases(product_metadata)
+
+      if product_metadata.explicit_stemcell?
+        copy_and_validate_stemcell(product_metadata)
+        copy_and_validate_compiled_packages(product_metadata)
+      end
+
+      FileUtils.mkdir('content_migrations')
+      FileUtils.cp(product_contents.content_migrations_path, 'content_migrations')
+
+      FileUtils.mkdir('metadata')
+      FileUtils.cp(product_contents.metadata_path, 'metadata')
+    end
+
+    def copy_and_validate_releases(product_metadata)
+      FileUtils.mkdir('releases')
+
+      product_contents.release_paths.each do |release_path|
+        FileUtils.cp(release_path, 'releases')
+      end
+
+      product_metadata.releases_metadata.each do |release_metadata|
+        ProductArtifactValidator.validate_file_checksum(
+          release_metadata,
+          File.join('releases', release_metadata.basename)
+        )
+      end
+    end
+
+    def copy_and_validate_stemcell(product_metadata)
+      FileUtils.mkdir('stemcells')
+      FileUtils.cp(product_contents.stemcell_path, 'stemcells')
+
+      ProductArtifactValidator.validate_file_checksum(
+        product_metadata.stemcell_metadata,
+        File.join('stemcells', product_metadata.stemcell_file)
+      )
+    end
+
+    def copy_and_validate_compiled_packages(product_metadata)
+      return unless product_contents.has_compiled_packages?
+
+      FileUtils.mkdir('compiled_packages')
+      FileUtils.cp(product_contents.compiled_packages_path, 'compiled_packages')
+
+      ProductArtifactValidator.validate_file_checksum(
+        product_metadata.compiled_packages_metadata,
+        File.join('compiled_packages', product_metadata.compiled_packages_file)
+      )
+    end
+
+    def create_zip!
+      log.info("Creating zip file #{artifact_path}")
+      # Split into two steps so that it works inside Docker on a mounted volume
+      `zip -r -0 local.pivotal .`
+      FileUtils.mv('local.pivotal', artifact_path)
+      log.info('Finished creating zip file')
+    end
+  end
+end
+
+# Copyright (c) 2014-2015 Pivotal Software, Inc.
+# All rights reserved.
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
+# INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+# USE OR OTHER DEALINGS IN THE SOFTWARE.

data/lib/vara/product_contents.rb

@@ -0,0 +1,64 @@
+require 'vara/product_metadata'
+
+module Vara
+  class ProductContents
+    # @param metadata_path [String] The path on disk to the metadata YAML file
+    # @return [Vara::ProductContents]
+    def self.from_metadata_path(metadata_path)
+      new(metadata_path, ProductMetadata.from_file(metadata_path))
+    end
+
+    attr_reader :metadata_path
+
+    # @param metadata_path [String] The path on disk to the metadata YAML file
+    # @param product_metadata [Vara::ProductMetadata] The Product Metadata as represented in the metadata_path file
+    def initialize(metadata_path, product_metadata)
+      @metadata_path    = File.expand_path(metadata_path).freeze
+      @product_metadata = product_metadata
+    end
+
+    # @return [String] The path to the content migration file on disk
+    def content_migrations_path
+      paths = Dir.glob(File.expand_path(File.join(root_path, 'content_migrations', '*.yml')))
+      fail "Exactly one content_migrations file is required in #{root_path}/content_migrations" if paths.size != 1
+      paths.first
+    end
+
+    # @return [<String>] The paths to the release tarballs on disk
+    def release_paths
+      product_metadata.releases_metadata.map { |release_metadata| File.join(root_path, 'releases', release_metadata.basename) }
+    end
+
+    # @return [String] The path to the stemcell file on disk
+    def stemcell_path
+      File.join(root_path, 'stemcells', product_metadata.stemcell_file)
+    end
+
+    # @return [String] The path to the compiled packages file on disk
+    def compiled_packages_path
+      File.join(root_path, 'compiled_packages', product_metadata.compiled_packages_file)
+    end
+
+    # @return [Boolean] Whether the product metadata has a compiled packages field
+    def has_compiled_packages?
+      product_metadata.has_compiled_packages?
+    end
+
+    private
+
+    def root_path
+      File.expand_path(File.join(metadata_path, '..', '..'))
+    end
+
+    attr_reader :product_metadata
+  end
+end
+
+# Copyright (c) 2014-2015 Pivotal Software, Inc.
+# All rights reserved.
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
+# INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+# USE OR OTHER DEALINGS IN THE SOFTWARE.

data/lib/vara/product_metadata.rb

@@ -0,0 +1,113 @@
+require 'yaml'
+require 'vara/metadata/stemcell'
+require 'vara/metadata/release'
+require 'vara/metadata/compiled_packages'
+
+module Vara
+  # "Struct" representing the information in metadata/<product_name>.yml
+  class ProductMetadata
+    # Build a ProductMetadata from a metadata.yml on disk
+    # @param metadata_path Path to the product metadata file on disk
+    # @return [Vara::ProductMetadata]
+    def self.from_file(metadata_path)
+      new(YAML.load_file(metadata_path))
+    end
+
+    # @param hash [Hash] the hash representing the product metadata
+    def initialize(hash)
+      @hash = hash
+    end
+
+    def explicit_stemcell?
+      hash.key? 'stemcell'
+    end
+
+    def stemcell_criteria
+      return nil if explicit_stemcell?
+      hash.fetch('stemcell_criteria')
+    end
+
+    def stemcell_criteria?
+      hash.key? 'stemcell_criteria'
+    end
+
+    # @return [Vara::Metadata::Stemcell] the stemcell metadata contained in this ProductMetadata
+    def stemcell_metadata
+      stemcell = hash.fetch('stemcell')
+      Metadata::Stemcell.new(
+        name: stemcell.fetch('name'),
+        version: stemcell.fetch('version'),
+        file: stemcell.fetch('file'),
+        md5: stemcell.fetch('md5', nil),
+        sha1: stemcell.fetch('sha1', nil)
+      )
+    end
+
+    # @return [<Vara::Metadata::Release>] the releases metadata contained in this ProductMetadata
+    def releases_metadata
+      releases = hash.fetch('releases')
+      releases.map { |release| release_metadata(release) }
+    end
+
+    # @raise [KeyNotFoundError] if this ProductMetadata does not contain compiled packages
+    # @return [Vara::Metadata::CompiledPackages] the compiled packages metadata contained in this ProductMetadata
+    def compiled_packages_metadata
+      c = hash.fetch('compiled_package')
+      Metadata::CompiledPackages.new(
+        c.fetch('name'), c.fetch('version'), c.fetch('file'), c.fetch('md5'), c.fetch('url', nil)
+      )
+    end
+
+    # @return [String] the filename of the stemcell
+    def stemcell_file
+      stemcell_metadata.basename
+    end
+
+    # @return [String] the name of the product
+    def name
+      hash.fetch('name')
+    end
+
+    # @return [String] the version of the product
+    def product_version
+      hash.fetch('product_version')
+    end
+
+    # @return [String] the filename of the compiled packages
+    def compiled_packages_file
+      compiled_packages_metadata.basename
+    end
+
+    # @return [Boolean] whether the compiled packages field is present
+    def has_compiled_packages?
+      hash.key?('compiled_package')
+    end
+
+    private
+
+    attr_reader :hash
+
+    # @return [Vara::Metadata::Release]
+    def release_metadata(release_hash)
+      release = release_hash
+      Metadata::Release.new(
+        name: release.fetch('name'),
+        version: release.fetch('version'),
+        file: release.fetch('file'),
+        md5: release.fetch('md5', nil),
+        sha1: release.fetch('sha1', nil),
+        url: release.fetch('url', nil),
+        aws: release.fetch('aws', nil)
+      )
+    end
+  end
+end
+
+# Copyright (c) 2014-2015 Pivotal Software, Inc.
+# All rights reserved.
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
+# INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+# USE OR OTHER DEALINGS IN THE SOFTWARE.

data/lib/vara/product_metadata_processor.rb

@@ -0,0 +1,98 @@
+require 'yaml'
+require 'vara/log'
+
+module Vara
+  class ProductMetadataProcessor
+    include Vara::Loggable
+
+    # @param product_dir [String] path to product directory root
+    # @param versioner [Vara::PrereleaseVersioner,Vara::StaticVersioner]
+    def initialize(product_dir, versioner, external_release_paths: [])
+      @product_dir = product_dir
+      @versioner = versioner
+      @external_release_paths = external_release_paths
+    end
+
+    # Composes metadata_parts/binaries.yml and metadata_parts/handcraft.yml into the properly named product metadata
+    # file under metadata/, expanding placeholders as necessary
+    # @return [String] The path to the generated metadata file
+    def process
+      metadata = merged_metadata
+
+      external_release_paths.each do |external_release_path|
+        update_release(external_release_path, metadata)
+      end
+
+      processed = versioner.update_metadata(metadata)
+      save_processed_metadata(processed)
+    end
+
+    private
+
+    attr_reader :product_dir, :versioner, :external_release_paths
+
+    ALLOWED_RELEASE_KEYS = %w(file name url md5 version)
+
+    def merged_metadata
+      binaries_path = File.join(product_dir, 'metadata_parts', 'binaries.yml')
+      binaries = YAML.load_file(binaries_path)
+      log.info("Composing metadata: loaded #{binaries_path}")
+
+      handcraft_path = File.join(product_dir, 'metadata_parts', 'handcraft.yml')
+      handcraft = YAML.load_file(handcraft_path)
+      log.info("Composing metadata: loaded #{handcraft_path}")
+
+      binaries.merge(handcraft)
+    end
+
+    def update_release(external_release_path, metadata)
+      log.info("Updating metadata: using #{File.expand_path(external_release_path)}")
+
+      external_releases = YAML.load_file(external_release_path)
+
+      external_releases.each do |release|
+        external_release = validate_release(release, external_release_path)
+
+        replace = metadata['releases'].find_index do |r|
+          r['name'] == external_release['name']
+        end
+
+        fail "Specified release #{external_release['name']} does not exist in binaries.yml" unless replace
+
+        metadata['releases'][replace] = external_release
+      end
+    end
+
+    def validate_release(external_release, external_release_path)
+      missing_keys = ALLOWED_RELEASE_KEYS - external_release.keys.to_a
+      fail "Specified release #{File.basename(external_release_path)} is missing keywords: #{missing_keys}" unless missing_keys.empty?
+      extra_keys = external_release.keys.to_a - ALLOWED_RELEASE_KEYS
+      fail "Specified release #{File.basename(external_release_path)} has extra keywords: #{extra_keys}" unless extra_keys.empty?
+      external_release
+    end
+
+    def save_processed_metadata(processed_metadata_hash)
+      product_name = processed_metadata_hash.fetch('name').tr('-', '_')
+      metadata_dir = File.join(product_dir, 'metadata')
+
+      FileUtils.mkdir_p(metadata_dir)
+
+      out_path = File.join(metadata_dir, "#{product_name}.yml")
+
+      File.open(out_path, 'w') do |out|
+        YAML.dump(processed_metadata_hash, out)
+      end
+
+      out_path
+    end
+  end
+end
+
+# Copyright (c) 2014-2015 Pivotal Software, Inc.
+# All rights reserved.
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
+# INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+# USE OR OTHER DEALINGS IN THE SOFTWARE.

data/lib/vara/product_resource_downloader.rb

@@ -0,0 +1,69 @@
+require 'vara/product_metadata'
+require 'vara/tarball'
+
+module Vara
+  class ProductResourceDownloader
+    def self.build(product_metadata_path)
+      product_metadata = Vara::ProductMetadata.from_file(product_metadata_path)
+      product_basedir = File.expand_path(File.join(File.dirname(product_metadata_path), '..'))
+      new(product_metadata, product_basedir)
+    end
+
+    attr_reader :product_basedir, :product_metadata
+
+    def initialize(product_metadata, product_basedir)
+      @product_basedir  = product_basedir
+      @product_metadata = product_metadata
+    end
+
+    def download
+      if product_metadata.explicit_stemcell? && product_metadata.stemcell_criteria?
+        fail 'binaries.yml includes both stemcell and stemcell criteria keys'
+      end
+
+      if product_metadata.explicit_stemcell?
+        download_stemcell
+        download_compiled_packages
+      end
+
+      download_releases
+    end
+
+    private
+
+    def download_stemcell
+      stemcell_metadata   = product_metadata.stemcell_metadata
+      stemcell_directory  = File.join(product_basedir, 'stemcells')
+      stemcell = Vara::Tarball.new(stemcell_metadata, stemcell_directory)
+      stemcell.sync_local_copy
+    end
+
+    def download_releases
+      releases_metadata = product_metadata.releases_metadata
+
+      releases_metadata.each do |release_metadata|
+        release_directory = File.join(product_basedir, 'releases')
+        release = Vara::Tarball.new(release_metadata, release_directory)
+        release.sync_local_copy
+      end
+    end
+
+    def download_compiled_packages
+      return unless product_metadata.has_compiled_packages?
+
+      compiled_packages_metadata   = product_metadata.compiled_packages_metadata
+      compiled_packages_dir        = File.join(product_basedir, 'compiled_packages')
+      compiled_package = Vara::Tarball.new(compiled_packages_metadata, compiled_packages_dir)
+      compiled_package.sync_local_copy
+    end
+  end
+end
+
+# Copyright (c) 2014-2015 Pivotal Software, Inc.
+# All rights reserved.
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
+# INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+# USE OR OTHER DEALINGS IN THE SOFTWARE.