minimart 0.0.1

data/lib/minimart/mirror/download_metadata.rb

@@ -0,0 +1,57 @@
+module Minimart
+  module Mirror
+    # This class can be used to parse, and create `.minimart.json` files to
+    # store information about when, and how Minimart downloaded a given cookbook.
+    class DownloadMetadata
+
+      FILE_NAME = '.minimart.json'
+
+      # @return [String] the path to the directory containing the cookbook.
+      attr_reader :path_to_cookbook
+
+      # @return [Hash] the contents of the metadata file.
+      attr_reader :metadata
+
+      # @param [String] path_to_cookbook The path to the directory containing the cookbook.
+      def initialize(path_to_cookbook)
+        @path_to_cookbook = path_to_cookbook
+        parse_file
+      end
+
+      # Write the given contents to the metadata file. This will overwrite any
+      # existing contents.
+      # @param [Hash] contents The hash of data to write to the file.
+      def write(contents = {})
+        File.open(file_path, 'w+') do |file|
+          @metadata = contents
+          @metadata.merge!('downloaded_at' => Time.now.utc.iso8601)
+
+          file.write(metadata.to_json)
+        end
+      end
+
+      # @return [Time] The downloaded_at time found in the metadata file.
+      def downloaded_at
+        return unless self['downloaded_at']
+        Time.iso8601(metadata['downloaded_at']).utc
+      end
+
+      def [](key)
+        metadata[key] if metadata
+      end
+
+      private
+
+      def parse_file
+        return unless File.exists?(file_path)
+        file_contents = File.open(file_path).read
+        @metadata = JSON.parse(file_contents)
+      end
+
+      def file_path
+        File.join(path_to_cookbook, FILE_NAME)
+      end
+
+    end
+  end
+end

data/lib/minimart/mirror/inventory_builder.rb

@@ -0,0 +1,143 @@
+module Minimart
+  module Mirror
+
+    # InventoryBuilder coordinates downloading any cookbooks, and their dependencies.
+    class InventoryBuilder
+
+      # @return [Minimart::Mirror::InventoryConfiguration] The user specified inventory config.
+      attr_reader :inventory_configuration
+
+      # @return [Minimart::Mirror::DependencyGraph]
+      attr_reader :graph
+
+      # @return [Minimart::Mirror::LocalStore] the local store manages the inventory directory contents
+      attr_reader :local_store
+
+      # @param [String] inventory_directory The directory to store the inventory.
+      # @param [Minimart::Mirror::InventoryConfiguration] inventory_configuration The inventory as defined by a user of Minimart.
+      def initialize(inventory_directory, inventory_configuration)
+        @graph                   = DependencyGraph.new
+        @local_store             = LocalStore.new(inventory_directory)
+        @inventory_configuration = inventory_configuration
+      end
+
+      # Build the inventory!
+      def build!
+        install_cookbooks_with_explicit_location
+        add_source_cookbooks_to_graph
+        add_requirements_to_graph
+        fetch_inventory
+        display_success_message
+
+      ensure
+        clear_cache
+      end
+
+      private
+
+      # First we must install any cookbooks with a location specification (git, local path, etc..).
+      # These cookbooks and their associated metadata (any dependencies they have) take
+      # precedence over information found elsewhere.
+      def install_cookbooks_with_explicit_location
+        inventory_requirements.each_with_explicit_location do |requirement|
+          requirement.fetch_cookbook do |cookbook|
+            validate_cookbook_against_local_store(cookbook, requirement)
+            add_artifact_to_graph(cookbook)
+            add_cookbook_to_local_store(cookbook.path, requirement.to_hash)
+          end
+        end
+      end
+
+      # Fetch the universe from any of the defined sources, and add them as artifacts
+      #  to the dependency resolution graph.
+      def add_source_cookbooks_to_graph
+        sources.each_cookbook { |cookbook| add_artifact_to_graph(cookbook) }
+      end
+
+      # Add any cookbooks defined in the inventory file as a requirement to the graph
+      def add_requirements_to_graph
+        inventory_requirements.each do |requirement|
+          graph.add_requirement(requirement.requirements)
+        end
+      end
+
+      def fetch_inventory
+        resolved_requirements.each do |resolved_requirement|
+          install_cookbook(*resolved_requirement)
+        end
+      end
+
+      def resolved_requirements
+        graph.resolved_requirements
+      end
+
+      def install_cookbook(name, version)
+        if cookbook_already_installed?(name, version)
+          Configuration.output.puts_yellow("cookbook already installed: #{name}-#{version}.")
+          return
+        end
+
+        verify_dependency_can_be_installed(name, version)
+
+        source_cookbook = cookbook_from_source(name, version)
+        source_cookbook.fetch do |cookbook|
+          add_cookbook_to_local_store(cookbook.path, source_cookbook.to_hash)
+        end
+      end
+
+      def cookbook_already_installed?(name, version)
+        local_store.installed?(name, version)
+      end
+
+      def verify_dependency_can_be_installed(name, version)
+        return unless non_required_version?(name, version)
+
+        msg = "The dependency #{name}-#{version} could not be installed."
+        msg << " This is because a cookbook listed in the inventory depends on a version of '#{name}'"
+        msg << " that does not match the explicit requirements for the '#{name}' cookbook."
+        raise Error::BrokenDependency, msg
+      end
+
+      def non_required_version?(name, version)
+        !inventory_requirements.version_required?(name, version)
+      end
+
+      # We need to validate that if this cookbook is in the store
+      # it came from the same source, otherwise we could
+      # possibly override a cookbook
+      def validate_cookbook_against_local_store(cookbook, requirement)
+        local_store.validate_resolved_requirement(cookbook, requirement)
+      end
+
+      def add_cookbook_to_local_store(cookbook_path, data = {})
+        local_store.add_cookbook_from_path(cookbook_path, data)
+      end
+
+      def cookbook_from_source(name, version)
+        sources.find_cookbook(name, version)
+      end
+
+      def add_artifact_to_graph(cookbook)
+        graph.add_artifact(cookbook)
+      end
+
+      def inventory_requirements
+        inventory_configuration.requirements
+      end
+
+      def sources
+        inventory_configuration.sources
+      end
+
+      def display_success_message
+        Configuration.output.puts_green %{Minimart is done building your inventory!}
+        Configuration.output.puts_green %{The inventory can be found in #{local_store.directory_path}}
+      end
+
+      def clear_cache
+        Minimart::Download::GitCache.instance.clear
+      end
+
+    end
+  end
+end

data/lib/minimart/mirror/inventory_configuration.rb

@@ -0,0 +1,74 @@
+require 'yaml'
+require 'erb'
+
+require 'minimart/mirror/inventory_requirements'
+require 'minimart/utils/file_helper'
+
+module Minimart
+  module Mirror
+
+    # This class is responsible for parsing a user defined Minimart configuration file.
+    class InventoryConfiguration
+
+      # The path to the inventory configuration file
+      attr_reader :inventory_config_path
+
+      # @param [String] inventory_config_path The path to the inventory configuration file
+      def initialize(inventory_config_path)
+        @inventory_config_path = inventory_config_path
+        @configuration         = parse_config_file
+        parse_global_configuration
+      end
+
+      # The collection of files defined in the inventory file
+      # @return [Minimart::Mirror::Sources]
+      def sources
+        @sources ||= Sources.new(raw_sources)
+      end
+
+      # The collection of cookbook requirements defined in the inventory file
+      # @return [Array]
+      def requirements
+        @cookbooks ||= InventoryRequirements.new(raw_cookbooks)
+      end
+
+      private
+
+      # The raw parsed configuration file
+      attr_reader :configuration
+
+      def parse_config_file
+        unless Utils::FileHelper.file_exists?(inventory_config_path)
+          raise Error::InvalidInventoryError, 'The inventory configuration file could not be found'
+        end
+
+        file = File.open(inventory_config_path).read
+        erb  = ERB.new(file).result(binding)
+        YAML.load(erb)
+      end
+
+      def parse_global_configuration
+        return unless (conf = configuration['configuration']) && conf.is_a?(Hash)
+
+        Minimart::Configuration.tap do |c|
+          c.chef_server_config = conf.fetch('chef', {})
+          c.github_config      = conf.fetch('github', [])
+          c.verify_ssl         = conf['verify_ssl']
+        end
+      end
+
+      def raw_sources
+        configuration['sources'] || []
+      end
+
+      def raw_cookbooks
+        configuration['cookbooks'].tap do |cookbooks|
+          if cookbooks.nil? || cookbooks.empty?
+            raise Error::InvalidInventoryError, 'Minimart could not find any cookbooks defined in the inventory'
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/minimart/mirror/inventory_requirements.rb

@@ -0,0 +1,104 @@
+require 'minimart/inventory_requirement/supermarket_requirements_builder'
+require 'minimart/inventory_requirement/git_requirements_builder'
+require 'minimart/inventory_requirement/local_requirements_builder'
+
+module Minimart
+  module Mirror
+
+    # The collection of requirements as defined in the inventory file.
+    class InventoryRequirements
+      include Enumerable
+
+      # @return [Hash<String, Hash>] The cookbooks listed in the inventory file
+      attr_reader :raw_cookbooks
+
+      # @param [Hash<String, Hash>] raw_cookbooks The cookbooks listed in the inventory file
+      def initialize(raw_cookbooks)
+        @raw_cookbooks = raw_cookbooks
+        @requirements  = {}
+
+        parse_cookbooks
+      end
+
+      # Iterate over the requirements
+      # @yield [Minimart::Inventory::BaseRequirement]
+      def each(&block)
+        requirements.values.flatten.each &block
+      end
+
+      def each_with_explicit_location(&block)
+        each do |req|
+          block.call(req) if req.explicit_location?
+        end
+      end
+
+      # This method will determine whether or not a version of a cookbook
+      # resolves a constraint defined in the inventory file. This method
+      # can be used to verify that we aren't downloading a non specified version
+      # of a cookbook (e.g. possibly downloading cookbooks that a user wants,
+      # but in versions that they do not)
+      # @param [String] name The name of the cookbook to verify
+      # @param [String] version The cookbook version to check
+      # @return [Boolean] Return true if this version solves a requirement
+      #   as specified in the inventory. This method will also return true for
+      #   any cookbooks not specified in the inventory.
+      def version_required?(name, version)
+        (!has_cookbook?(name)) ||
+          solves_explicit_requirement?(name, version)
+      end
+
+      private
+
+      attr_reader :requirements
+
+      def has_cookbook?(name)
+        requirements.has_key?(name)
+      end
+
+      def solves_explicit_requirement?(name, version)
+        requirements[name].each do |req|
+          next unless req.version_requirement?
+          return true if Semverse::Constraint.new(req.version_requirement).satisfies?(version)
+        end
+        return false
+      end
+
+      # Build Minimart::Inventory requirements from the inventory.
+      def parse_cookbooks
+        raw_cookbooks.each do |name, reqs|
+          @requirements[name] = build_requirements_for(name, (reqs || {}))
+        end
+      end
+
+      def build_requirements_for(name, reqs)
+        get_requirements_for(name, reqs).tap do |parsed_requirements|
+          validate_requirements(name, parsed_requirements)
+        end
+      end
+
+      def get_requirements_for(name, reqs)
+        (market_requirements(name, reqs) +
+          git_requirements(name, reqs) +
+          local_path_requirements(name, reqs)).flatten.compact
+      end
+
+      def validate_requirements(name, reqs)
+        return unless reqs.nil? || reqs.empty?
+        raise Minimart::Error::InvalidInventoryError,
+          "Minimart could not find any requirements for '#{name}'"
+      end
+
+      def market_requirements(name, reqs)
+        InventoryRequirement::SupermarketRequirementsBuilder.new(name, reqs).build
+      end
+
+      def git_requirements(name, reqs)
+        InventoryRequirement::GitRequirementsBuilder.new(name, reqs).build
+      end
+
+      def local_path_requirements(name, reqs)
+        InventoryRequirement::LocalRequirementsBuilder.new(name, reqs).build
+      end
+    end
+  end
+end

data/lib/minimart/mirror/local_store.rb

@@ -0,0 +1,107 @@
+require 'minimart/mirror/download_metadata'
+require 'minimart/utils/file_helper'
+
+module Minimart
+  module Mirror
+
+    # The LocalStore manages what is stored in the local inventory, and adding
+    # cookbook directories to the local inventory. The LocalStore will automatically
+    # load anything that has been stored in the inventory upon initialization.
+    class LocalStore
+
+      # @return [String] the path to the local inventory
+      attr_reader :directory_path
+
+      # @param [String] directory_path The path to the local inventory
+      def initialize(directory_path)
+        @directory_path = directory_path
+        @cookbooks      = {}
+
+        load_local_inventory
+      end
+
+      # :nodoc:
+      def add_cookbook_to_store(name, version)
+        cookbooks[name] ||= []
+        cookbooks[name] << version
+      end
+
+      # Copy a given cookbook to the local store, and record any metadata
+      # about how the cookbook was downloaded.
+      # @param [String] path The path to the cookbook to add to the store
+      # @param [Hash] download_data Any data to record about the cookbook in a Minimart metadata file.
+      def add_cookbook_from_path(path, download_data = {})
+        cookbook_from_path(path).tap do |cookbook|
+          add_cookbook_to_store(cookbook.name, cookbook.version)
+          copy_cookbook(cookbook.path, local_path_for(cookbook))
+          write_download_data(cookbook, download_data)
+        end
+      end
+
+      # Determine whether or not a cookbook has been added to the LocalStore
+      # @return [Boolean]
+      def installed?(cookbook_name, cookbook_version)
+        !!(cookbooks[cookbook_name] &&
+          cookbooks[cookbook_name].include?(cookbook_version))
+      end
+
+      # Validate that a new resolved requirement is not in the local store
+      # with different requirements. If we download two different branches
+      # of the same cookbook and they both resolve to the same version, then we
+      # raise an exception.
+      # @param [Minimart::Cookbook] new_cookbook
+      # @param [Minimart::InventoryRequirement::BaseRequirement] requirement
+      # @raise [Minimart::Error::BrokenDependency]
+      def validate_resolved_requirement(new_cookbook, requirement)
+        return unless installed?(new_cookbook.name, new_cookbook.version)
+
+        existing_cookbook = load_cookbook(new_cookbook)
+        unless requirement.matching_source?(existing_cookbook.download_metadata)
+          raise Minimart::Error::BrokenDependency, "A version of #{new_cookbook} already exists in the inventory from a different source."
+        end
+      end
+
+      private
+
+      attr_reader :cookbooks
+
+      def copy_cookbook(source, destination)
+        FileUtils.rm_rf(destination) if Dir.exists?(destination)
+        FileUtils.cp_r(source, destination)
+
+        # clean destination directory
+        git_dir = File.join(destination, '/.git')
+        FileUtils.rm_rf(git_dir) if Dir.exists?(git_dir)
+      end
+
+      def load_local_inventory
+        Utils::FileHelper.find_cookbooks_in_directory(directory).each do |path|
+          cookbook = cookbook_from_path(path)
+          add_cookbook_to_store(cookbook.name, cookbook.version)
+        end
+      end
+
+      def load_cookbook(cookbook)
+        cookbook_from_path(local_path_for(cookbook))
+      end
+
+      def cookbook_from_path(path)
+        Minimart::Cookbook.from_path(Utils::FileHelper.cookbook_path_in_directory(path))
+      end
+
+      def directory
+        # lazily create the directory to hold the store
+        @directory ||= FileUtils.mkdir_p(directory_path).first
+      end
+
+      def local_path_for(cookbook)
+        File.join(directory, "/#{cookbook}")
+      end
+
+      def write_download_data(cookbook, download_data = {})
+        Minimart::Mirror::DownloadMetadata.new(local_path_for(cookbook)).write(download_data)
+      end
+
+    end
+  end
+end