chef-cli 1.0.3

data/lib/chef-cli/policyfile/source_uri.rb

@@ -0,0 +1,57 @@
+#
+# Copyright:: Copyright (c) 2016 Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "addressable/uri" unless defined?(Addressable::URI)
+require_relative "../exceptions"
+
+module ChefCLI
+  module Policyfile
+    class SourceURI < Addressable::URI
+      class << self
+        # Returns a URI object based on the parsed string.
+        #
+        # @param [String, Addressable::URI, #to_str] uri The URI
+        # string to parse.  No parsing is performed if the object
+        # is already an <code>Addressable::URI</code>.
+        #
+        # @raise [ChefCLI::InvalidPolicyfileSourceURI]
+        #
+        # @return [ChefCLI::Policyfile::SourceURI]
+        def parse(uri)
+          parsed_uri = super(uri)
+          parsed_uri.send(:validate)
+          parsed_uri
+        rescue TypeError, ArgumentError => ex
+          raise ChefCLI::InvalidPolicyfileSourceURI.new(uri, ex)
+        end
+      end
+
+      VALID_SCHEMES = %w{ https http }.freeze
+
+      # @raise [ChefCLI::InvalidPolicyfileSourceURI]
+      def validate
+        super
+
+        unless VALID_SCHEMES.include?(scheme)
+          raise InvalidPolicyfileSourceURI.new(self, "invalid URI scheme '#{scheme}'. Valid schemes: #{VALID_SCHEMES}")
+        end
+      rescue Addressable::URI::InvalidURIError => ex
+        raise InvalidPolicyfileSourceURI.new(self, ex)
+      end
+    end
+  end
+end

data/lib/chef-cli/policyfile/storage_config.rb

@@ -0,0 +1,112 @@
+#
+# Copyright:: Copyright (c) 2014-2018 Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require_relative "../cookbook_omnifetch"
+require_relative "../exceptions"
+
+module ChefCLI
+  module Policyfile
+
+    # Provides handling of relative paths, such as on disk cookbooks which
+    # are specified relative to the Policyfile and local caching of compiled
+    # Policyfile artifacts.
+    #
+    # @author Dan DeLeo
+    # @since 0.11.0
+    # @attr_accessor [String] relative_paths_root Root path relative to the named Policyfile
+    # @attr_accessor [String] cache_path Local cache directory for
+    # @attr_reader [String] policyfile_filename Uncompiled Policyfile filename.
+    # @attr_reader [String] policyfile_lock_filename Filename of compiled Policyfile.
+    class StorageConfig
+
+      attr_accessor :relative_paths_root
+      attr_accessor :cache_path
+
+      attr_reader :policyfile_filename
+      attr_reader :policyfile_lock_filename
+
+      def initialize(options = {})
+        @relative_paths_root = Dir.pwd
+        @cache_path = CookbookOmnifetch.storage_path
+        @policyfile_filename = "<< Policyfile filename not specified >>"
+        @policyfile_lock_filename = "<< Policyfile lock filename not specified >>"
+        handle_options(options)
+      end
+
+      def use_policyfile(policyfile_filename)
+        if policyfile_filename.end_with?(".lock.json")
+          return use_policyfile_lock(policyfile_filename)
+        end
+        unless policyfile_filename.end_with?(".rb")
+          raise InvalidPolicyfileFilename, "Policyfile filenames must end with `.rb' extension (you gave: `#{policyfile_filename}')"
+        end
+        @policyfile_filename = policyfile_filename
+        @policyfile_lock_filename = policyfile_filename.sub(/\.rb\Z/, ".lock.json")
+        @relative_paths_root = File.dirname(policyfile_filename)
+        self
+      end
+
+      def use_policyfile_lock(policyfile_lock_filename)
+        @policyfile_lock_filename = policyfile_lock_filename
+        @policyfile_filename = policyfile_lock_filename.sub(/\.lock\.json\Z/, ".rb")
+        @relative_paths_root = File.dirname(policyfile_lock_filename)
+        self
+      end
+
+      def policyfile_expanded_path
+        File.expand_path(policyfile_filename, relative_paths_root)
+      end
+
+      def policyfile_lock_expanded_path
+        File.expand_path(policyfile_lock_filename, relative_paths_root)
+      end
+
+      private
+
+      def handle_options(options)
+        @cache_path = options[:cache_path] if options[:cache_path]
+        @relative_paths_root = options[:relative_paths_root] if options.key?(:relative_paths_root)
+      end
+
+    end
+
+    module StorageConfigDelegation
+
+      def cache_path
+        storage_config.cache_path
+      end
+
+      def relative_paths_root
+        storage_config.relative_paths_root
+      end
+
+      def policyfile_filename
+        storage_config.policyfile_filename
+      end
+
+      def policyfile_expanded_path
+        storage_config.policyfile_expanded_path
+      end
+
+      def policyfile_lock_expanded_path
+        storage_config.policyfile_lock_expanded_path
+      end
+
+    end
+
+  end
+end

data/lib/chef-cli/policyfile/undo_record.rb

@@ -0,0 +1,139 @@
+#
+# Copyright:: Copyright (c) 2015 Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require_relative "../exceptions"
+
+module ChefCLI
+  module Policyfile
+
+    class UndoRecord
+
+      PolicyGroupRestoreData = Struct.new(:policy_name, :policy_group, :data) do
+
+        def load(data)
+          self.policy_name = data["policy_name"]
+          self.policy_group = data["policy_group"]
+          self.data = data["data"]
+          self
+        end
+
+        def for_serialization
+          {
+            "policy_name" => policy_name,
+            "policy_group" => policy_group,
+            "data" => data,
+          }
+        end
+
+      end
+
+      attr_reader :policy_groups
+
+      attr_reader :policy_revisions
+
+      attr_accessor :description
+
+      def initialize
+        reset!
+      end
+
+      def ==(other)
+        other.kind_of?(UndoRecord) &&
+          other.policy_groups == policy_groups &&
+          other.policy_revisions == policy_revisions
+      end
+
+      def add_policy_group(name)
+        @policy_groups << name
+      end
+
+      def add_policy_revision(policy_name, policy_group, data)
+        @policy_revisions << PolicyGroupRestoreData.new(policy_name, policy_group, data)
+      end
+
+      def load(data)
+        reset!
+
+        unless data.kind_of?(Hash)
+          raise InvalidUndoRecord, "Undo data is incorrectly formatted. Must be a Hash, got '#{data}'."
+        end
+        missing_fields = %w{ format_version description backup_data }.select { |key| !data.key?(key) }
+        unless missing_fields.empty?
+          raise InvalidUndoRecord, "Undo data is missing mandatory field(s) #{missing_fields.join(', ')}. Undo data: '#{data}'"
+        end
+
+        @description = data["description"]
+
+        policy_data = data["backup_data"]
+        unless policy_data.kind_of?(Hash)
+          raise InvalidUndoRecord, "'backup_data' in the undo record is incorrectly formatted. Must be a Hash, got '#{policy_data}'"
+        end
+        missing_policy_data_fields = %w{ policy_groups policy_revisions }.select { |key| !policy_data.key?(key) }
+        unless missing_policy_data_fields.empty?
+          raise InvalidUndoRecord,
+            "'backup_data' in the undo record is missing mandatory field(s) #{missing_policy_data_fields.join(', ')}. Backup data: #{policy_data}"
+        end
+
+        policy_groups = policy_data["policy_groups"]
+
+        unless policy_groups.kind_of?(Array)
+          raise InvalidUndoRecord,
+            "'policy_groups' data in the undo record is incorrectly formatted. Must be an Array, got '#{policy_groups}'"
+        end
+
+        @policy_groups = policy_groups
+
+        policy_revisions = policy_data["policy_revisions"]
+        unless policy_revisions.kind_of?(Array)
+          raise InvalidUndoRecord,
+            "'policy_revisions' data in the undo record is incorrectly formatted. Must be an Array, got '#{policy_revisions}'"
+        end
+
+        policy_revisions.each do |revision|
+          unless revision.kind_of?(Hash)
+            raise InvalidUndoRecord,
+              "Invalid item in 'policy_revisions' in the undo record. Must be a Hash, got '#{revision}'"
+          end
+
+          @policy_revisions << PolicyGroupRestoreData.new.load(revision)
+        end
+
+        self
+      end
+
+      def for_serialization
+        {
+          "format_version" => 1,
+          "description" => description,
+          "backup_data" => {
+            "policy_groups" => policy_groups,
+            "policy_revisions" => policy_revisions.map(&:for_serialization),
+          },
+        }
+      end
+
+      private
+
+      def reset!
+        @description = ""
+        @policy_groups = []
+        @policy_revisions = []
+      end
+
+    end
+  end
+end

data/lib/chef-cli/policyfile/undo_stack.rb

@@ -0,0 +1,128 @@
+#
+# Copyright:: Copyright (c) 2015 Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "fileutils" unless defined?(FileUtils)
+
+require "ffi_yajl" unless defined?(FFI_Yajl)
+
+require_relative "../helpers"
+require_relative "undo_record"
+
+module ChefCLI
+  module Policyfile
+
+    class UndoStack
+
+      MAX_SIZE = 10
+
+      include Helpers
+
+      def undo_dir
+        File.join(Helpers.package_home, "undo")
+      end
+
+      def size
+        undo_record_files.size
+      end
+
+      def empty?
+        size == 0
+      end
+
+      def has_id?(id)
+        File.exist?(undo_file_for(id))
+      end
+
+      def each_with_id
+        undo_record_files.each do |filename|
+          yield File.basename(filename), load_undo_record(filename)
+        end
+      end
+
+      def undo_records
+        undo_record_files.map { |f| load_undo_record(f) }
+      end
+
+      def push(undo_record)
+        ensure_undo_dir_exists
+
+        record_id = Time.new.utc.strftime("%Y%m%d%H%M%S")
+        path = File.join(undo_dir, record_id)
+
+        with_file(path) do |f|
+          f.print(FFI_Yajl::Encoder.encode(undo_record.for_serialization, pretty: true))
+        end
+
+        records_to_delete = undo_record_files.size - MAX_SIZE
+        if records_to_delete > 0
+          undo_record_files.take(records_to_delete).each do |file|
+            File.unlink(file)
+          end
+        end
+
+        self
+      end
+
+      def pop
+        file_to_pop = undo_record_files.last
+        if file_to_pop.nil?
+          raise CantUndo, "No undo records exist in #{undo_dir}"
+        end
+
+        record = load_undo_record(file_to_pop)
+        # if this hits an exception, we skip unlink
+        yield record if block_given?
+        File.unlink(file_to_pop)
+        record
+      end
+
+      def delete(id)
+        undo_file = undo_file_for(id)
+        unless File.exist?(undo_file)
+          raise UndoRecordNotFound, "No undo record for id '#{id}' exists at #{undo_file}"
+        end
+
+        record = load_undo_record(undo_file)
+        yield record if block_given?
+        File.unlink(undo_file)
+        record
+      end
+
+      private
+
+      def undo_file_for(id)
+        File.join(undo_dir, id)
+      end
+
+      def load_undo_record(file)
+        data = FFI_Yajl::Parser.parse(IO.read(file))
+        UndoRecord.new.load(data)
+      end
+
+      def undo_record_files
+        Dir[File.join(undo_dir, "*")].sort
+      end
+
+      def ensure_undo_dir_exists
+        return false if File.directory?(undo_dir)
+
+        FileUtils.mkdir_p(undo_dir)
+      end
+    end
+
+  end
+end

data/lib/chef-cli/policyfile/uploader.rb

@@ -0,0 +1,222 @@
+#
+# Copyright:: Copyright (c) 2014-2018 Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "chef/cookbook_uploader"
+require_relative "read_cookbook_for_compat_mode_upload"
+
+require_relative "../ui"
+require_relative "reports/upload"
+
+module ChefCLI
+  module Policyfile
+    class Uploader
+
+      LockedCookbookForUpload = Struct.new(:cookbook, :lock)
+
+      COMPAT_MODE_DATA_BAG_NAME = "policyfiles".freeze
+
+      attr_reader :policyfile_lock
+      attr_reader :policy_group
+      attr_reader :http_client
+      attr_reader :ui
+
+      def initialize(policyfile_lock, policy_group, ui: nil, http_client: nil, policy_document_native_api: false)
+        @policyfile_lock = policyfile_lock
+        @policy_group = policy_group
+        @http_client = http_client
+        @ui = ui || UI.null
+        @policy_document_native_api = policy_document_native_api
+
+        @policy_lock_for_transport = nil
+        @cookbook_versions_for_policy = nil
+      end
+
+      def policy_name
+        policyfile_lock.name
+      end
+
+      def upload
+        ui.msg("Uploading policy #{policy_name} (#{short_revision_id}) to policy group #{policy_group}")
+
+        if !using_policy_document_native_api?
+          ui.msg(<<~DRAGONS)
+            WARN: Uploading policy to policy group #{policy_group} in compatibility mode.
+            Cookbooks will be uploaded with very large version numbers, which may be picked
+            up by existing nodes.
+          DRAGONS
+        end
+
+        upload_cookbooks
+        upload_policy
+      end
+
+      def upload_policy
+        if using_policy_document_native_api?
+          upload_policy_native
+        else
+          data_bag_create
+          data_bag_item_create
+        end
+      end
+
+      def upload_policy_native
+        http_client.put("/policy_groups/#{policy_group}/policies/#{policy_name}", policy_lock_for_transport)
+      end
+
+      def data_bag_create
+        http_client.post("data", { "name" => COMPAT_MODE_DATA_BAG_NAME })
+      rescue Net::HTTPServerException => e
+        raise e unless e.response.code == "409"
+      end
+
+      def data_bag_item_create
+        policy_id = "#{policy_name}-#{policy_group}"
+        lock_data = policy_lock_for_transport.dup
+
+        lock_data["id"] = policy_id
+
+        data_item = {
+          "id" => policy_id,
+          "name" => "data_bag_item_#{COMPAT_MODE_DATA_BAG_NAME}_#{policy_id}",
+          "data_bag" => COMPAT_MODE_DATA_BAG_NAME,
+          "raw_data" => lock_data,
+          # we'd prefer to leave this out, but the "compatibility mode"
+          # implementation in chef-client relies on magical class inflation
+          "json_class" => "Chef::DataBagItem",
+        }
+
+        upload_lockfile_as_data_bag_item(policy_id, data_item)
+        ui.msg("Policy uploaded as data bag item #{COMPAT_MODE_DATA_BAG_NAME}/#{policy_id}")
+        true
+      end
+
+      def uploader
+        # TODO: uploader runs cookbook validation; we want to do this at a different time.
+        @uploader ||= Chef::CookbookUploader.new(cookbook_versions_to_upload, rest: http_client, policy_mode: using_policy_document_native_api?)
+      end
+
+      def cookbook_versions_to_upload
+        cookbook_versions_for_policy.inject([]) do |versions_to_upload, cookbook_with_lock|
+          cb = cookbook_with_lock.cookbook
+          # When we abandon custom identifier support in favor of the one true
+          # hash, identifier generation code can be moved into chef proper and
+          # this can be removed.
+          cb.identifier = cookbook_with_lock.lock.identifier
+
+          versions_to_upload << cb unless remote_already_has_cookbook?(cb)
+          versions_to_upload
+        end
+      end
+
+      def remote_already_has_cookbook?(cookbook)
+        return false unless existing_cookbook_on_remote.key?(cookbook.name.to_s)
+
+        if using_policy_document_native_api?
+          native_mode_cookbook_exists_on_remote?(cookbook)
+        else
+          compat_mode_cookbook_exists_on_remote?(cookbook)
+        end
+      end
+
+      def native_mode_cookbook_exists_on_remote?(cookbook)
+        existing_cookbook_on_remote[cookbook.name.to_s]["versions"].any? do |cookbook_info|
+          cookbook_info["identifier"] == cookbook.identifier
+        end
+      end
+
+      def compat_mode_cookbook_exists_on_remote?(cookbook)
+        existing_cookbook_on_remote[cookbook.name.to_s]["versions"].any? do |cookbook_info|
+          cookbook_info["version"] == cookbook.version
+        end
+      end
+
+      def existing_cookbook_on_remote
+        @existing_cookbook_on_remote ||= http_client.get(list_cookbooks_url)
+      end
+
+      # An Array of Chef::CookbookVersion objects representing the full set that
+      # the policyfile lock requires.
+      def cookbook_versions_for_policy
+        return @cookbook_versions_for_policy if @cookbook_versions_for_policy
+        policyfile_lock.validate_cookbooks!
+        @cookbook_versions_for_policy =
+          if using_policy_document_native_api?
+            load_cookbooks_in_native_mode
+          else
+            load_cookbooks_in_compat_mode
+          end
+      end
+
+      def load_cookbooks_in_native_mode
+        policyfile_lock.cookbook_locks.map do |name, lock|
+          cb = CookbookLoaderWithChefignore.load(name, lock.cookbook_path)
+          LockedCookbookForUpload.new(cb, lock)
+        end
+      end
+
+      def load_cookbooks_in_compat_mode
+        policyfile_lock.cookbook_locks.map do |name, lock|
+          cb = ReadCookbookForCompatModeUpload.load(name, lock.dotted_decimal_identifier, lock.cookbook_path)
+          LockedCookbookForUpload.new(cb, lock)
+        end
+      end
+
+      def using_policy_document_native_api?
+        @policy_document_native_api
+      end
+
+      private
+
+      def short_revision_id
+        policy_lock_for_transport["revision_id"][0, 10]
+      end
+
+      def policy_lock_for_transport
+        @policy_lock_for_transport ||= policyfile_lock.to_lock
+      end
+
+      def list_cookbooks_url
+        if using_policy_document_native_api?
+          "cookbook_artifacts?num_versions=all"
+        else
+          "cookbooks?num_versions=all"
+        end
+      end
+
+      def upload_cookbooks
+        ui.msg("WARN: Uploading cookbooks using semver compat mode") unless using_policy_document_native_api?
+
+        uploader.upload_cookbooks unless cookbook_versions_to_upload.empty?
+
+        reused_cbs, uploaded_cbs = cookbook_versions_for_policy.partition do |cb_with_lock|
+          remote_already_has_cookbook?(cb_with_lock.cookbook)
+        end
+
+        Reports::Upload.new(reused_cbs: reused_cbs, uploaded_cbs: uploaded_cbs, ui: ui).show
+
+        true
+      end
+
+      def upload_lockfile_as_data_bag_item(policy_id, data_item)
+        http_client.put("data/#{COMPAT_MODE_DATA_BAG_NAME}/#{policy_id}", data_item)
+      rescue Net::HTTPServerException => e
+        raise e unless e.response.code == "404"
+        http_client.post("data/#{COMPAT_MODE_DATA_BAG_NAME}", data_item)
+      end
+    end
+  end
+end