chef-cli 1.0.3

data/lib/chef-cli/policyfile_services/clean_policies.rb

@@ -0,0 +1,95 @@
+#
+# Copyright:: Copyright (c) 2015 Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require_relative "../exceptions"
+require_relative "../service_exceptions"
+require_relative "../policyfile/lister"
+
+module ChefCLI
+  module PolicyfileServices
+    class CleanPolicies
+
+      Orphan = Struct.new(:policy_name, :revision_id)
+
+      attr_reader :chef_config
+      attr_reader :ui
+
+      def initialize(config: nil, ui: nil)
+        @chef_config = config
+        @ui = ui
+      end
+
+      def run
+        revisions_to_remove = orphaned_policies
+
+        if revisions_to_remove.empty?
+          ui.err("No policy revisions deleted")
+          return true
+        end
+
+        results = revisions_to_remove.map do |policy|
+          [ remove_policy(policy), policy ]
+        end
+
+        failures = results.select { |result, _policy| result.kind_of?(Exception) }
+
+        unless failures.empty?
+          details = failures.map do |result, policy|
+            "- #{policy.policy_name} (#{policy.revision_id}): #{result.class} #{result}"
+          end
+
+          message = "Failed to delete some policy revisions:\n" + details.join("\n") + "\n"
+
+          raise PolicyfileCleanError.new(message, MultipleErrors.new("multiple errors"))
+        end
+
+        true
+      end
+
+      def orphaned_policies
+        policy_lister.policies_by_name.keys.inject([]) do |orphans, policy_name|
+          orphans + policy_lister.orphaned_revisions(policy_name).map do |revision_id|
+            Orphan.new(policy_name, revision_id)
+          end
+        end
+      rescue => e
+        raise PolicyfileCleanError.new("Failed to list policies for cleaning.", e)
+      end
+
+      def policy_lister
+        @policy_lister ||= Policyfile::Lister.new(config: chef_config)
+      end
+
+      def http_client
+        @http_client ||= Chef::ServerAPI.new(chef_config.chef_server_url,
+                                               signing_key_filename: chef_config.client_key,
+                                               client_name: chef_config.node_name)
+      end
+
+      private
+
+      def remove_policy(policy)
+        ui.msg("DELETE #{policy.policy_name} #{policy.revision_id}")
+        http_client.delete("/policies/#{policy.policy_name}/revisions/#{policy.revision_id}")
+        :ok
+      rescue => e
+        e
+      end
+
+    end
+  end
+end

data/lib/chef-cli/policyfile_services/clean_policy_cookbooks.rb

@@ -0,0 +1,123 @@
+#
+# Copyright:: Copyright (c) 2015 Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "set" unless defined?(Set)
+
+require "chef/server_api"
+require_relative "../service_exceptions"
+
+module ChefCLI
+  module PolicyfileServices
+
+    class CleanPolicyCookbooks
+
+      attr_reader :chef_config
+
+      attr_reader :ui
+
+      def initialize(config: nil, ui: nil)
+        @chef_config = config
+        @ui = ui
+
+        @all_cookbooks = nil
+        @active_cookbooks = nil
+        @all_policies = nil
+      end
+
+      def run
+        gc_cookbooks
+      rescue => e
+        raise PolicyCookbookCleanError.new("Failed to cleanup policy cookbooks", e)
+      end
+
+      def gc_cookbooks
+        cookbooks = cookbooks_to_clean
+
+        if cookbooks.empty?
+          ui.msg("No cookbooks deleted.")
+        end
+
+        cookbooks.each do |name, identifiers|
+          identifiers.each do |identifier|
+            http_client.delete("/cookbook_artifacts/#{name}/#{identifier}")
+            ui.msg("DELETE #{name} #{identifier}")
+          end
+        end
+      end
+
+      def all_cookbooks
+        cookbook_list = http_client.get("/cookbook_artifacts")
+        cookbook_list.inject({}) do |cb_map, (name, cb_info)|
+          cb_map[name] = cb_info["versions"].map { |v| v["identifier"] }
+          cb_map
+        end
+      end
+
+      def active_cookbooks
+        policy_revisions_by_name.inject({}) do |cb_map, (policy_name, revision_ids)|
+          revision_ids.each do |revision_id|
+            cookbook_revisions_in_policy(policy_name, revision_id).each do |cb_name, identifier|
+              cb_map[cb_name] ||= Set.new
+              cb_map[cb_name] << identifier
+            end
+          end
+          cb_map
+        end
+      end
+
+      def cookbooks_to_clean
+        active_cbs = active_cookbooks
+
+        all_cookbooks.inject({}) do |cb_map, (cb_name, revisions)|
+          active_revs = active_cbs[cb_name] || Set.new
+          inactive_revs = Set.new(revisions) - active_revs
+          cb_map[cb_name] = inactive_revs unless inactive_revs.empty?
+
+          cb_map
+        end
+      end
+
+      # @api private
+      def policy_revisions_by_name
+        policies_list = http_client.get("/policies")
+        policies_list.inject({}) do |policies_map, (name, policy_info)|
+          policies_map[name] = policy_info["revisions"].keys
+          policies_map
+        end
+      end
+
+      # @api private
+      def cookbook_revisions_in_policy(name, revision_id)
+        policy_revision_data = http_client.get("/policies/#{name}/revisions/#{revision_id}")
+
+        policy_revision_data["cookbook_locks"].inject({}) do |cb_map, (cb_name, lock_info)|
+          cb_map[cb_name] = lock_info["identifier"]
+          cb_map
+        end
+      end
+
+      # @api private
+      # An instance of Chef::ServerAPI configured with the user's
+      # server URL and credentials.
+      def http_client
+        @http_client ||= Chef::ServerAPI.new(chef_config.chef_server_url,
+                                             signing_key_filename: chef_config.client_key,
+                                             client_name: chef_config.node_name)
+      end
+    end
+  end
+end

data/lib/chef-cli/policyfile_services/export_repo.rb

@@ -0,0 +1,419 @@
+#
+# Copyright:: Copyright (c) 2014-2019 Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "pathname" unless defined?(Pathname)
+require "fileutils" unless defined?(FileUtils)
+require "tmpdir" unless defined?(Dir.mktmpdir)
+require "zlib"
+
+require "archive/tar/minitar"
+
+require "chef/cookbook/chefignore"
+
+require_relative "../service_exceptions"
+require_relative "../policyfile_lock"
+require_relative "../policyfile/storage_config"
+
+module ChefCLI
+  module PolicyfileServices
+
+    class ExportRepo
+
+      # Policy groups provide namespaces for policies so that a Chef Infra Server can
+      # have multiple active iterations of a policy at once, but we don't need
+      # this when serving a single exported policy via Chef Zero, so hardcode
+      # it to a "well known" value:
+      POLICY_GROUP = "local".freeze
+
+      include Policyfile::StorageConfigDelegation
+
+      attr_reader :storage_config
+      attr_reader :root_dir
+      attr_reader :export_dir
+
+      def initialize(policyfile: nil, export_dir: nil, root_dir: nil, archive: false, force: false)
+        @root_dir = root_dir
+        @export_dir = File.expand_path(export_dir)
+        @archive = archive
+        @force_export = force
+
+        @policy_data = nil
+        @policyfile_lock = nil
+
+        policyfile_rel_path = policyfile || "Policyfile.rb"
+        policyfile_full_path = File.expand_path(policyfile_rel_path, root_dir)
+        @storage_config = Policyfile::StorageConfig.new.use_policyfile(policyfile_full_path)
+
+        @staging_dir = nil
+      end
+
+      def archive?
+        @archive
+      end
+
+      def policy_name
+        policyfile_lock.name
+      end
+
+      def run
+        assert_lockfile_exists!
+        assert_export_dir_clean!
+
+        validate_lockfile
+        write_updated_lockfile
+        export
+      end
+
+      def policy_data
+        @policy_data ||= FFI_Yajl::Parser.parse(IO.read(policyfile_lock_expanded_path))
+      rescue => error
+        raise PolicyfileExportRepoError.new("Error reading lockfile #{policyfile_lock_expanded_path}", error)
+      end
+
+      def policyfile_lock
+        @policyfile_lock || validate_lockfile
+      end
+
+      def archive_file_location
+        return nil unless archive?
+        filename = "#{policyfile_lock.name}-#{policyfile_lock.revision_id}.tgz"
+        File.join(export_dir, filename)
+      end
+
+      def export
+        with_staging_dir do
+          create_repo_structure
+          copy_cookbooks
+          create_policyfile_repo_item
+          create_policy_group_repo_item
+          copy_policyfile_lock
+          create_client_rb
+          create_readme_md
+          if archive?
+            create_archive
+          else
+            mv_staged_repo
+          end
+        end
+      rescue => error
+        msg = "Failed to export policy (in #{policyfile_filename}) to #{export_dir}"
+        raise PolicyfileExportRepoError.new(msg, error)
+      end
+
+      private
+
+      def with_staging_dir
+        p = Process.pid
+        t = Time.new.utc.strftime("%Y%m%d%H%M%S")
+        Dir.mktmpdir("chefcli-export-#{p}-#{t}") do |d|
+          begin
+            @staging_dir = d
+            yield
+          ensure
+            @staging_dir = nil
+          end
+        end
+      end
+
+      def create_archive
+        Dir.chdir(staging_dir) do
+          targets = Find.find(".").collect { |e| e }
+          Mixlib::Archive.new(archive_file_location).create(targets, gzip: true)
+        end
+      end
+
+      def staging_dir
+        @staging_dir
+      end
+
+      def create_repo_structure
+        FileUtils.mkdir_p(export_dir)
+        FileUtils.mkdir_p(dot_chef_staging_dir)
+        FileUtils.mkdir_p(cookbook_artifacts_staging_dir)
+        FileUtils.mkdir_p(policies_staging_dir)
+        FileUtils.mkdir_p(policy_groups_staging_dir)
+      end
+
+      def copy_cookbooks
+        policyfile_lock.cookbook_locks.each do |name, lock|
+          copy_cookbook(lock)
+        end
+      end
+
+      def copy_cookbook(lock)
+        dirname = "#{lock.name}-#{lock.identifier}"
+        export_path = File.join(staging_dir, "cookbook_artifacts", dirname)
+        metadata_rb_path = File.join(export_path, "metadata.rb")
+        FileUtils.mkdir(export_path) if not File.directory?(export_path)
+        copy_unignored_cookbook_files(lock, export_path)
+        FileUtils.rm_f(metadata_rb_path)
+        metadata = lock.cookbook_version.metadata
+
+        metadata_json_path = File.join(export_path, "metadata.json")
+
+        File.open(metadata_json_path, "wb+") do |f|
+          f.print(FFI_Yajl::Encoder.encode(metadata.to_hash, pretty: true ))
+        end
+      end
+
+      def copy_unignored_cookbook_files(lock, export_path)
+        cookbook_files_to_copy(lock.cookbook_path).each do |rel_path|
+          full_source_path = File.join(lock.cookbook_path, rel_path)
+          full_dest_path = File.join(export_path, rel_path)
+          dest_dirname = File.dirname(full_dest_path)
+          FileUtils.mkdir_p(dest_dirname) unless File.directory?(dest_dirname)
+          FileUtils.cp(full_source_path, full_dest_path)
+        end
+      end
+
+      def cookbook_files_to_copy(cookbook_path)
+        cookbook = cookbook_loader_for(cookbook_path).cookbook_version
+
+        root = Pathname.new(cookbook.root_dir)
+
+        cookbook.all_files.map do |full_path|
+          Pathname.new(full_path).relative_path_from(root).to_s
+        end
+      end
+
+      def cookbook_loader_for(cookbook_path)
+        loader = Chef::Cookbook::CookbookVersionLoader.new(cookbook_path, chefignore_for(cookbook_path))
+        loader.load!
+        loader
+      end
+
+      def chefignore_for(cookbook_path)
+        Chef::Cookbook::Chefignore.new(File.join(cookbook_path, "chefignore"))
+      end
+
+      def create_policyfile_repo_item
+        File.open(policyfile_repo_item_path, "wb+") do |f|
+          f.print(FFI_Yajl::Encoder.encode(policyfile_lock.to_lock, pretty: true ))
+        end
+      end
+
+      def create_policy_group_repo_item
+        data = {
+          "policies" => {
+            policyfile_lock.name => {
+              "revision_id" => policyfile_lock.revision_id,
+            },
+          },
+        }
+
+        File.open(policy_group_repo_item_path, "wb+") do |f|
+          f.print(FFI_Yajl::Encoder.encode(data, pretty: true ))
+        end
+      end
+
+      def copy_policyfile_lock
+        File.open(lockfile_staging_path, "wb+") do |f|
+          f.print(FFI_Yajl::Encoder.encode(policyfile_lock.to_lock, pretty: true ))
+        end
+      end
+
+      def create_client_rb
+        File.open(client_rb_staging_path, "wb+") do |f|
+          f.print( <<~CONFIG )
+            ### Chef Infra Client Configuration ###
+            # The settings in this file will configure chef to apply the exported policy in
+            # this directory. To use it, run:
+            #
+            # chef-client -z
+            #
+
+            policy_name '#{policy_name}'
+            policy_group 'local'
+
+            use_policyfile true
+            policy_document_native_api true
+
+            # In order to use this repo, you need a version of Chef Infra Client and Chef Zero
+            # that supports policyfile "native mode" APIs:
+            current_version = Gem::Version.new(Chef::VERSION)
+            unless Gem::Requirement.new(">= 12.7").satisfied_by?(current_version)
+              puts("!" * 80)
+              puts(<<-MESSAGE)
+            This Chef Repo requires features introduced in Chef Infra Client 12.7, but you are using
+            Chef \#{Chef::VERSION}. Please upgrade to Chef Infra Client 12.7 or later.
+            MESSAGE
+              puts("!" * 80)
+              exit!(1)
+            end
+
+          CONFIG
+        end
+      end
+
+      def create_readme_md
+        File.open(readme_staging_path, "wb+") do |f|
+          f.print( <<~README )
+            # Exported Chef Infra Repository for Policy '#{policy_name}'
+
+            Policy revision: #{policyfile_lock.revision_id}
+
+            This directory contains all the cookbooks and configuration necessary for Chef
+            to converge a system using this exported policy. To converge a system with the
+            exported policy, use a privileged account to run `chef-client -z` from the
+            directory containing the exported policy.
+
+            ## Contents:
+
+            ### Policyfile.lock.json
+
+            A copy of the exported policy, used by the `chef push-archive` command.
+
+            ### .chef/config.rb
+
+            A configuration file for Chef Infra Client. This file configures Chef Infra Client to
+            use the correct `policy_name` and `policy_group` for this exported repository. Chef
+            Infra Client will use this configuration automatically if you've set your working
+            directory properly.
+
+            ### cookbook_artifacts/
+
+            All of the cookbooks required by the policy will be stored in this directory.
+
+            ### policies/
+
+            A different copy of the exported policy, used by the `chef-client` command.
+
+            ### policy_groups/
+
+            Policy groups are used by Chef Infra Server to manage multiple revisions of the same
+            policy. However, exported policies contain only a single policy revision, so
+            this policy group name is hardcoded to "local" and should not be changed.
+
+          README
+        end
+      end
+
+      def mv_staged_repo
+        # If we got here, either these dirs are empty/don't exist or force is
+        # set to true.
+        FileUtils.rm_rf(cookbook_artifacts_dir)
+        FileUtils.rm_rf(policies_dir)
+        FileUtils.rm_rf(policy_groups_dir)
+        FileUtils.rm_rf(dot_chef_dir)
+
+        FileUtils.mv(cookbook_artifacts_staging_dir, export_dir)
+        FileUtils.mv(policies_staging_dir, export_dir)
+        FileUtils.mv(policy_groups_staging_dir, export_dir)
+        FileUtils.mv(lockfile_staging_path, export_dir)
+        FileUtils.mv(dot_chef_staging_dir, export_dir)
+        FileUtils.mv(readme_staging_path, export_dir)
+      end
+
+      def validate_lockfile
+        return @policyfile_lock if @policyfile_lock
+        @policyfile_lock = ChefCLI::PolicyfileLock.new(storage_config).build_from_lock_data(policy_data)
+        # TODO: enumerate any cookbook that have been updated
+        @policyfile_lock.validate_cookbooks!
+        @policyfile_lock
+      rescue PolicyfileExportRepoError
+        raise
+      rescue => error
+        raise PolicyfileExportRepoError.new("Invalid lockfile data", error)
+      end
+
+      def write_updated_lockfile
+        File.open(policyfile_lock_expanded_path, "wb+") do |f|
+          f.print(FFI_Yajl::Encoder.encode(policyfile_lock.to_lock, pretty: true ))
+        end
+      end
+
+      def assert_lockfile_exists!
+        unless File.exist?(policyfile_lock_expanded_path)
+          raise LockfileNotFound, "No lockfile at #{policyfile_lock_expanded_path} - you need to run `install` before `push`"
+        end
+      end
+
+      def assert_export_dir_clean!
+        if !force_export? && !conflicting_fs_entries.empty? && !archive?
+          msg = "Export dir (#{export_dir}) not clean. Refusing to export. (Conflicting files: #{conflicting_fs_entries.join(', ')})"
+          raise ExportDirNotEmpty, msg
+        end
+      end
+
+      def force_export?
+        @force_export
+      end
+
+      def conflicting_fs_entries
+        Dir.glob(File.join(cookbook_artifacts_dir, "*")) +
+          Dir.glob(File.join(policies_dir, "*")) +
+          Dir.glob(File.join(policy_groups_dir, "*")) +
+          Dir.glob(File.join(export_dir, "Policyfile.lock.json"))
+      end
+
+      def cookbook_artifacts_dir
+        File.join(export_dir, "cookbook_artifacts")
+      end
+
+      def policies_dir
+        File.join(export_dir, "policies")
+      end
+
+      def policy_groups_dir
+        File.join(export_dir, "policy_groups")
+      end
+
+      def dot_chef_dir
+        File.join(export_dir, ".chef")
+      end
+
+      def policyfile_repo_item_path
+        basename = "#{policyfile_lock.name}-#{policyfile_lock.revision_id}"
+        File.join(staging_dir, "policies", "#{basename}.json")
+      end
+
+      def policy_group_repo_item_path
+        File.join(staging_dir, "policy_groups", "local.json")
+      end
+
+      def dot_chef_staging_dir
+        File.join(staging_dir, ".chef")
+      end
+
+      def cookbook_artifacts_staging_dir
+        File.join(staging_dir, "cookbook_artifacts")
+      end
+
+      def policies_staging_dir
+        File.join(staging_dir, "policies")
+      end
+
+      def policy_groups_staging_dir
+        File.join(staging_dir, "policy_groups")
+      end
+
+      def lockfile_staging_path
+        File.join(staging_dir, "Policyfile.lock.json")
+      end
+
+      def client_rb_staging_path
+        File.join(dot_chef_staging_dir, "config.rb")
+      end
+
+      def readme_staging_path
+        File.join(staging_dir, "README.md")
+      end
+
+    end
+
+  end
+end