chef-dk 0.10.0 → 0.11.0

data/lib/chef-dk/policyfile_services/export_repo.rb

@@ -21,6 +21,8 @@ require 'zlib'
 
 require 'archive/tar/minitar'
 
+require 'chef/cookbook/chefignore'
+
 require 'chef-dk/service_exceptions'
 require 'chef-dk/policyfile_lock'
 require 'chef-dk/policyfile/storage_config'
@@ -95,9 +97,11 @@ module ChefDK
         with_staging_dir do
           create_repo_structure
           copy_cookbooks
-          create_policyfile_data_item
+          create_policyfile_repo_item
+          create_policy_group_repo_item
           copy_policyfile_lock
           create_client_rb
+          create_readme_md
           if archive?
             create_archive
           else
@@ -138,8 +142,10 @@ module ChefDK
 
       def create_repo_structure
         FileUtils.mkdir_p(export_dir)
-        FileUtils.mkdir_p(cookbooks_staging_dir)
-        FileUtils.mkdir_p(policyfiles_data_bag_staging_dir)
+        FileUtils.mkdir_p(dot_chef_staging_dir)
+        FileUtils.mkdir_p(cookbook_artifacts_staging_dir)
+        FileUtils.mkdir_p(policies_staging_dir)
+        FileUtils.mkdir_p(policy_groups_staging_dir)
       end
 
       def copy_cookbooks
@@ -149,13 +155,13 @@ module ChefDK
       end
 
       def copy_cookbook(lock)
-        dirname = "#{lock.name}-#{lock.dotted_decimal_identifier}"
-        export_path = File.join(staging_dir, "cookbooks", dirname)
+        dirname = "#{lock.name}-#{lock.identifier}"
+        export_path = File.join(staging_dir, "cookbook_artifacts", dirname)
         metadata_rb_path = File.join(export_path, "metadata.rb")
-        FileUtils.cp_r(lock.cookbook_path, export_path)
+        FileUtils.mkdir(export_path) if not File.directory?(export_path)
+        copy_unignored_cookbook_files(lock, export_path)
         FileUtils.rm_f(metadata_rb_path)
         metadata = lock.cookbook_version.metadata
-        metadata.version(lock.dotted_decimal_identifier)
 
         metadata_json_path = File.join(export_path, "metadata.json")
 
@@ -164,23 +170,47 @@ module ChefDK
         end
       end
 
-      def create_policyfile_data_item
-        lock_data = policyfile_lock.to_lock.dup
+      def copy_unignored_cookbook_files(lock, export_path)
+        cookbook_files_to_copy(lock.cookbook_path).each do |rel_path|
+          full_source_path = File.join(lock.cookbook_path, rel_path)
+          full_dest_path = File.join(export_path, rel_path)
+          dest_dirname = File.dirname(full_dest_path)
+          FileUtils.mkdir_p(dest_dirname) unless File.directory?(dest_dirname)
+          FileUtils.cp(full_source_path, full_dest_path)
+        end
+      end
+
+      def cookbook_files_to_copy(cookbook_path)
+        cookbook_loader_for(cookbook_path).cookbook_version.manifest_records_by_path.keys
+      end
 
-        lock_data["id"] = policy_id
+      def cookbook_loader_for(cookbook_path)
+        loader = Chef::Cookbook::CookbookVersionLoader.new(cookbook_path, chefignore_for(cookbook_path))
+        loader.load!
+        loader
+      end
 
-        data_item = {
-          "id" => policy_id,
-          "name" => "data_bag_item_policyfiles_#{policy_id}",
-          "data_bag" => "policyfiles",
-          "raw_data" => lock_data,
-          # we'd prefer to leave this out, but the "compatibility mode"
-          # implementation in chef-client relies on magical class inflation
-          "json_class" => "Chef::DataBagItem"
+      def chefignore_for(cookbook_path)
+        Chef::Cookbook::Chefignore.new(File.join(cookbook_path, "chefignore"))
+      end
+
+      def create_policyfile_repo_item
+        File.open(policyfile_repo_item_path, "wb+") do |f|
+          f.print(FFI_Yajl::Encoder.encode(policyfile_lock.to_lock, pretty: true ))
+        end
+      end
+
+      def create_policy_group_repo_item
+        data = {
+          "policies" => {
+            policyfile_lock.name => {
+              "revision_id" => policyfile_lock.revision_id
+            }
+          }
         }
 
-        File.open(item_path, "wb+") do |f|
-          f.print(FFI_Yajl::Encoder.encode(data_item, pretty: true ))
+        File.open(policy_group_repo_item_path, "wb+") do |f|
+          f.print(FFI_Yajl::Encoder.encode(data, pretty: true ))
         end
       end
 
@@ -197,32 +227,89 @@ module ChefDK
 # The settings in this file will configure chef to apply the exported policy in
 # this directory. To use it, run:
 #
-# chef-client -c client.rb -z
+# chef-client -z
 #
 
-use_policyfile true
+policy_name '#{policy_name}'
+policy_group 'local'
 
-# compatibility mode settings are used because chef-zero doesn't yet support
-# native mode:
-deployment_group '#{policy_name}-local'
-versioned_cookbooks true
-policy_document_native_api false
+use_policyfile true
+policy_document_native_api true
+
+# In order to use this repo, you need a version of Chef Client and Chef Zero
+# that supports policyfile "native mode" APIs:
+current_version = Gem::Version.new(Chef::VERSION)
+unless Gem::Requirement.new(">= 12.7").satisfied_by?(current_version)
+  puts("!" * 80)
+  puts(<<-MESSAGE)
+This Chef Repo requires features introduced in Chef 12.7, but you are using
+Chef \#{Chef::VERSION}. Please upgrade to Chef 12.7 or later.
+MESSAGE
+  puts("!" * 80)
+  exit!(1)
+end
 
 CONFIG
         end
       end
 
+      def create_readme_md
+        File.open(readme_staging_path, "wb+") do |f|
+          f.print( <<-README )
+# Exported Chef Repository for Policy '#{policy_name}'
+
+Policy revision: #{policyfile_lock.revision_id}
+
+This directory contains all the cookbooks and configuration necessary for Chef
+to converge a system using this exported policy. To converge a system with the
+exported policy, use a privileged account to run `chef-client -z` from the
+directory containing the exported policy.
+
+## Contents:
+
+### Policyfile.lock.json
+
+A copy of the exported policy, used by the `chef push-archive` command.
+
+### .chef/config.rb
+
+A configuration file for Chef Client. This file configures Chef Client to use
+the correct `policy_name` and `policy_group` for this exported repository. Chef
+Client will use this configuration automatically if you've set your working
+directory properly.
+
+### cookbook_artifacts/
+
+All of the cookbooks required by the policy will be stored in this directory.
+
+### policies/
+
+A different copy of the exported policy, used by the `chef-client` command.
+
+### policy_groups/
+
+Policy groups are used by Chef Server to manage multiple revisions of the same
+policy. However, exported policies contain only a single policy revision, so
+this policy group name is hardcoded to "local" and should not be changed.
+
+README
+        end
+      end
+
       def mv_staged_repo
         # If we got here, either these dirs are empty/don't exist or force is
         # set to true.
-        FileUtils.rm_rf(cookbooks_dir)
-        FileUtils.rm_rf(policyfiles_data_bag_dir)
-
-        FileUtils.mv(cookbooks_staging_dir, export_dir)
-        FileUtils.mkdir_p(export_data_bag_dir)
-        FileUtils.mv(policyfiles_data_bag_staging_dir, export_data_bag_dir)
+        FileUtils.rm_rf(cookbook_artifacts_dir)
+        FileUtils.rm_rf(policies_dir)
+        FileUtils.rm_rf(policy_groups_dir)
+        FileUtils.rm_rf(dot_chef_dir)
+
+        FileUtils.mv(cookbook_artifacts_staging_dir, export_dir)
+        FileUtils.mv(policies_staging_dir, export_dir)
+        FileUtils.mv(policy_groups_staging_dir, export_dir)
         FileUtils.mv(lockfile_staging_path, export_dir)
-        FileUtils.mv(client_rb_staging_path, export_dir)
+        FileUtils.mv(dot_chef_staging_dir, export_dir)
+        FileUtils.mv(readme_staging_path, export_dir)
       end
 
       def validate_lockfile
@@ -261,37 +348,51 @@ CONFIG
       end
 
       def conflicting_fs_entries
-        Dir.glob(File.join(cookbooks_dir, "*")) +
-          Dir.glob(File.join(policyfiles_data_bag_dir, "*")) +
+        Dir.glob(File.join(cookbook_artifacts_dir, "*")) +
+          Dir.glob(File.join(policies_dir, "*")) +
+          Dir.glob(File.join(policy_groups_dir, "*")) +
           Dir.glob(File.join(export_dir, "Policyfile.lock.json"))
       end
 
-      def cookbooks_dir
-        File.join(export_dir, "cookbooks")
+      def cookbook_artifacts_dir
+        File.join(export_dir, "cookbook_artifacts")
+      end
+
+      def policies_dir
+        File.join(export_dir, "policies")
+      end
+
+      def policy_groups_dir
+        File.join(export_dir, "policy_groups")
+      end
+
+      def dot_chef_dir
+        File.join(export_dir, ".chef")
       end
 
-      def export_data_bag_dir
-        File.join(export_dir, "data_bags")
+      def policyfile_repo_item_path
+        basename = "#{policyfile_lock.name}-#{policyfile_lock.revision_id}"
+        File.join(staging_dir, "policies", "#{basename}.json")
       end
 
-      def policyfiles_data_bag_dir
-        File.join(export_data_bag_dir, "policyfiles")
+      def policy_group_repo_item_path
+        File.join(staging_dir, "policy_groups", "local.json")
       end
 
-      def policy_id
-        "#{policyfile_lock.name}-#{POLICY_GROUP}"
+      def dot_chef_staging_dir
+        File.join(staging_dir, ".chef")
       end
 
-      def item_path
-        File.join(staging_dir, "data_bags", "policyfiles", "#{policy_id}.json")
+      def cookbook_artifacts_staging_dir
+        File.join(staging_dir, "cookbook_artifacts")
       end
 
-      def cookbooks_staging_dir
-        File.join(staging_dir, "cookbooks")
+      def policies_staging_dir
+        File.join(staging_dir, "policies")
       end
 
-      def policyfiles_data_bag_staging_dir
-        File.join(staging_dir, "data_bags", "policyfiles")
+      def policy_groups_staging_dir
+        File.join(staging_dir, "policy_groups")
       end
 
       def lockfile_staging_path
@@ -299,7 +400,11 @@ CONFIG
       end
 
       def client_rb_staging_path
-        File.join(staging_dir, "client.rb")
+        File.join(dot_chef_staging_dir, "config.rb")
+      end
+
+      def readme_staging_path
+        File.join(staging_dir, "README.md")
       end
 
     end

data/lib/chef-dk/policyfile_services/install.rb

@@ -104,6 +104,7 @@ module ChefDK
         ui.msg ""
 
         ui.msg "Lockfile written to #{policyfile_lock_expanded_path}"
+        ui.msg "Policy revision id: #{policyfile_lock.revision_id}"
       rescue => error
         raise PolicyfileInstallError.new("Failed to generate Policyfile.lock", error)
       end

data/lib/chef-dk/policyfile_services/push_archive.rb

@@ -86,12 +86,22 @@ module ChefDK
       def read_policyfile_lock(staging_dir)
         policyfile_lock_path = File.join(staging_dir, "Policyfile.lock.json")
 
+        if looks_like_old_format_archive?(staging_dir)
+          raise InvalidPolicyArchive, <<-MESSAGE
+This archive is in an unsupported format.
+
+This archive was created with an older version of ChefDK. This version of
+ChefDK does not support archives in the older format. Re-create the archive
+with a newer version of ChefDK or downgrade ChefDK.
+MESSAGE
+        end
+
         unless File.exist?(policyfile_lock_path)
           raise InvalidPolicyArchive, "Archive does not contain a Policyfile.lock.json"
         end
 
-        unless File.directory?(File.join(staging_dir, "cookbooks"))
-          raise InvalidPolicyArchive, "Archive does not contain a cookbooks directory"
+        unless File.directory?(File.join(staging_dir, "cookbook_artifacts"))
+          raise InvalidPolicyArchive, "Archive does not contain a cookbook_artifacts directory"
         end
 
 
@@ -167,6 +177,27 @@ module ChefDK
         end
       end
 
+      def looks_like_old_format_archive?(staging_dir)
+        cookbooks_dir = File.join(staging_dir, "cookbooks")
+        data_bags_dir = File.join(staging_dir, "data_bags")
+
+        cookbook_artifacts_dir = File.join(staging_dir, "cookbook_artifacts")
+        policies_dir = File.join(staging_dir, "policies")
+        policy_groups_dir = File.join(staging_dir, "policy_groups")
+
+        # Old archives just had these two dirs
+        have_old_dirs = File.exist?(cookbooks_dir) && File.exist?(data_bags_dir)
+
+        # New archives created by `chef export` will have all of these; it's
+        # also possible we'll encounter an "artisanal" archive, which might
+        # only be missing one of these by accident. In that case we want to
+        # trigger a different error than we're detecting here.
+        have_any_new_dirs = File.exist?(cookbook_artifacts_dir) ||
+          File.exist?(policies_dir) ||
+          File.exist?(policy_groups_dir)
+
+        have_old_dirs && !have_any_new_dirs
+      end
 
     end
   end

data/lib/chef-dk/skeletons/code_generator/files/default/chefignore

@@ -1,5 +1,5 @@
 # Put files/directories that should be ignored in this file when uploading
-# or sharing to the community site.
+# to a chef-server or supermarket.
 # Lines that start with '# ' are comments.
 
 # OS generated files #
@@ -56,6 +56,11 @@ Guardfile
 Procfile
 .kitchen*
 .rubocop.yml
+spec/*
+Rakefile
+.travis.yml
+.foodcritic
+.codeclimate.yml
 
 # SCM #
 #######
@@ -82,6 +87,7 @@ tmp
 CONTRIBUTING*
 CHANGELOG*
 TESTING*
+MAINTAINERS.toml
 
 # Strainer #
 ############
@@ -94,7 +100,3 @@ Strainerfile
 ###########
 .vagrant
 Vagrantfile
-
-# Travis #
-##########
-.travis.yml

data/lib/chef-dk/skeletons/code_generator/files/default/repo/policies/README.md

@@ -13,7 +13,7 @@ This will create a lockfile `policies/my-app-frontend.lock.json`.
 To update locked dependencies, run `chef update` like this:
 
 ```
-chef update policies/my-app-fronend.rb
+chef update policies/my-app-frontend.rb
 ```
 
 You can upload the policy (with associated cookbooks) to the server

data/lib/chef-dk/version.rb

@@ -16,5 +16,5 @@
 #
 
 module ChefDK
-  VERSION = "0.10.0"
+  VERSION = "0.11.0"
 end

data/lib/kitchen/provisioner/policyfile_zero.rb

@@ -62,6 +62,7 @@ module Kitchen
       # Since it makes no sense to modify these, they are hardcoded elsewhere.
       default_config :client_rb, {}
       default_config :json_attributes, true
+      default_config :named_run_list, nil
       default_config :chef_zero_host, nil
       default_config :chef_zero_port, 8889
       default_config :policyfile, "Policyfile.rb"
@@ -111,6 +112,10 @@ module Kitchen
           args << "--logfile #{config[:log_file]}"
         end
 
+        if config[:named_run_list]
+          args << "--named-run-list #{config[:named_run_list]}"
+        end
+
         wrap_shell_code(
           [cmd, *args].join(" ").
           tap { |str| str.insert(0, reload_ps1_path) if windows_os? }
@@ -172,9 +177,9 @@ module Kitchen
 
         data["use_policyfile"] = true
         data["versioned_cookbooks"] = true
-        data["deployment_group"] = "#{policy_exporter.policy_name}-local"
-        # TODO this will need to be updated when chef-zero supports erchef paths (policy_group vs policies)
-        data["policy_document_native_api"] = false
+        data["policy_name"] = policy_exporter.policy_name
+        data["policy_group"] = "local"
+        data["policy_document_native_api"] = true
 
         info("Preparing client.rb")
         debug("Creating client.rb from #{data.inspect}")

data/spec/shared/custom_generator_cookbook.rb

@@ -10,6 +10,9 @@ shared_examples_for "custom generator cookbook" do
     let(:default_generator_cookbook_path) { File.expand_path('lib/chef-dk/skeletons/code_generator', project_root) }
 
     let(:generator_cookbook_path) { File.join(tempdir, 'a_generator_cookbook') }
+    let(:generator_copyright_holder) { 'Chef' }
+    let(:generator_email) { 'mail@chef.io' }
+    let(:generator_license) { 'Free as in Beer'}
 
     let(:argv) { [generator_arg, "--generator-cookbook", generator_cookbook_path] }
 
@@ -32,7 +35,18 @@ shared_examples_for "custom generator cookbook" do
 
       let(:argv) { [generator_arg] }
 
-      let(:chefdk_config) { double("Mixlib::Config context for ChefDK", generator_cookbook: generator_cookbook_path) }
+      let(:generator_config) do
+        double("Generator Config Context",
+          license: generator_license,
+          copyright_holder: generator_copyright_holder,
+          email: generator_email)
+      end
+
+      let(:chefdk_config) do
+        double("Mixlib::Config context for ChefDK",
+          generator_cookbook: generator_cookbook_path,
+          generator: generator_config)
+      end
 
       before do
         allow(code_generator).to receive(:chefdk_config).and_return(chefdk_config)
@@ -114,4 +128,3 @@ shared_examples_for "custom generator cookbook" do
     end
   end
 end
-