chef-dk 0.6.2 → 0.7.0

data/lib/chef-dk/policyfile/null_cookbook_source.rb

@@ -32,6 +32,14 @@ module ChefDK
         raise UnsupportedFeature, 'You must set a default_source in your Policyfile to download cookbooks without explicit sources'
       end
 
+      def null?
+        true
+      end
+
+      def desc
+        "null_cookbook_source"
+      end
+
     end
   end
 end

data/lib/chef-dk/policyfile_compiler.rb

@@ -24,6 +24,7 @@ require 'chef-dk/policyfile/dsl'
 require 'chef-dk/policyfile_lock'
 require 'chef-dk/ui'
 require 'chef-dk/policyfile/reports/install'
+require 'chef-dk/exceptions'
 
 module ChefDK
 
@@ -59,6 +60,8 @@ module ChefDK
       @dsl = Policyfile::DSL.new(storage_config)
       @artifact_server_cookbook_location_specs = {}
 
+      @merged_graph = nil
+
       @ui = ui || UI.null
       @install_report = Policyfile::Reports::Install.new(ui: @ui, policyfile_compiler: self)
     end
@@ -123,7 +126,11 @@ module ChefDK
     end
 
     def create_spec_for_cookbook(cookbook_name, version)
-      source_options = default_source.source_options_for(cookbook_name, version)
+      matching_source = default_source.find { |s|
+        s.universe_graph.has_key?(cookbook_name)
+      }
+
+      source_options = matching_source.source_options_for(cookbook_name, version)
       spec = Policyfile::CookbookLocationSpecification.new(cookbook_name, "= #{version}", source_options, storage_config)
       @artifact_server_cookbook_location_specs[cookbook_name] = spec
     end
@@ -208,9 +215,21 @@ module ChefDK
     end
 
     def remote_artifacts_graph
-      default_source.universe_graph
+      @merged_graph ||=
+        begin
+          conflicting_cb_names = []
+          merged = {}
+          default_source.each do |source|
+            merged.merge!(source.universe_graph) do |conflicting_cb_name, _old, _new|
+              conflicting_cb_names << conflicting_cb_name
+            end
+          end
+          handle_conflicting_cookbooks(conflicting_cb_names)
+          merged
+        end
     end
 
+
     def version_constraint_for(cookbook_name)
       if (cookbook_location_spec = cookbook_location_spec_for(cookbook_name)) and cookbook_location_spec.version_fixed?
         version = cookbook_location_spec.version
@@ -285,5 +304,19 @@ module ChefDK
       CookbookOmnifetch.storage_path
     end
 
+    def handle_conflicting_cookbooks(conflicting_cookbooks)
+      # ignore any cookbooks that have a source set.
+      cookbooks_wo_source = conflicting_cookbooks.select do |cookbook_name|
+        location_spec = cookbook_location_spec_for(cookbook_name)
+        location_spec.nil? || location_spec.source_options.empty?
+      end
+
+      if cookbooks_wo_source.empty?
+        nil
+      else
+        raise CookbookSourceConflict.new(cookbooks_wo_source, default_source)
+      end
+    end
+
   end
 end

data/lib/chef-dk/policyfile_lock.rb

@@ -261,6 +261,15 @@ module ChefDK
       self
     end
 
+    def build_from_archive(lock_data)
+      set_name_from_lock_data(lock_data)
+      set_run_list_from_lock_data(lock_data)
+      set_cookbook_locks_as_archives_from_lock_data(lock_data)
+      set_attributes_from_lock_data(lock_data)
+      set_solution_dependencies_from_lock_data(lock_data)
+      self
+    end
+
     def install_cookbooks
       # note: duplicates PolicyfileCompiler#ensure_cache_dir_exists
       ensure_cache_dir_exists
@@ -423,6 +432,22 @@ module ChefDK
       end
     end
 
+    def set_cookbook_locks_as_archives_from_lock_data(lock_data)
+      cookbook_lock_data = lock_data["cookbook_locks"]
+
+      if cookbook_lock_data.nil?
+        raise InvalidLockfile, "lockfile does not have a cookbook_locks attribute"
+      end
+
+      unless cookbook_lock_data.kind_of?(Hash)
+        raise InvalidLockfile, "lockfile's cookbook_locks attribute must be a Hash (JSON object). (got: #{cookbook_lock_data.inspect})"
+      end
+
+      lock_data["cookbook_locks"].each do |name, lock_info|
+        build_cookbook_lock_as_archive_from_lock_data(name, lock_info)
+      end
+    end
+
     def set_attributes_from_lock_data(lock_data)
       default_attr_data = lock_data["default_attributes"]
 
@@ -475,5 +500,23 @@ module ChefDK
       end
     end
 
+    def build_cookbook_lock_as_archive_from_lock_data(name, lock_info)
+      unless lock_info.kind_of?(Hash)
+        raise InvalidLockfile, "lockfile cookbook_locks entries must be a Hash (JSON object). (got: #{lock_info.inspect})"
+      end
+
+      if lock_info["cache_key"].nil?
+        local_cookbook = Policyfile::LocalCookbook.new(name, storage_config)
+        local_cookbook.build_from_lock_data(lock_info)
+        archived = Policyfile::ArchivedCookbook.new(local_cookbook, storage_config)
+        @cookbook_locks[name] = archived
+      else
+        cached_cookbook = Policyfile::CachedCookbook.new(name, storage_config)
+        cached_cookbook.build_from_lock_data(lock_info)
+        archived = Policyfile::ArchivedCookbook.new(cached_cookbook, storage_config)
+        @cookbook_locks[name] = archived
+      end
+    end
+
   end
 end

data/lib/chef-dk/policyfile_services/clean_policies.rb

@@ -0,0 +1,94 @@
+#
+# Copyright:: Copyright (c) 2015 Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef-dk/service_exceptions'
+require 'chef-dk/policyfile/lister'
+
+module ChefDK
+  module PolicyfileServices
+    class CleanPolicies
+
+      Orphan = Struct.new(:policy_name, :revision_id)
+
+      attr_reader :chef_config
+      attr_reader :ui
+
+      def initialize(config: nil, ui: nil)
+        @chef_config = config
+        @ui = ui
+      end
+
+      def run
+        revisions_to_remove = orphaned_policies
+
+        if revisions_to_remove.empty?
+          ui.err("No policy revisions deleted")
+          return true
+        end
+
+        results = revisions_to_remove.map do |policy|
+          [ remove_policy(policy), policy ]
+        end
+
+        failures = results.select { |result, _policy| result.kind_of?(Exception) }
+
+        unless failures.empty?
+          details = failures.map do |result, policy|
+            "- #{policy.policy_name} (#{policy.revision_id}): #{result.class} #{result}"
+          end
+
+          message = "Failed to delete some policy revisions:\n" + details.join("\n") + "\n"
+
+          raise PolicyfileCleanError.new(message, nil)
+        end
+
+        true
+      end
+
+      def orphaned_policies
+        policy_lister.policies_by_name.keys.inject([]) do |orphans, policy_name|
+          orphans + policy_lister.orphaned_revisions(policy_name).map do |revision_id|
+            Orphan.new(policy_name, revision_id)
+          end
+        end
+      rescue => e
+        raise PolicyfileCleanError.new("Failed to list policies for cleaning.", e)
+      end
+
+      def policy_lister
+        @policy_lister ||= Policyfile::Lister.new(config: chef_config)
+      end
+
+      def http_client
+        @http_client ||= ChefDK::AuthenticatedHTTP.new(config.chef_server_url,
+                                                       signing_key_filename: config.client_key,
+                                                       client_name: config.node_name)
+      end
+
+      private
+
+      def remove_policy(policy)
+        ui.msg("DELETE #{policy.policy_name} #{policy.revision_id}")
+        http_client.delete("/policies/#{policy.policy_name}/revisions/#{policy.revision_id}")
+        :ok
+      rescue => e
+        e
+      end
+
+    end
+  end
+end

data/lib/chef-dk/policyfile_services/export_repo.rb

@@ -16,6 +16,10 @@
 #
 
 require 'fileutils'
+require 'tmpdir'
+require 'zlib'
+
+require 'archive/tar/minitar'
 
 require 'chef-dk/service_exceptions'
 require 'chef-dk/policyfile_lock'
@@ -38,9 +42,10 @@ module ChefDK
       attr_reader :root_dir
       attr_reader :export_dir
 
-      def initialize(policyfile: nil, export_dir: nil, root_dir: nil, force: false)
+      def initialize(policyfile: nil, export_dir: nil, root_dir: nil, archive: false, force: false)
         @root_dir = root_dir
         @export_dir = File.expand_path(export_dir)
+        @archive = archive
         @force_export = force
 
         @policy_data = nil
@@ -49,6 +54,12 @@ module ChefDK
         policyfile_rel_path = policyfile || "Policyfile.rb"
         policyfile_full_path = File.expand_path(policyfile_rel_path, root_dir)
         @storage_config = Policyfile::StorageConfig.new.use_policyfile(policyfile_full_path)
+
+        @staging_dir = nil
+      end
+
+      def archive?
+        @archive
       end
 
       def policy_name
@@ -74,10 +85,24 @@ module ChefDK
         @policyfile_lock || validate_lockfile
       end
 
+      def archive_file_location
+        return nil unless archive?
+        filename = "#{policyfile_lock.name}-#{policyfile_lock.revision_id}.tgz"
+        File.join(export_dir, filename)
+      end
+
       def export
-        create_repo_structure
-        copy_cookbooks
-        create_policyfile_data_item
+        with_staging_dir do
+          create_repo_structure
+          copy_cookbooks
+          create_policyfile_data_item
+          copy_policyfile_lock
+          if archive?
+            create_archive
+          else
+            mv_staged_repo
+          end
+        end
       rescue => error
         msg = "Failed to export policy (in #{policyfile_filename}) to #{export_dir}"
         raise PolicyfileExportRepoError.new(msg, error)
@@ -85,13 +110,35 @@ module ChefDK
 
       private
 
-      def create_repo_structure
-        FileUtils.rm_rf(cookbooks_dir)
-        FileUtils.rm_rf(policyfiles_data_bag_dir)
+      def with_staging_dir
+        p = Process.pid
+        t = Time.new.utc.strftime("%Y%m%d%H%M%S")
+        Dir.mktmpdir("chefdk-export-#{p}-#{t}") do |d|
+          begin
+            @staging_dir = d
+            yield
+          ensure
+            @staging_dir = nil
+          end
+        end
+      end
+
+      def create_archive
+        Zlib::GzipWriter.open(archive_file_location) do |gz_file|
+          Dir.chdir(staging_dir) do
+            Archive::Tar::Minitar.pack(".", gz_file)
+          end
+        end
+      end
 
+      def staging_dir
+        @staging_dir
+      end
+
+      def create_repo_structure
         FileUtils.mkdir_p(export_dir)
-        FileUtils.mkdir_p(cookbooks_dir)
-        FileUtils.mkdir_p(policyfiles_data_bag_dir)
+        FileUtils.mkdir_p(cookbooks_staging_dir)
+        FileUtils.mkdir_p(policyfiles_data_bag_staging_dir)
       end
 
       def copy_cookbooks
@@ -102,7 +149,7 @@ module ChefDK
 
       def copy_cookbook(lock)
         dirname = "#{lock.name}-#{lock.dotted_decimal_identifier}"
-        export_path = File.join(export_dir, "cookbooks", dirname)
+        export_path = File.join(staging_dir, "cookbooks", dirname)
         metadata_rb_path = File.join(export_path, "metadata.rb")
         FileUtils.cp_r(lock.cookbook_path, export_path)
         FileUtils.rm_f(metadata_rb_path)
@@ -117,9 +164,6 @@ module ChefDK
       end
 
       def create_policyfile_data_item
-        policy_id = "#{policyfile_lock.name}-#{POLICY_GROUP}"
-        item_path = File.join(export_dir, "data_bags", "policyfiles", "#{policy_id}.json")
-
         lock_data = policyfile_lock.to_lock.dup
 
         lock_data["id"] = policy_id
@@ -139,6 +183,24 @@ module ChefDK
         end
       end
 
+      def copy_policyfile_lock
+        File.open(lockfile_staging_path, "wb+") do |f|
+          f.print(FFI_Yajl::Encoder.encode(policyfile_lock.to_lock, pretty: true ))
+        end
+      end
+
+      def mv_staged_repo
+        # If we got here, either these dirs are empty/don't exist or force is
+        # set to true.
+        FileUtils.rm_rf(cookbooks_dir)
+        FileUtils.rm_rf(policyfiles_data_bag_dir)
+
+        FileUtils.mv(cookbooks_staging_dir, export_dir)
+        FileUtils.mkdir_p(export_data_bag_dir)
+        FileUtils.mv(policyfiles_data_bag_staging_dir, export_data_bag_dir)
+        FileUtils.mv(lockfile_staging_path, export_dir)
+      end
+
       def validate_lockfile
         return @policyfile_lock if @policyfile_lock
         @policyfile_lock = ChefDK::PolicyfileLock.new(storage_config).build_from_lock_data(policy_data)
@@ -164,7 +226,7 @@ module ChefDK
       end
 
       def assert_export_dir_clean!
-        if !force_export? && !conflicting_fs_entries.empty?
+        if !force_export? && !conflicting_fs_entries.empty? && !archive?
           msg = "Export dir (#{export_dir}) not clean. Refusing to export. (Conflicting files: #{conflicting_fs_entries.join(', ')})"
           raise ExportDirNotEmpty, msg
         end
@@ -176,15 +238,40 @@ module ChefDK
 
       def conflicting_fs_entries
         Dir.glob(File.join(cookbooks_dir, "*")) +
-          Dir.glob(File.join(policyfiles_data_bag_dir, "*"))
+          Dir.glob(File.join(policyfiles_data_bag_dir, "*")) +
+          Dir.glob(File.join(export_dir, "Policyfile.lock.json"))
       end
 
       def cookbooks_dir
         File.join(export_dir, "cookbooks")
       end
 
+      def export_data_bag_dir
+        File.join(export_dir, "data_bags")
+      end
+
       def policyfiles_data_bag_dir
-        File.join(export_dir, "data_bags", "policyfiles")
+        File.join(export_data_bag_dir, "policyfiles")
+      end
+
+      def policy_id
+        "#{policyfile_lock.name}-#{POLICY_GROUP}"
+      end
+
+      def item_path
+        File.join(staging_dir, "data_bags", "policyfiles", "#{policy_id}.json")
+      end
+
+      def cookbooks_staging_dir
+        File.join(staging_dir, "cookbooks")
+      end
+
+      def policyfiles_data_bag_staging_dir
+        File.join(staging_dir, "data_bags", "policyfiles")
+      end
+
+      def lockfile_staging_path
+        File.join(staging_dir, "Policyfile.lock.json")
       end
 
     end

data/lib/chef-dk/policyfile_services/push_archive.rb

@@ -0,0 +1,173 @@
+#
+# Copyright:: Copyright (c) 2015 Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'zlib'
+require 'archive/tar/minitar'
+
+require 'chef-dk/service_exceptions'
+require 'chef-dk/policyfile_lock'
+require 'chef-dk/authenticated_http'
+require 'chef-dk/policyfile/uploader'
+
+module ChefDK
+  module PolicyfileServices
+    class PushArchive
+
+      USTAR_INDICATOR = "ustar\0".force_encoding(Encoding::ASCII_8BIT).freeze
+
+      attr_reader :archive_file
+      attr_reader :policy_group
+      attr_reader :root_dir
+      attr_reader :ui
+      attr_reader :config
+
+      attr_reader :policyfile_lock
+
+
+      def initialize(archive_file: nil, policy_group: nil, root_dir: nil, ui: nil, config: nil)
+        @archive_file = archive_file
+        @policy_group = policy_group
+        @root_dir = root_dir || Dir.pwd
+        @ui = ui
+        @config = config
+
+        @policyfile_lock = nil
+      end
+
+      def archive_file_path
+        File.expand_path(archive_file, root_dir)
+      end
+
+      def run
+        unless File.exist?(archive_file_path)
+          raise InvalidPolicyArchive, "Archive file #{archive_file_path} not found"
+        end
+        stage_unpacked_archive do |staging_dir|
+          read_policyfile_lock(staging_dir)
+
+          uploader.upload
+        end
+
+      rescue => e
+        raise PolicyfilePushArchiveError.new("Failed to publish archived policy", e)
+      end
+
+      # @api private
+      def uploader
+        ChefDK::Policyfile::Uploader.new(policyfile_lock, policy_group,
+                                         ui: ui,
+                                         http_client: http_client,
+                                         policy_document_native_api: config.policy_document_native_api)
+      end
+
+      # @api private
+      def http_client
+        @http_client ||= ChefDK::AuthenticatedHTTP.new(config.chef_server_url,
+                                                       signing_key_filename: config.client_key,
+                                                       client_name: config.node_name)
+      end
+
+      private
+
+      def read_policyfile_lock(staging_dir)
+        policyfile_lock_path = File.join(staging_dir, "Policyfile.lock.json")
+
+        unless File.exist?(policyfile_lock_path)
+          raise InvalidPolicyArchive, "Archive does not contain a Policyfile.lock.json"
+        end
+
+        unless File.directory?(File.join(staging_dir, "cookbooks"))
+          raise InvalidPolicyArchive, "Archive does not contain a cookbooks directory"
+        end
+
+
+        policy_data = load_policy_data(policyfile_lock_path)
+        storage_config = Policyfile::StorageConfig.new.use_policyfile_lock(policyfile_lock_path)
+        @policyfile_lock = ChefDK::PolicyfileLock.new(storage_config).build_from_archive(policy_data)
+
+        missing_cookbooks = policyfile_lock.cookbook_locks.select do |name, lock|
+          !lock.installed?
+        end
+
+        unless missing_cookbooks.empty?
+          message = "Archive does not have all cookbooks required by the Policyfile.lock. " +
+            "Missing cookbooks: '#{missing_cookbooks.keys.join('", "')}'."
+          raise InvalidPolicyArchive, message
+        end
+
+      end
+
+      def load_policy_data(policyfile_lock_path)
+        FFI_Yajl::Parser.parse(IO.read(policyfile_lock_path))
+      end
+
+      def stage_unpacked_archive
+        p = Process.pid
+        t = Time.new.utc.strftime("%Y%m%d%H%M%S")
+        Dir.mktmpdir("chefdk-push-archive-#{p}-#{t}") do |staging_dir|
+          unpack_to(staging_dir)
+          yield staging_dir
+        end
+
+      end
+
+      def unpack_to(staging_dir)
+        Zlib::GzipReader.open(archive_file_path) do |gz_file|
+          untar_to(gz_file, staging_dir)
+        end
+
+        # untar_to can raise InvalidPolicyArchive, let it through
+      rescue InvalidPolicyArchive
+        raise
+      rescue => e
+        raise InvalidPolicyArchive, "Archive file #{archive_file_path} could not be unpacked. #{e}"
+      end
+
+      def untar_to(tar_file, staging_dir)
+        # Minitar doesn't do much input checking, so if you feed it a
+        # garbage-enough file it will just do weird things and blow up. For
+        # example, if tar_file is just a bunch of nul characters, then tar will
+        # try to open a file named '.'; if you give it some random string that
+        # fits in the size of the filename header, it will create that file.
+        #
+        # Tar archives that we create via `chef export -a` and probably
+        # everything else we might encounter should be in ustar format. For
+        # such a tar file, bytes 257-263 should be "ustar\0", so we use this as
+        # a sanity check.
+        # https://en.wikipedia.org/wiki/Tar_(computing)
+
+        first_tar_header = tar_file.read(512)
+        ustar_indicator = first_tar_header[257, 6]
+
+        unless ustar_indicator == USTAR_INDICATOR
+          raise InvalidPolicyArchive, "Archive file #{archive_file_path} could not be unpacked. Tar archive looks corrupt."
+        end
+
+        # "undo" read of the first 512 bytes
+        tar_file.rewind
+
+        Archive::Tar::Minitar::Input.open(tar_file) do |stream|
+          stream.each do |entry|
+            stream.extract_entry(staging_dir, entry)
+          end
+        end
+      end
+
+
+    end
+  end
+end