cookbook-omnifetch 0.10.0 → 0.12.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9491fb99bc2194cfb805a87ab4d543be68e51a4fc4c3d27e954c62e96b42df33
-  data.tar.gz: 1f364b47c66fe2c7ac35d45f9a343dc8a14e6eaad3050007db5c4c59f255029b
+  metadata.gz: a84aa2f4551b271e3c13a1260d6070ed4a637ca777ce81343dc3087bf73d9122
+  data.tar.gz: b93c4da8a8ab2a64f7c62e4b9a677ca0276f8a7a9ae6c82f73e21a644f03e07b
 SHA512:
-  metadata.gz: bfe5d751f1891af706db45ebf57867207078876419eaa7f7629ee4ae9b5437897528782062186e2eec04cee2516dae01c602d68581b6b759f7b1a8eb7eea6fea
-  data.tar.gz: 6f1668723a48ca94e337e1aba18138a745b82a65008ad88c41fea1e63318120002a516334d238ad61c3faf8b623f5625220a291fa53fd1e17ae6229151389b8d
+  metadata.gz: e5043f1653e630012f06582d1917ddb72a7904e0a6b283ad5db6f2020a7b26859280bcf50e8ee19298dd00798d1738f644aa5348aafc5571c003addc3fe69fbb
+  data.tar.gz: 7568ed165159ef9800da6211ad8d3bb53ffedde3cd6923a37dd7506dd425f53f0994f49c05cdff8858fbb63ceacf271b4e4f7c54add4ebeb646e6597dc4f1daa

data/lib/cookbook-omnifetch/artifactory.rb

@@ -1,7 +1,7 @@
 require_relative "base"
 
-require "mixlib/archive"
-require "tmpdir"
+require "mixlib/archive" unless defined?(Mixlib::Archive)
+require "tmpdir" unless defined?(Dir.mktmpdir)
 
 module CookbookOmnifetch
 

data/lib/cookbook-omnifetch/artifactserver.rb

@@ -1,7 +1,7 @@
 require_relative "base"
 
-require "mixlib/archive"
-require "tmpdir"
+require "mixlib/archive" unless defined?(Mixlib::Archive)
+require "tmpdir" unless defined?(Dir.mktmpdir)
 
 module CookbookOmnifetch
 

data/lib/cookbook-omnifetch/exceptions.rb

@@ -106,4 +106,9 @@ module CookbookOmnifetch
     end
   end
 
+  class StagingAreaNotAvailable < OmnifetchError
+    def initialize
+      super "failed to access a StagingArea that is no longer available"
+    end
+  end
 end

data/lib/cookbook-omnifetch/git.rb

@@ -1,4 +1,4 @@
-require "tmpdir"
+require "tmpdir" unless defined?(Dir.mktmpdir)
 require_relative "../cookbook-omnifetch"
 require_relative "base"
 require_relative "exceptions"

data/lib/cookbook-omnifetch/metadata_based_installer.rb

@@ -1,4 +1,6 @@
 require_relative "threaded_job_queue"
+require_relative "staging_area"
+require "digest/md5" unless defined?(Digest::MD5)
 
 module CookbookOmnifetch
 
@@ -27,7 +29,7 @@ module CookbookOmnifetch
           next unless @metadata.key?(type.to_s)
 
           @metadata[type.to_s].each do |file|
-            yield file["url"], file["path"]
+            yield file["url"], file["path"], file["checksum"]
           end
         end
       end
@@ -46,50 +48,61 @@ module CookbookOmnifetch
     end
 
     def install
-      FileUtils.rm_rf(staging_path) # ensure we have a clean dir, just in case
-      FileUtils.mkdir_p(staging_root) unless staging_root.exist?
-      md = http_client.get(url_path)
-
-      queue = ThreadedJobQueue.new
+      StagingArea.stage(install_path) do |staging_path|
+        FileUtils.cp_r("#{install_path}/.", staging_path) if Dir.exist?(install_path)
+        metadata = http_client.get(url_path)
+        clean_cache(staging_path, metadata)
+        sync_cache(staging_path, metadata)
+      end
+    end
 
-      CookbookMetadata.new(md).files do |url, path|
-        stage = staging_path.join(path)
-        FileUtils.mkdir_p(File.dirname(stage))
+    # Removes files from cache that are not supposed to be there, based on
+    # files in metadata.
+    def clean_cache(staging_path, metadata)
+      actual_file_list = Dir.glob(File.join(staging_path, "**/*"))
+      expected_file_list = []
+      CookbookMetadata.new(metadata).files { |_, path, _| expected_file_list << File.join(staging_path, path) }
+
+      extra_files = actual_file_list - expected_file_list
+      extra_files.each do |path|
+        if File.file?(path)
+          FileUtils.rm(path)
+        end
+      end
+    end
 
-        queue << lambda do |_lock|
-          http_client.streaming_request(url) do |tempfile|
-            tempfile.close
-            FileUtils.mv(tempfile.path, stage)
+    # Downloads any out-of-date files into installer cache, overwriting
+    # those that don't match the checksum provided the metadata @ url_path
+    def sync_cache(staging_path, metadata)
+      queue = ThreadedJobQueue.new
+      CookbookMetadata.new(metadata).files do |url, path, checksum|
+        dest_path = File.join(staging_path, path)
+        FileUtils.mkdir_p(File.dirname(dest_path))
+        if file_outdated?(dest_path, checksum)
+          queue << lambda do |_lock|
+            http_client.streaming_request(url) do |tempfile|
+              tempfile.close
+              FileUtils.mv(tempfile.path, dest_path)
+            end
           end
         end
       end
-
       queue.process(CookbookOmnifetch.chef_server_download_concurrency)
-
-      FileUtils.mv(staging_path, install_path)
     end
 
-    # The path where files are downloaded to.  On certain platforms you have a
-    # better chance of getting an atomic move if your temporary working
-    # directory is on the same device/volume as the  destination.  To support
-    # this, we use a staging directory located under the cache path under the
-    # rather mild assumption that everything under the cache path is going to
-    # be on one device.
+    # Check if a given file (at absolute path) is missing or does has a mismatched md5sum
     #
-    # @return [Pathname]
-    def staging_root
-      Pathname.new(CookbookOmnifetch.cache_path).join(".cache_tmp", "metadata-installer")
-    end
-
-    def staging_path
-      staging_root.join(staging_cache_key)
-    end
-
-    # Convert the URL to a safe name for a file and append our random slug.
-    # This helps us avoid colliding in the case that there are multiple
-    # processes installing the same cookbook at the same time.
-    def staging_cache_key
-      "#{url_path.gsub(/[^[:alnum:]]/, "_")}_#{slug}"
+    # @return [TrueClass, FalseClass]
+    def file_outdated?(path, expected_md5sum)
+      return true unless File.exist?(path)
+
+      md5 = Digest::MD5.new
+      File.open(path, "r") do |file|
+        while (chunk = file.read(1024))
+          md5.update chunk
+        end
+      end
+      md5.to_s != expected_md5sum
     end
   end
 end

data/lib/cookbook-omnifetch/staging_area.rb

@@ -0,0 +1,192 @@
+require_relative "exceptions"
+
+module CookbookOmnifetch
+  # A staging area in which the caller can stage files and publish them to a
+  # local directory.
+  #
+  # When performing long operations such as installing or updating a cookbook
+  # from the web, {StagingArea} allows you to minimize the risk that a process
+  # running in parallel might retrieve an incomplete cookbook from the local
+  # cache before it is completely installed. (See {publish!} for details.)
+  #
+  # {StagingArea} allocates temporary directories on the local file system.  It
+  # is the caller's responsibility to use {discard!} when it is done to remove
+  # those directories.  The {.stage} method handles directory cleanup for the
+  # staging area it creates before returning.
+  #
+  # @example installing files using the {.stage} helper
+  #   CookbookOmnifetch::StagingArea.stage(install_path) do |staging_path|
+  #     # Copy files to staging_path
+  #   end
+  #
+  # @example creating a staging area and publishing it manually
+  #   stage = CookbookOmnifetch::StagingArea.new
+  #   # Copy files to stage.path
+  #   stage.publish!(install_path)
+  #   stage.discard!
+  class StagingArea
+    # Creates a staging area, calls a block to populate it, then publishes it.
+    #
+    # {stage} creates a staging area and calls the provided block to populate it
+    # with files. If the staging area does not contain any changes for
+    # +target_path+ (see {#match?}), it cleans up the staging area without
+    # modifying +target_path+.  Otherwise, it publishes its contents to
+    # +target_path+ and deletes the staging area.  As a safety measure, {stage}
+    # will not publish an empty staging area.
+    #
+    # @param [Pathname] target_path
+    #   directory to which the staging area will publish its contents
+    #
+    # @yieldparam staging_path [Pathname]
+    #   the directory in which the block should stage its files
+    def self.stage(target_path)
+      sa = new
+      begin
+        yield(sa.path)
+        sa.publish!(target_path) unless sa.empty? || sa.match?(target_path)
+      ensure
+        sa.discard!
+      end
+    end
+
+    # Returns true if the staging area is no longer available for use.
+    #
+    # The staging area is no longer available once {discard!} removes it from
+    # the file system.
+    #
+    # @return [Boolean] whether the staging area is unavailable
+    def unavailable?
+      !!@unavailable
+    end
+
+    # Returns true if the staging area is empty.
+    #
+    # A staging area is considered empty when it has no files or directories in
+    # its path or the staging directory does not exist.
+    #
+    # @raise [StagingAreaNotAvailable]
+    #   when called after the staging area destroyed with {discard!}
+    #
+    # @return [Boolean] whether the staging area is empty
+    def empty?
+      !path.exist? || path.empty?
+    end
+
+    # Returns true if the staging area's contents match those of a given path.
+    #
+    # {#match?} compares the contents of the staging area with the contents of
+    # the +compare_path+.  It considers the staging area to match if it contains
+    # all of and nothing more than the files and directories present in
+    # +compare_path+ and the content of each file is the same as that of its
+    # corresponding file in +compare_path+.  {match?} does not compare file
+    # metadata or the contents of special files.
+    #
+    # @param [String] compare_path
+    #   the directory to which the staging area will compare its contents
+    #
+    # @raise [StagingAreaNotAvailable]
+    #   when called after the staging area destroyed with {discard!}
+    #
+    # @return [Boolean] whether the staging area matches +compare_path+
+    def match?(compare_path)
+      raise StagingAreaNotAvailable if unavailable?
+
+      target = Pathname(compare_path)
+      return false unless target.exist?
+
+      files = Dir.glob("**/*", File::FNM_DOTMATCH, base: path)
+      target_files = Dir.glob("**/*", File::FNM_DOTMATCH, base: target)
+      return false unless files.sort == target_files.sort
+
+      files.each do |subpath|
+        return false if files_different?(path, target, subpath)
+      end
+
+      true
+    end
+
+    # Path to the staging folder on the file system.
+    #
+    # @raise [StagingAreaNotAvailable]
+    #   when called after the staging area destroyed with {discard!}
+    #
+    # @return [Pathname] path to the staging folder
+    def path
+      raise StagingAreaNotAvailable if unavailable?
+
+      return @path unless @path.nil?
+
+      # Dir.mktmpdir returns a directory with restrictive permissions that it
+      # doesn't support modifying, so create a subdirectory under it with
+      # regular permissions for staging.
+      @stage_tmp = Dir.mktmpdir
+      @path = Pathname.new(File.join(@stage_tmp, "staging"))
+      FileUtils.mkdir(@path)
+      @path
+    end
+
+    # Removes the staging area and its contents from the file system.
+    #
+    # The staging area is no longer available once {discard!} removes it from
+    # the file system.  Future attempts to use it will raise
+    # {StagingAreaNotAvailable}.
+    def discard!
+      FileUtils.rm_rf(@stage_tmp) unless @stage_tmp.nil?
+      @unavailable = true
+    end
+
+    # Replaces +install_path+ with the contents of the staging area.
+    #
+    # {publish!} removes the target and copies the new content into place using
+    # two atomic file system operations.  This eliminates much of the risk
+    # associated with updating the target in a multiprocess environment by
+    # ensuring that another process does not see a partially removed or
+    # populated directory at the +target_path+ while this operation is being
+    # performed.
+    #
+    # Note that it is still possible for the {publish!} to interrupt another
+    # process performing a long operation, such as creating a recursive copy of
+    # the target.  In this situation, the other process may create a copy that
+    # consists of a combination of content from the old target directory and the
+    # newly staged files.  The other process may also raise an exception should
+    # it try to access the target during a small window in the {publish!}
+    # operation where the target directory does not exist, or tries to open a
+    # file that is no longer part of the target tree after {publish!} completes.
+    # The other process can detect this situation by verifying that the content
+    # of its copy matches the content of +target_path+ after its copy is
+    # complete.
+    #
+    # @param [String] install_path
+    #   directory to which the staging area will publish its contents
+    #
+    # @raise [StagingAreaNotAvailable]
+    #   when called after the staging area destroyed with {discard!}
+    def publish!(install_path)
+      target = Pathname(install_path)
+      cache_dir = target.parent
+      cache_dir.mkpath
+      Dir.mktmpdir("_STAGING_TMP_", cache_dir) do |tmpdir|
+        newtmp = File.join(tmpdir, "new_cookbook")
+        oldtmp = File.join(tmpdir, "old_cookbook")
+        FileUtils.cp_r(path, newtmp)
+
+        # We could achieve an atomic replace using symbolic links, if they are
+        # supported on all platforms.
+        File.rename(target, oldtmp) if target.exist?
+        File.rename(newtmp, target)
+      end
+    end
+
+    private
+
+    # compares two files
+    def files_different?(base1, base2, subpath)
+      file1 = File.join(base1, subpath)
+      file2 = File.join(base2, subpath)
+      return true unless File.ftype(file1) == File.ftype(file2)
+      return true if File.file?(file1) && !FileUtils.cmp(file1, file2)
+
+      false
+    end
+  end
+end

data/lib/cookbook-omnifetch/version.rb

@@ -1,3 +1,3 @@
 module CookbookOmnifetch
-  VERSION = "0.10.0".freeze
+  VERSION = "0.12.2".freeze
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cookbook-omnifetch
 version: !ruby/object:Gem::Version
-  version: 0.10.0
+  version: 0.12.2
 platform: ruby
 authors:
 - Jamie Winsor
@@ -13,7 +13,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-07-15 00:00:00.000000000 Z
+date: 2022-03-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mixlib-archive
@@ -60,6 +60,7 @@ files:
 - lib/cookbook-omnifetch/integration.rb
 - lib/cookbook-omnifetch/metadata_based_installer.rb
 - lib/cookbook-omnifetch/path.rb
+- lib/cookbook-omnifetch/staging_area.rb
 - lib/cookbook-omnifetch/threaded_job_queue.rb
 - lib/cookbook-omnifetch/version.rb
 homepage: https://github.com/chef/cookbook-omnifetch
@@ -74,14 +75,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.4'
+      version: '2.5'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: Library code to fetch Chef cookbooks from a variety of sources to a local