pray-cli 1.20.0 → 1.22.0

data/lib/pray/git_materialize.rb

@@ -5,7 +5,10 @@ require "open3"
 require_relative "error"
 require_relative "git_clone"
 require_relative "git_run"
+require_relative "git_store"
 require_relative "path_safety"
+require_relative "resource_limits"
+require_relative "tar_validation"
 
 module Pray
   module GitMaterialize
@@ -14,56 +17,129 @@ module Pray
 
     module_function
 
-    def read_local_artifact_bytes(source_root, artifact)
-      if artifact.start_with?("file://")
-        path = PathSafety.join_under_root(source_root, artifact.delete_prefix("file://"))
-        raise Error.resolution("package artifact path escapes distribution root") unless path
-
-        return File.binread(path)
-      end
-      if artifact.match?(%r{\A[a-z][a-z0-9+.-]*:}i)
-        raise Error.integrity("remote artifact path must be relative: #{artifact}")
+    def read_local_artifact_bytes(source_root, artifact, offline: false)
+      if artifact.include?("://") || artifact.start_with?("file:")
+        raise Error.integrity(
+          "local and git catalog artifacts must be relative paths under the distribution root, not URLs: #{artifact}"
+        )
       end
 
       path = PathSafety.join_under_root(source_root, artifact)
       raise Error.resolution("package artifact path escapes distribution root") unless path
 
-      materialize_catalog_file(source_root, artifact) unless File.exist?(path)
+      materialize_catalog_file(source_root, artifact, offline: offline) unless File.exist?(path)
       raise Error.resolution("package artifact missing at #{path}") unless File.exist?(path)
 
       File.binread(path)
     end
 
-    def materialize_catalog_tree(git_dir, dest, revision, subdir, refresh)
-      return if !refresh && catalog_matches?(dest, git_dir, revision)
+    def materialize_catalog_tree(git_dir, dest, revision, subdir, refresh, offline: false)
+      require_revision_present!(git_dir, revision, offline: offline)
+      GitStore.with_git_cache_lock(dest) do
+        return if !refresh && catalog_matches?(dest, git_dir, revision)
 
-      FileUtils.rm_rf(dest) if File.exist?(dest)
-      FileUtils.mkdir_p(dest)
-      unpacked = GitClone.catalog_sparse_cones(subdir).any? do |prefix|
-        unpack_archive_prefix(git_dir, dest, revision, prefix)
+        staging = "#{dest}.partial"
+        FileUtils.rm_rf(staging)
+        FileUtils.mkdir_p(staging)
+        unpacked = GitClone.catalog_sparse_cones(subdir).any? do |prefix|
+          unpack_archive_prefix(git_dir, staging, revision, prefix, offline: offline)
+        end
+        unless unpacked
+          FileUtils.rm_rf(staging)
+          if offline
+            raise Error.resolution(
+              "git catalog at revision #{revision} is not materializable locally and offline mode is enabled"
+            )
+          end
+          raise Error.resolution("no pray distribution root in git source at revision #{revision}")
+        end
+
+        File.write(File.join(staging, REVISION_MARKER), revision)
+        File.write(File.join(staging, GIT_DIR_MARKER), git_dir)
+        publish_catalog_tree!(dest, staging)
       end
-      raise Error.resolution("no pray distribution root in git source at revision #{revision}") unless unpacked
+    end
 
-      File.write(File.join(dest, REVISION_MARKER), revision)
-      File.write(File.join(dest, GIT_DIR_MARKER), git_dir)
+    def publish_catalog_tree!(dest, staging)
+      FileUtils.mkdir_p(File.dirname(dest))
+      if File.exist?(dest)
+        backup = "#{dest}.previous"
+        FileUtils.rm_rf(backup)
+        File.rename(dest, backup)
+        begin
+          File.rename(staging, dest)
+        rescue
+          File.rename(backup, dest) if File.exist?(backup)
+          FileUtils.rm_rf(staging)
+          raise
+        end
+        FileUtils.rm_rf(backup)
+      else
+        File.rename(staging, dest)
+      end
     end
+    private_class_method :publish_catalog_tree!
 
-    def materialize_catalog_file(source_root, relative)
-      return if File.file?(File.join(source_root, relative))
+    def materialize_catalog_file(source_root, relative, offline: false)
+      full_path = File.join(source_root, relative)
+      return if File.file?(full_path)
 
       markers = find_catalog_markers(source_root)
       return unless markers
 
-      full_path = File.join(source_root, relative)
-      repo_relative = full_path.delete_prefix(markers.fetch(:work_tree)).delete_prefix("/")
-      GitRun.run_git(
-        markers.fetch(:git_dir),
-        "--work-tree",
-        markers.fetch(:work_tree),
-        "checkout",
-        markers.fetch(:revision),
-        "--",
-        repo_relative
+      GitStore.with_git_cache_lock(markers.fetch(:catalog_root)) do
+        return if File.file?(full_path)
+
+        require_revision_present!(markers.fetch(:git_dir), markers.fetch(:revision), offline: offline)
+        repo_relative = full_path.delete_prefix(markers.fetch(:catalog_root)).delete_prefix("/")
+        object = "#{markers.fetch(:revision)}:#{repo_relative}"
+        require_blob_within_limit!(markers.fetch(:git_dir), object, offline: offline)
+        env = GitRun.git_env(offline_git_env(offline))
+        stdout, status = Open3.capture2(
+          env,
+          "git",
+          "-c",
+          "protocol.file.allow=always",
+          "-C",
+          markers.fetch(:git_dir),
+          "show",
+          object,
+          stdin_data: ""
+        )
+        unless status.success?
+          if offline
+            raise Error.resolution(
+              "git catalog blob #{repo_relative} at revision #{markers.fetch(:revision)} is not available locally and offline mode is enabled"
+            )
+          end
+          annotate_git_materialize_failure!(
+            markers.fetch(:git_dir),
+            markers.fetch(:revision),
+            "git show failed"
+          )
+        end
+        write_catalog_blob_atomic!(full_path, stdout)
+      end
+    end
+
+    def require_revision_present!(git_dir, revision, offline: false)
+      return if GitRun.try_run_git(git_dir, "cat-file", "-e", revision)
+
+      if offline
+        raise Error.resolution(
+          "git revision #{revision} is not available in #{git_dir} and offline mode is enabled (git_revision_unfetched)."
+        )
+      end
+      raise Error.resolution(
+        "git revision #{revision} is not available in #{git_dir} (git_revision_unfetched). " \
+        "Run `pray install --locked`, or clear the project git cache and shared bare then install again."
+      )
+    end
+
+    def annotate_git_materialize_failure!(git_dir, revision, detail)
+      raise Error.resolution(
+        "git catalog materialize failed for revision #{revision} in #{git_dir} (git_cache_stale): #{detail}. " \
+        "Run `pray install --locked`, or clear `.pray/cache/git` and the shared bare then install again."
       )
     end
 
@@ -73,6 +149,7 @@ module Pray
       return false unless File.file?(revision_path) && File.file?(git_dir_path)
       return false unless File.read(revision_path).strip == revision
       return false unless File.read(git_dir_path).strip == git_dir
+      return false unless GitRun.try_run_git(git_dir, "cat-file", "-e", revision)
 
       File.directory?(File.join(dest, "v1", "packages")) ||
         File.directory?(File.join(dest, "prayers", "v1", "packages")) ||
@@ -81,8 +158,10 @@ module Pray
       false
     end
 
-    def unpack_archive_prefix(git_dir, dest, revision, prefix)
-      env = ENV.to_h.merge("GIT_TERMINAL_PROMPT" => "0")
+    def unpack_archive_prefix(git_dir, dest, revision, prefix, offline: false)
+      return false unless tree_prefix_within_limits?(git_dir, revision, prefix, offline: offline)
+
+      env = GitRun.git_env(offline_git_env(offline).merge("GIT_TERMINAL_PROMPT" => "0"))
       stdout, _stderr, status = Open3.capture3(
         env,
         "git",
@@ -98,13 +177,78 @@ module Pray
         stdin_data: ""
       )
       return false unless status.success? && !stdout.empty?
+      if stdout.bytesize > ResourceLimits::MAX_ARCHIVE_TOTAL_BYTES
+        raise Error.integrity(
+          "git catalog archive exceeds #{ResourceLimits::MAX_ARCHIVE_TOTAL_BYTES} bytes"
+        )
+      end
 
-      _ignored, extract_status = Open3.capture2(env, "tar", "-x", "-C", dest, stdin_data: stdout)
-      raise Error.resolution("failed to unpack git catalog archive") unless extract_status.success?
-
+      TarValidation.extract!(stdout, dest)
       true
     end
 
+    def require_blob_within_limit!(git_dir, object, offline: false)
+      env = GitRun.git_env(offline_git_env(offline))
+      stdout, status = Open3.capture2(
+        env, "git", "-c", "protocol.file.allow=always", "-C", git_dir, "cat-file", "-s", object,
+        stdin_data: ""
+      )
+      return unless status.success?
+
+      size = stdout.strip.to_i
+      return if size <= ResourceLimits::MAX_ARCHIVE_ENTRY_BYTES
+
+      raise Error.integrity(
+        "git catalog blob exceeds #{ResourceLimits::MAX_ARCHIVE_ENTRY_BYTES} bytes"
+      )
+    end
+
+    def tree_prefix_within_limits?(git_dir, revision, prefix, offline: false)
+      env = GitRun.git_env(offline_git_env(offline))
+      stdout, status = Open3.capture2(
+        env, "git", "-c", "protocol.file.allow=always", "-C", git_dir,
+        "ls-tree", "-r", "-l", revision, "--", prefix, stdin_data: ""
+      )
+      return false unless status.success? && !stdout.strip.empty?
+
+      entry_count = 0
+      total_bytes = 0
+      stdout.each_line do |line|
+        parts = line.split
+        next unless parts[1] == "blob"
+
+        size = parts[3].to_i
+        entry_count += 1
+        if entry_count > ResourceLimits::MAX_ARCHIVE_ENTRIES
+          raise Error.integrity(
+            "git catalog archive exceeds #{ResourceLimits::MAX_ARCHIVE_ENTRIES} entries"
+          )
+        end
+        if size > ResourceLimits::MAX_ARCHIVE_ENTRY_BYTES
+          raise Error.integrity(
+            "git catalog archive entry exceeds #{ResourceLimits::MAX_ARCHIVE_ENTRY_BYTES} bytes"
+          )
+        end
+        total_bytes += size
+        if total_bytes > ResourceLimits::MAX_ARCHIVE_TOTAL_BYTES
+          raise Error.integrity(
+            "git catalog archive exceeds #{ResourceLimits::MAX_ARCHIVE_TOTAL_BYTES} bytes"
+          )
+        end
+      end
+      entry_count.positive?
+    end
+
+    def write_catalog_blob_atomic!(full_path, bytes)
+      FileUtils.mkdir_p(File.dirname(full_path))
+      partial = "#{full_path}.partial"
+      File.binwrite(partial, bytes)
+      File.rename(partial, full_path)
+    rescue
+      FileUtils.rm_f(partial)
+      raise
+    end
+
     def find_catalog_markers(start)
       current = start
       8.times do
@@ -113,9 +257,9 @@ module Pray
         if File.file?(revision_path) && File.file?(git_dir_path)
           revision = File.read(revision_path).strip
           git_dir = File.read(git_dir_path).strip
-          return if revision.empty? || !File.exist?(git_dir)
+          return unless acceptable_revision?(revision) && acceptable_shared_bare?(git_dir)
 
-          return {git_dir: git_dir, revision: revision, work_tree: current}
+          return {git_dir: git_dir, revision: revision, catalog_root: current}
         end
         parent = File.dirname(current)
         return if parent == current
@@ -124,5 +268,27 @@ module Pray
       end
       nil
     end
+
+    def acceptable_revision?(revision)
+      return false if revision.empty? || revision.start_with?("-")
+      return false unless [40, 64].include?(revision.length)
+
+      revision.match?(/\A[0-9a-fA-F]+\z/)
+    end
+
+    def acceptable_shared_bare?(git_dir)
+      return false unless File.file?(File.join(git_dir, "HEAD"))
+      return false unless File.directory?(File.join(git_dir, "objects"))
+
+      root = GitStore.global_cache_root
+      return false unless root
+
+      expected = File.join(root, "git")
+      PathSafety.path_under_root?(expected, git_dir)
+    end
+
+    def offline_git_env(offline)
+      offline ? {"GIT_NO_LAZY_FETCH" => "1"} : {}
+    end
   end
 end

data/lib/pray/git_run.rb

@@ -5,8 +5,37 @@ require_relative "error"
 
 module Pray
   module GitRun
+    # Mirrors git's own sanitize_repo_env: `git rev-parse --local-env-vars` minus
+    # GIT_CONFIG_PARAMETERS and GIT_CONFIG_COUNT. Git exports these to hooks, so
+    # pray started from a hook would otherwise run every cache command against the
+    # host repository. The two config variables pass through, as they do when git
+    # enters a submodule, because callers such as CI jobs use them to hand git
+    # credentials or URL rewrites, and they cannot select a repository.
+    REPOSITORY_LOCAL_VARIABLES = %w[
+      GIT_ALTERNATE_OBJECT_DIRECTORIES
+      GIT_CONFIG
+      GIT_OBJECT_DIRECTORY
+      GIT_DIR
+      GIT_WORK_TREE
+      GIT_IMPLICIT_WORK_TREE
+      GIT_GRAFT_FILE
+      GIT_INDEX_FILE
+      GIT_NO_REPLACE_OBJECTS
+      GIT_REPLACE_REF_BASE
+      GIT_PREFIX
+      GIT_SHALLOW_FILE
+      GIT_COMMON_DIR
+    ].freeze
+
     module_function
 
+    # Process.spawn merges this hash into the inherited environment, so a nil
+    # value is what removes a variable. Credential, SSH, proxy and global config
+    # variables still pass through.
+    def git_env(extra = {})
+      REPOSITORY_LOCAL_VARIABLES.to_h { |name| [name, nil] }.merge(extra)
+    end
+
     def run_git(cwd, *arguments)
       output, status = capture_git(cwd, *arguments)
       return if status.success?
@@ -20,7 +49,7 @@ module Pray
     end
 
     def capture_git(cwd, *arguments)
-      env = ENV.to_h.merge("GIT_TERMINAL_PROMPT" => "0")
+      env = git_env("GIT_TERMINAL_PROMPT" => "0")
       Open3.capture2e(
         env,
         "git",

data/lib/pray/git_store.rb

@@ -8,27 +8,28 @@ require_relative "git_clone"
 
 module Pray
   module GitStore
+    READY_MARKER = ".pray-ready"
+
     module_function
 
     def ensure_global_git_db(clone_url, pinned_revision:, refresh:, offline:, working_directory:)
       db = global_git_cache_directory(clone_url)
       raise Error.resolution("git object cache is not configured") unless db
 
-      unless global_git_cache_ready?(db)
-        raise Error.resolution(offline_git_source_uncached(clone_url)) if offline
+      with_git_cache_lock(db) do
+        unless global_git_cache_ready?(db)
+          raise Error.resolution(offline_git_source_uncached(clone_url)) if offline
 
-        FileUtils.rm_rf(db) if File.exist?(db)
-        FileUtils.mkdir_p(File.dirname(db))
-        GitClone.clone_bare_git_db(working_directory, clone_url, db, quiet: false)
-        ensure_git_remote_origin(db, clone_url)
-      end
-      fetch_origin_tip(db, clone_url) if refresh && !offline
-      if pinned_revision
-        ensure_revision_in_db(db, clone_url, pinned_revision, !offline)
-        return [db, pinned_revision]
-      end
+          install_bare_clone(working_directory, clone_url, db)
+        end
+        fetch_origin_tip(db, clone_url) if refresh && !offline
+        if pinned_revision
+          ensure_revision_in_db(db, clone_url, pinned_revision, !offline)
+          return [db, pinned_revision]
+        end
 
-      [db, git_head_revision(db)]
+        [db, git_head_revision(db)]
+      end
     end
 
     def global_cache_root
@@ -45,13 +46,98 @@ module Pray
     end
 
     def global_git_cache_ready?(global_cache)
-      File.file?(File.join(global_cache, "HEAD")) || File.directory?(File.join(global_cache, ".git"))
+      return false unless File.file?(File.join(global_cache, "HEAD"))
+      return false unless File.directory?(File.join(global_cache, "objects"))
+      return true if File.file?(File.join(global_cache, READY_MARKER))
+
+      begin
+        git_head_revision(global_cache)
+        true
+      rescue Error
+        false
+      end
+    end
+
+    def mark_git_cache_ready(global_cache)
+      File.write(File.join(global_cache, READY_MARKER), "1")
     end
 
     def offline_git_source_uncached(clone_url)
       "git source #{clone_url} is not cached locally and offline mode is enabled"
     end
 
+    def install_bare_clone(working_directory, clone_url, db)
+      FileUtils.mkdir_p(File.dirname(db))
+      staging = "#{db}.partial"
+      FileUtils.rm_rf(staging) if File.exist?(staging)
+      GitClone.clone_bare_git_db(working_directory, clone_url, staging, quiet: false)
+      ensure_git_remote_origin(staging, clone_url)
+      git_head_revision(staging)
+      mark_git_cache_ready(staging)
+      if File.exist?(db)
+        if global_git_cache_ready?(db)
+          FileUtils.rm_rf(staging)
+          return
+        end
+        FileUtils.rm_rf(db)
+      end
+      FileUtils.mv(staging, db)
+    end
+
+    LOCK_ATTEMPTS = 2_400
+    LOCK_RETRY_SECONDS = 0.025
+    STALE_LOCK_SECONDS = 600
+
+    def with_git_cache_lock(db)
+      lock_directory = "#{db}.lock.d"
+      FileUtils.mkdir_p(File.dirname(lock_directory))
+      acquired = false
+      LOCK_ATTEMPTS.times do
+        Dir.mkdir(lock_directory)
+        write_lock_owner!(lock_directory)
+        acquired = true
+        break
+      rescue Errno::EEXIST
+        clear_stale_lock!(lock_directory)
+        sleep LOCK_RETRY_SECONDS
+      end
+      unless acquired
+        raise Error.resolution(
+          "timed out waiting for git cache lock #{lock_directory}. " \
+          "If no other pray process is using that cache, remove the lock directory and retry."
+        )
+      end
+
+      begin
+        yield
+      ensure
+        FileUtils.rm_rf(lock_directory)
+      end
+    end
+
+    def write_lock_owner!(lock_directory)
+      File.write(File.join(lock_directory, "owner"), "#{Process.pid}\n#{Time.now.to_i}\n")
+    end
+
+    def clear_stale_lock!(lock_directory)
+      age = lock_age_seconds(lock_directory)
+      return if age < STALE_LOCK_SECONDS
+
+      FileUtils.rm_rf(lock_directory)
+    end
+
+    def lock_age_seconds(lock_directory)
+      owner = File.join(lock_directory, "owner")
+      if File.file?(owner)
+        stamp = File.read(owner).lines[1].to_s.strip.to_i
+        return Time.now.to_i - stamp if stamp.positive?
+      end
+      Time.now.to_i - File.mtime(lock_directory).to_i
+    rescue Errno::ENOENT
+      0
+    end
+    private_class_method :write_lock_owner!, :clear_stale_lock!, :lock_age_seconds
+
     def ensure_revision_in_db(db, clone_url, revision, allow_fetch)
       return if GitRun.try_run_git(db, "cat-file", "-e", revision)
       unless allow_fetch
@@ -73,6 +159,7 @@ module Pray
     end
 
     def ensure_git_remote_origin(repository, clone_url)
+      require_own_git_dir(repository)
       if GitRun.try_run_git(repository, "remote", "get-url", "origin")
         GitRun.run_git(repository, "remote", "set-url", "origin", clone_url)
       else
@@ -80,12 +167,36 @@ module Pray
       end
     end
 
+    # Git searches parent directories when `repository` is not a valid git
+    # directory, so a damaged cache inside another checkout would otherwise
+    # rewrite that checkout's origin.
+    def require_own_git_dir(repository)
+      output, status = GitRun.capture_git(repository, "rev-parse", "--absolute-git-dir")
+      resolved = status.success? ? output.strip : ""
+      own = [repository, File.join(repository, ".git")].any? do |candidate|
+        real_path(candidate) && real_path(candidate) == real_path(resolved)
+      end
+      return if own
+
+      raise Error.resolution(
+        "git cache #{repository} is not a usable git repository (git resolved #{resolved.inspect}). " \
+        "Remove it and run the command again."
+      )
+    end
+
+    def real_path(path)
+      File.realpath(path)
+    rescue SystemCallError
+      nil
+    end
+
     def fetch_origin_tip(db, clone_url)
       ensure_git_remote_origin(db, clone_url)
       unless GitRun.try_run_git(db, "fetch", "--depth", "1", "--filter=blob:none", "origin")
         GitRun.run_git(db, "fetch", "--depth", "1", "origin")
       end
       GitRun.run_git(db, "update-ref", "HEAD", "FETCH_HEAD")
+      mark_git_cache_ready(db)
     end
 
     def fetch_unshallow(db)

data/lib/pray/package_spec.rb

@@ -55,6 +55,7 @@ module Pray
     end
 
     def tree_hash_for_root(root)
+      validate_relative_paths!
       file_bytes = {}
       files.each do |file|
         path = File.join(root, file)
@@ -196,7 +197,11 @@ module Pray
         expect_start!
         spec = PackageSpec.new
         while (statement = next_statement)
-          return spec.canonicalized if statement == "end"
+          if statement == "end"
+            package = spec.canonicalized
+            package.validate_relative_paths!
+            return package
+          end
 
           apply_statement(spec, statement)
         end

data/lib/pray/package_spec_paths.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require_relative "path_safety"
+
+module Pray
+  class PackageSpec
+    def validate_relative_paths!
+      files.each { |file| PathSafety.validate_archive_member_path!(file) }
+      exports.each_value do |export|
+        PathSafety.validate_archive_member_path!(export.path)
+        PathSafety.validate_archive_member_path!(export.default_path) if export.default_path
+      end
+      skills.each_value { |skill| PathSafety.validate_archive_member_path!(skill.path) }
+      templates.each_value { |template| PathSafety.validate_archive_member_path!(template.path) }
+      self
+    end
+  end
+end