pray-cli 1.21.0 → 1.22.0

data/lib/pray/git_materialize.rb

@@ -5,7 +5,10 @@ require "open3"
 require_relative "error"
 require_relative "git_clone"
 require_relative "git_run"
+require_relative "git_store"
 require_relative "path_safety"
+require_relative "resource_limits"
+require_relative "tar_validation"
 
 module Pray
   module GitMaterialize
@@ -14,56 +17,129 @@ module Pray
 
     module_function
 
-    def read_local_artifact_bytes(source_root, artifact)
-      if artifact.start_with?("file://")
-        path = PathSafety.join_under_root(source_root, artifact.delete_prefix("file://"))
-        raise Error.resolution("package artifact path escapes distribution root") unless path
-
-        return File.binread(path)
-      end
-      if artifact.match?(%r{\A[a-z][a-z0-9+.-]*:}i)
-        raise Error.integrity("remote artifact path must be relative: #{artifact}")
+    def read_local_artifact_bytes(source_root, artifact, offline: false)
+      if artifact.include?("://") || artifact.start_with?("file:")
+        raise Error.integrity(
+          "local and git catalog artifacts must be relative paths under the distribution root, not URLs: #{artifact}"
+        )
       end
 
       path = PathSafety.join_under_root(source_root, artifact)
       raise Error.resolution("package artifact path escapes distribution root") unless path
 
-      materialize_catalog_file(source_root, artifact) unless File.exist?(path)
+      materialize_catalog_file(source_root, artifact, offline: offline) unless File.exist?(path)
       raise Error.resolution("package artifact missing at #{path}") unless File.exist?(path)
 
       File.binread(path)
     end
 
-    def materialize_catalog_tree(git_dir, dest, revision, subdir, refresh)
-      return if !refresh && catalog_matches?(dest, git_dir, revision)
+    def materialize_catalog_tree(git_dir, dest, revision, subdir, refresh, offline: false)
+      require_revision_present!(git_dir, revision, offline: offline)
+      GitStore.with_git_cache_lock(dest) do
+        return if !refresh && catalog_matches?(dest, git_dir, revision)
 
-      FileUtils.rm_rf(dest) if File.exist?(dest)
-      FileUtils.mkdir_p(dest)
-      unpacked = GitClone.catalog_sparse_cones(subdir).any? do |prefix|
-        unpack_archive_prefix(git_dir, dest, revision, prefix)
+        staging = "#{dest}.partial"
+        FileUtils.rm_rf(staging)
+        FileUtils.mkdir_p(staging)
+        unpacked = GitClone.catalog_sparse_cones(subdir).any? do |prefix|
+          unpack_archive_prefix(git_dir, staging, revision, prefix, offline: offline)
+        end
+        unless unpacked
+          FileUtils.rm_rf(staging)
+          if offline
+            raise Error.resolution(
+              "git catalog at revision #{revision} is not materializable locally and offline mode is enabled"
+            )
+          end
+          raise Error.resolution("no pray distribution root in git source at revision #{revision}")
+        end
+
+        File.write(File.join(staging, REVISION_MARKER), revision)
+        File.write(File.join(staging, GIT_DIR_MARKER), git_dir)
+        publish_catalog_tree!(dest, staging)
       end
-      raise Error.resolution("no pray distribution root in git source at revision #{revision}") unless unpacked
+    end
 
-      File.write(File.join(dest, REVISION_MARKER), revision)
-      File.write(File.join(dest, GIT_DIR_MARKER), git_dir)
+    def publish_catalog_tree!(dest, staging)
+      FileUtils.mkdir_p(File.dirname(dest))
+      if File.exist?(dest)
+        backup = "#{dest}.previous"
+        FileUtils.rm_rf(backup)
+        File.rename(dest, backup)
+        begin
+          File.rename(staging, dest)
+        rescue
+          File.rename(backup, dest) if File.exist?(backup)
+          FileUtils.rm_rf(staging)
+          raise
+        end
+        FileUtils.rm_rf(backup)
+      else
+        File.rename(staging, dest)
+      end
     end
+    private_class_method :publish_catalog_tree!
 
-    def materialize_catalog_file(source_root, relative)
-      return if File.file?(File.join(source_root, relative))
+    def materialize_catalog_file(source_root, relative, offline: false)
+      full_path = File.join(source_root, relative)
+      return if File.file?(full_path)
 
       markers = find_catalog_markers(source_root)
       return unless markers
 
-      full_path = File.join(source_root, relative)
-      repo_relative = full_path.delete_prefix(markers.fetch(:work_tree)).delete_prefix("/")
-      GitRun.run_git(
-        markers.fetch(:git_dir),
-        "--work-tree",
-        markers.fetch(:work_tree),
-        "checkout",
-        markers.fetch(:revision),
-        "--",
-        repo_relative
+      GitStore.with_git_cache_lock(markers.fetch(:catalog_root)) do
+        return if File.file?(full_path)
+
+        require_revision_present!(markers.fetch(:git_dir), markers.fetch(:revision), offline: offline)
+        repo_relative = full_path.delete_prefix(markers.fetch(:catalog_root)).delete_prefix("/")
+        object = "#{markers.fetch(:revision)}:#{repo_relative}"
+        require_blob_within_limit!(markers.fetch(:git_dir), object, offline: offline)
+        env = GitRun.git_env(offline_git_env(offline))
+        stdout, status = Open3.capture2(
+          env,
+          "git",
+          "-c",
+          "protocol.file.allow=always",
+          "-C",
+          markers.fetch(:git_dir),
+          "show",
+          object,
+          stdin_data: ""
+        )
+        unless status.success?
+          if offline
+            raise Error.resolution(
+              "git catalog blob #{repo_relative} at revision #{markers.fetch(:revision)} is not available locally and offline mode is enabled"
+            )
+          end
+          annotate_git_materialize_failure!(
+            markers.fetch(:git_dir),
+            markers.fetch(:revision),
+            "git show failed"
+          )
+        end
+        write_catalog_blob_atomic!(full_path, stdout)
+      end
+    end
+
+    def require_revision_present!(git_dir, revision, offline: false)
+      return if GitRun.try_run_git(git_dir, "cat-file", "-e", revision)
+
+      if offline
+        raise Error.resolution(
+          "git revision #{revision} is not available in #{git_dir} and offline mode is enabled (git_revision_unfetched)."
+        )
+      end
+      raise Error.resolution(
+        "git revision #{revision} is not available in #{git_dir} (git_revision_unfetched). " \
+        "Run `pray install --locked`, or clear the project git cache and shared bare then install again."
+      )
+    end
+
+    def annotate_git_materialize_failure!(git_dir, revision, detail)
+      raise Error.resolution(
+        "git catalog materialize failed for revision #{revision} in #{git_dir} (git_cache_stale): #{detail}. " \
+        "Run `pray install --locked`, or clear `.pray/cache/git` and the shared bare then install again."
       )
     end
 
@@ -73,6 +149,7 @@ module Pray
       return false unless File.file?(revision_path) && File.file?(git_dir_path)
       return false unless File.read(revision_path).strip == revision
       return false unless File.read(git_dir_path).strip == git_dir
+      return false unless GitRun.try_run_git(git_dir, "cat-file", "-e", revision)
 
       File.directory?(File.join(dest, "v1", "packages")) ||
         File.directory?(File.join(dest, "prayers", "v1", "packages")) ||
@@ -81,8 +158,10 @@ module Pray
       false
     end
 
-    def unpack_archive_prefix(git_dir, dest, revision, prefix)
-      env = GitRun.git_env("GIT_TERMINAL_PROMPT" => "0")
+    def unpack_archive_prefix(git_dir, dest, revision, prefix, offline: false)
+      return false unless tree_prefix_within_limits?(git_dir, revision, prefix, offline: offline)
+
+      env = GitRun.git_env(offline_git_env(offline).merge("GIT_TERMINAL_PROMPT" => "0"))
       stdout, _stderr, status = Open3.capture3(
         env,
         "git",
@@ -98,13 +177,78 @@ module Pray
         stdin_data: ""
       )
       return false unless status.success? && !stdout.empty?
+      if stdout.bytesize > ResourceLimits::MAX_ARCHIVE_TOTAL_BYTES
+        raise Error.integrity(
+          "git catalog archive exceeds #{ResourceLimits::MAX_ARCHIVE_TOTAL_BYTES} bytes"
+        )
+      end
 
-      _ignored, extract_status = Open3.capture2(env, "tar", "-x", "-C", dest, stdin_data: stdout)
-      raise Error.resolution("failed to unpack git catalog archive") unless extract_status.success?
-
+      TarValidation.extract!(stdout, dest)
       true
     end
 
+    def require_blob_within_limit!(git_dir, object, offline: false)
+      env = GitRun.git_env(offline_git_env(offline))
+      stdout, status = Open3.capture2(
+        env, "git", "-c", "protocol.file.allow=always", "-C", git_dir, "cat-file", "-s", object,
+        stdin_data: ""
+      )
+      return unless status.success?
+
+      size = stdout.strip.to_i
+      return if size <= ResourceLimits::MAX_ARCHIVE_ENTRY_BYTES
+
+      raise Error.integrity(
+        "git catalog blob exceeds #{ResourceLimits::MAX_ARCHIVE_ENTRY_BYTES} bytes"
+      )
+    end
+
+    def tree_prefix_within_limits?(git_dir, revision, prefix, offline: false)
+      env = GitRun.git_env(offline_git_env(offline))
+      stdout, status = Open3.capture2(
+        env, "git", "-c", "protocol.file.allow=always", "-C", git_dir,
+        "ls-tree", "-r", "-l", revision, "--", prefix, stdin_data: ""
+      )
+      return false unless status.success? && !stdout.strip.empty?
+
+      entry_count = 0
+      total_bytes = 0
+      stdout.each_line do |line|
+        parts = line.split
+        next unless parts[1] == "blob"
+
+        size = parts[3].to_i
+        entry_count += 1
+        if entry_count > ResourceLimits::MAX_ARCHIVE_ENTRIES
+          raise Error.integrity(
+            "git catalog archive exceeds #{ResourceLimits::MAX_ARCHIVE_ENTRIES} entries"
+          )
+        end
+        if size > ResourceLimits::MAX_ARCHIVE_ENTRY_BYTES
+          raise Error.integrity(
+            "git catalog archive entry exceeds #{ResourceLimits::MAX_ARCHIVE_ENTRY_BYTES} bytes"
+          )
+        end
+        total_bytes += size
+        if total_bytes > ResourceLimits::MAX_ARCHIVE_TOTAL_BYTES
+          raise Error.integrity(
+            "git catalog archive exceeds #{ResourceLimits::MAX_ARCHIVE_TOTAL_BYTES} bytes"
+          )
+        end
+      end
+      entry_count.positive?
+    end
+
+    def write_catalog_blob_atomic!(full_path, bytes)
+      FileUtils.mkdir_p(File.dirname(full_path))
+      partial = "#{full_path}.partial"
+      File.binwrite(partial, bytes)
+      File.rename(partial, full_path)
+    rescue
+      FileUtils.rm_f(partial)
+      raise
+    end
+
     def find_catalog_markers(start)
       current = start
       8.times do
@@ -113,9 +257,9 @@ module Pray
         if File.file?(revision_path) && File.file?(git_dir_path)
           revision = File.read(revision_path).strip
           git_dir = File.read(git_dir_path).strip
-          return if revision.empty? || !File.exist?(git_dir)
+          return unless acceptable_revision?(revision) && acceptable_shared_bare?(git_dir)
 
-          return {git_dir: git_dir, revision: revision, work_tree: current}
+          return {git_dir: git_dir, revision: revision, catalog_root: current}
         end
         parent = File.dirname(current)
         return if parent == current
@@ -124,5 +268,27 @@ module Pray
       end
       nil
     end
+
+    def acceptable_revision?(revision)
+      return false if revision.empty? || revision.start_with?("-")
+      return false unless [40, 64].include?(revision.length)
+
+      revision.match?(/\A[0-9a-fA-F]+\z/)
+    end
+
+    def acceptable_shared_bare?(git_dir)
+      return false unless File.file?(File.join(git_dir, "HEAD"))
+      return false unless File.directory?(File.join(git_dir, "objects"))
+
+      root = GitStore.global_cache_root
+      return false unless root
+
+      expected = File.join(root, "git")
+      PathSafety.path_under_root?(expected, git_dir)
+    end
+
+    def offline_git_env(offline)
+      offline ? {"GIT_NO_LAZY_FETCH" => "1"} : {}
+    end
   end
 end

data/lib/pray/git_store.rb

@@ -8,27 +8,28 @@ require_relative "git_clone"
 
 module Pray
   module GitStore
+    READY_MARKER = ".pray-ready"
+
     module_function
 
     def ensure_global_git_db(clone_url, pinned_revision:, refresh:, offline:, working_directory:)
       db = global_git_cache_directory(clone_url)
       raise Error.resolution("git object cache is not configured") unless db
 
-      unless global_git_cache_ready?(db)
-        raise Error.resolution(offline_git_source_uncached(clone_url)) if offline
+      with_git_cache_lock(db) do
+        unless global_git_cache_ready?(db)
+          raise Error.resolution(offline_git_source_uncached(clone_url)) if offline
 
-        FileUtils.rm_rf(db) if File.exist?(db)
-        FileUtils.mkdir_p(File.dirname(db))
-        GitClone.clone_bare_git_db(working_directory, clone_url, db, quiet: false)
-        ensure_git_remote_origin(db, clone_url)
-      end
-      fetch_origin_tip(db, clone_url) if refresh && !offline
-      if pinned_revision
-        ensure_revision_in_db(db, clone_url, pinned_revision, !offline)
-        return [db, pinned_revision]
-      end
+          install_bare_clone(working_directory, clone_url, db)
+        end
+        fetch_origin_tip(db, clone_url) if refresh && !offline
+        if pinned_revision
+          ensure_revision_in_db(db, clone_url, pinned_revision, !offline)
+          return [db, pinned_revision]
+        end
 
-      [db, git_head_revision(db)]
+        [db, git_head_revision(db)]
+      end
     end
 
     def global_cache_root
@@ -45,13 +46,98 @@ module Pray
     end
 
     def global_git_cache_ready?(global_cache)
-      File.file?(File.join(global_cache, "HEAD")) || File.directory?(File.join(global_cache, ".git"))
+      return false unless File.file?(File.join(global_cache, "HEAD"))
+      return false unless File.directory?(File.join(global_cache, "objects"))
+      return true if File.file?(File.join(global_cache, READY_MARKER))
+
+      begin
+        git_head_revision(global_cache)
+        true
+      rescue Error
+        false
+      end
+    end
+
+    def mark_git_cache_ready(global_cache)
+      File.write(File.join(global_cache, READY_MARKER), "1")
     end
 
     def offline_git_source_uncached(clone_url)
       "git source #{clone_url} is not cached locally and offline mode is enabled"
     end
 
+    def install_bare_clone(working_directory, clone_url, db)
+      FileUtils.mkdir_p(File.dirname(db))
+      staging = "#{db}.partial"
+      FileUtils.rm_rf(staging) if File.exist?(staging)
+      GitClone.clone_bare_git_db(working_directory, clone_url, staging, quiet: false)
+      ensure_git_remote_origin(staging, clone_url)
+      git_head_revision(staging)
+      mark_git_cache_ready(staging)
+      if File.exist?(db)
+        if global_git_cache_ready?(db)
+          FileUtils.rm_rf(staging)
+          return
+        end
+        FileUtils.rm_rf(db)
+      end
+      FileUtils.mv(staging, db)
+    end
+
+    LOCK_ATTEMPTS = 2_400
+    LOCK_RETRY_SECONDS = 0.025
+    STALE_LOCK_SECONDS = 600
+
+    def with_git_cache_lock(db)
+      lock_directory = "#{db}.lock.d"
+      FileUtils.mkdir_p(File.dirname(lock_directory))
+      acquired = false
+      LOCK_ATTEMPTS.times do
+        Dir.mkdir(lock_directory)
+        write_lock_owner!(lock_directory)
+        acquired = true
+        break
+      rescue Errno::EEXIST
+        clear_stale_lock!(lock_directory)
+        sleep LOCK_RETRY_SECONDS
+      end
+      unless acquired
+        raise Error.resolution(
+          "timed out waiting for git cache lock #{lock_directory}. " \
+          "If no other pray process is using that cache, remove the lock directory and retry."
+        )
+      end
+
+      begin
+        yield
+      ensure
+        FileUtils.rm_rf(lock_directory)
+      end
+    end
+
+    def write_lock_owner!(lock_directory)
+      File.write(File.join(lock_directory, "owner"), "#{Process.pid}\n#{Time.now.to_i}\n")
+    end
+
+    def clear_stale_lock!(lock_directory)
+      age = lock_age_seconds(lock_directory)
+      return if age < STALE_LOCK_SECONDS
+
+      FileUtils.rm_rf(lock_directory)
+    end
+
+    def lock_age_seconds(lock_directory)
+      owner = File.join(lock_directory, "owner")
+      if File.file?(owner)
+        stamp = File.read(owner).lines[1].to_s.strip.to_i
+        return Time.now.to_i - stamp if stamp.positive?
+      end
+      Time.now.to_i - File.mtime(lock_directory).to_i
+    rescue Errno::ENOENT
+      0
+    end
+    private_class_method :write_lock_owner!, :clear_stale_lock!, :lock_age_seconds
+
     def ensure_revision_in_db(db, clone_url, revision, allow_fetch)
       return if GitRun.try_run_git(db, "cat-file", "-e", revision)
       unless allow_fetch
@@ -110,6 +196,7 @@ module Pray
         GitRun.run_git(db, "fetch", "--depth", "1", "origin")
       end
       GitRun.run_git(db, "update-ref", "HEAD", "FETCH_HEAD")
+      mark_git_cache_ready(db)
     end
 
     def fetch_unshallow(db)

data/lib/pray/package_spec.rb

@@ -55,6 +55,7 @@ module Pray
     end
 
     def tree_hash_for_root(root)
+      validate_relative_paths!
       file_bytes = {}
       files.each do |file|
         path = File.join(root, file)
@@ -196,7 +197,11 @@ module Pray
         expect_start!
         spec = PackageSpec.new
         while (statement = next_statement)
-          return spec.canonicalized if statement == "end"
+          if statement == "end"
+            package = spec.canonicalized
+            package.validate_relative_paths!
+            return package
+          end
 
           apply_statement(spec, statement)
         end

data/lib/pray/package_spec_paths.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require_relative "path_safety"
+
+module Pray
+  class PackageSpec
+    def validate_relative_paths!
+      files.each { |file| PathSafety.validate_archive_member_path!(file) }
+      exports.each_value do |export|
+        PathSafety.validate_archive_member_path!(export.path)
+        PathSafety.validate_archive_member_path!(export.default_path) if export.default_path
+      end
+      skills.each_value { |skill| PathSafety.validate_archive_member_path!(skill.path) }
+      templates.each_value { |template| PathSafety.validate_archive_member_path!(template.path) }
+      self
+    end
+  end
+end

data/lib/pray/publish.rb

@@ -4,6 +4,7 @@ require "json"
 require "fileutils"
 require "time"
 require "tmpdir"
+require_relative "terminal"
 
 module Pray
   RegistryIndex = Struct.new(:spec, :packages) do
@@ -15,11 +16,14 @@ module Pray
   module Publish
     module_function
 
-    def publish_to_root(project, root, signer: "local", signer_fingerprint: nil)
+    def publish_to_root(project, root, signer: "local", signer_fingerprint: nil, dest_name: nil, target: nil)
       root = File.expand_path(root)
+      dest_name ||= root
+      target ||= root
       index = load_registry_index(root)
       distribution = Distribution.read_settings(root)
       package_names = index.packages.to_set
+      stats = {published: 0, unchanged: 0}
 
       project.packages.each do |package|
         package.spec.require_release_version!
@@ -31,6 +35,8 @@ module Pray
         if stored_artifact && stored_publish_matches?(stored_artifact, package, existing)
           package_names << package.declaration.name
           write_registry_package_metadata(metadata_path, metadata)
+          report_unchanged(package, dest_name, target)
+          stats[:unchanged] += 1
           next
         end
 
@@ -56,14 +62,20 @@ module Pray
         metadata.versions << version_entry
         write_registry_package_metadata(metadata_path, metadata)
         package_names << package.declaration.name
+        report_published(package, dest_name, target, artifact_path)
+        stats[:published] += 1
       end
 
       index.packages = package_names.sort
       write_registry_index(root, index)
+      stats
     end
 
-    def publish_to_server(project, server_url, signer: "local", signer_fingerprint: nil)
+    def publish_to_server(project, server_url, signer: "local", signer_fingerprint: nil, dest_name: nil, target: nil)
+      dest_name ||= server_url
+      target ||= server_url
       distribution = Distribution.fetch_settings(server_url)
+      stats = {published: 0, unchanged: 0}
       project.packages.each do |package|
         archive_bytes = Archive.build_package_archive_bytes(package)
         artifact_path = registry_artifact_path(package.declaration.name, package.spec.version)
@@ -99,9 +111,38 @@ module Pray
           "application/json",
           JSON.pretty_generate(metadata_to_hash(metadata))
         )
+        report_published(package, dest_name, target, artifact_path)
+        stats[:published] += 1
       end
+      stats
     end
 
+    def report_published(package, dest_name, target, artifact_path)
+      Terminal.println_stdout(
+        "published #{package.declaration.name} #{package.spec.version} -> #{redact_publish_target(dest_name)} (#{redact_publish_target(target)})"
+      )
+      Terminal.println_verbose("  artifact #{artifact_path}")
+    end
+
+    def report_unchanged(package, dest_name, target)
+      Terminal.println_stdout(
+        "unchanged #{package.declaration.name} #{package.spec.version} -> #{redact_publish_target(dest_name)} (#{redact_publish_target(target)})"
+      )
+      Terminal.println_verbose("  skip reason: stored publish matches")
+    end
+
+    def redact_publish_target(target)
+      text = target.to_s
+      return text unless text.include?("://")
+
+      scheme, remainder = text.split("://", 2)
+      return text unless remainder&.include?("@")
+
+      _userinfo, host_and_path = remainder.split("@", 2)
+      "#{scheme}://***@#{host_and_path}"
+    end
+    private_class_method :report_published, :report_unchanged
+
     def published_registry_package_version(package, signer, signer_fingerprint, archive_bytes, artifact_path)
       RegistryPackageVersion.new(
         version: package.spec.version,
@@ -114,7 +155,8 @@ module Pray
         signer: signer,
         signer_fingerprint: signer_fingerprint,
         published_at: Time.now.to_i,
-        signature: Registry.registry_artifact_signature(archive_bytes, package.tree_hash, signer)
+        signature: Registry.registry_artifact_signature(archive_bytes, package.tree_hash, signer),
+        derived_metadata: DerivedMetadata.from_package_root(package.root, package.spec)
       )
     end
 
@@ -163,6 +205,7 @@ module Pray
 
       version_entry.yanked = existing.yanked
       version_entry.published_at = existing.published_at if stored_artifact
+      version_entry.derived_metadata ||= existing.derived_metadata
     end
 
     def load_registry_index(root)
@@ -181,7 +224,7 @@ module Pray
 
     def load_registry_package_metadata(path, package_name)
       if File.exist?(path)
-        data = JSON.parse(File.read(path))
+        data = JSON.parse(File.read(path, encoding: "UTF-8"))
         RegistryPackageMetadata.new(
           name: data["name"],
           versions: Array(data["versions"]).map { |entry| Registry.version_from_hash(entry) }
@@ -193,7 +236,7 @@ module Pray
 
     def write_registry_package_metadata(path, metadata)
       FileUtils.mkdir_p(File.dirname(path))
-      File.write(path, JSON.pretty_generate(metadata_to_hash(metadata)))
+      File.write(path, JSON.pretty_generate(metadata_to_hash(metadata)), encoding: "UTF-8")
     end
 
     def metadata_to_hash(metadata)
@@ -217,6 +260,7 @@ module Pray
       hash["signer_fingerprint"] = entry.signer_fingerprint if entry.signer_fingerprint
       hash["published_at"] = entry.published_at if entry.published_at
       hash["signature"] = entry.signature if entry.signature
+      hash["derived_metadata"] = entry.derived_metadata if entry.derived_metadata
       hash
     end