pray-cli 1.9.1 → 1.10.0

data/lib/pray/render_dest.rb

@@ -0,0 +1,215 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "pathname"
+
+module Pray
+  module RenderDest
+    module_function
+
+    def materialize(project, previous_lockfile = nil)
+      planned = Render.planned_provisioned_files(project)
+      previous = previous_map(previous_lockfile)
+      planned_paths = planned.map(&:path).to_h { |path| [path, true] }
+      planned.each { |file| write_leaf(project, file, previous) }
+      prune_dropped(project, previous_lockfile, planned_paths) if previous_lockfile
+    end
+
+    def provisioned_records(project)
+      symbols = project.manifest.symbols || {}
+      Render.planned_provisioned_files(project).map do |file|
+        expected = Render.expected_provisioned_bytes(file.source, symbols)
+        ProvisionedFileRecord.new(
+          path: file.path.to_s.tr("\\", "/"),
+          content_hash: Hashing.sha256_prefixed(expected.b),
+          package: file.package,
+          export: file.export
+        )
+      end
+    end
+
+    def fail_if_symlink!(path, display)
+      return unless File.symlink?(path)
+
+      raise Error.render("refusing to write `#{display}` because it is a symbolic link")
+    end
+
+    def previous_map(lockfile)
+      return {} unless lockfile
+
+      Array(lockfile.provisioned).to_h { |record| [record.path, record] }
+    end
+
+    def destination_status(project, file, previous_lockfile = nil)
+      PathSafety.validate_destination_path!(file.path)
+      ensure_safe_destination_ancestors!(project.project_root, file.path, file.path)
+      destination = File.join(project.project_root, file.path)
+      expected = Render.expected_provisioned_bytes(file.source, project.manifest.symbols || {})
+      record = previous_map(previous_lockfile)[file.path.to_s.tr("\\", "/")]
+      classify_destination(destination, file.path, expected, record)
+    end
+
+    def write_leaf(project, file, previous)
+      PathSafety.validate_destination_path!(file.path)
+      ensure_safe_destination_ancestors!(project.project_root, file.path, file.path)
+      destination = File.join(project.project_root, file.path)
+      expected = Render.expected_provisioned_bytes(file.source, project.manifest.symbols || {})
+      record = previous[file.path.to_s.tr("\\", "/")]
+      status = classify_destination(destination, file.path, expected, record)
+      if status == :write
+        FileUtils.mkdir_p(File.dirname(destination))
+        ensure_safe_destination_ancestors!(project.project_root, file.path, file.path)
+        return create_bytes(destination, file.path, expected)
+      end
+      return if status == :unchanged
+
+      unless record
+        raise Error.render("missing lock ownership for `#{file.path}`")
+      end
+
+      ensure_safe_destination_ancestors!(project.project_root, file.path, file.path)
+      update_bytes(destination, file.path, expected, record.content_hash)
+    end
+
+    def classify_destination(destination, display, expected, record)
+      kind = destination_kind(destination)
+      return :write if kind == :missing
+
+      fail_if_symlink!(destination, display)
+      unless kind == :regular
+        raise Error.render("refusing to write `#{display}`; destination is not a regular file")
+      end
+      on_disk = read_regular_bytes(destination, display)
+      return :unchanged if on_disk.b == expected.b
+      return :update if record && Hashing.sha256_prefixed(on_disk) == record.content_hash
+
+      if record
+        raise Error.render("refusing to overwrite `#{display}`; it was provisioned and then edited")
+      end
+
+      raise Error.render(
+        "refusing to overwrite `#{display}`; it already exists and is not the expected provisioned file"
+      )
+    end
+
+    def prune_dropped(project, previous, planned_paths)
+      Array(previous.provisioned).each do |record|
+        PathSafety.validate_destination_path!(record.path)
+        next if planned_paths[record.path]
+
+        destination = File.join(project.project_root, record.path)
+        ensure_safe_destination_ancestors!(project.project_root, record.path, record.path)
+        next unless destination_kind(destination) == :regular
+
+        on_disk = read_regular_bytes(destination, record.path)
+        if Hashing.sha256_prefixed(on_disk) == record.content_hash
+          ensure_safe_destination_ancestors!(project.project_root, record.path, record.path)
+          File.delete(destination)
+        end
+      end
+    end
+
+    def layout_rendered_content(path, display, fresh)
+      kind = destination_kind(path)
+      return fresh if kind == :missing
+
+      fail_if_symlink!(path, display)
+      unless kind == :regular
+        raise Error.render("refusing to write `#{display}`; destination is not a regular file")
+      end
+      RenderPatch.patch_rendered_content(decode_utf8(read_regular_bytes(path, display), display), fresh)
+    end
+
+    def write_rendered_content(path, display, fresh)
+      return create_bytes(path, display, fresh) if destination_kind(path) == :missing
+
+      open_regular(path, display, File::RDWR) do |file|
+        existing = decode_utf8(file.read, display)
+        content = RenderPatch.patch_rendered_content(existing, fresh)
+        file.rewind
+        file.truncate(0)
+        file.write(content)
+      end
+    end
+
+    def ensure_safe_destination_ancestors!(project_root, relative_path, display)
+      parent = File.dirname(relative_path.to_s.tr("\\", "/"))
+      return if parent == "."
+
+      current = project_root
+      parent.split("/").each do |component|
+        next if component.empty? || component == "."
+
+        current = File.join(current, component)
+        metadata = File.lstat(current)
+        if metadata.symlink?
+          raise Error.render(
+            "refusing to write `#{display}` because a destination parent is a symbolic link"
+          )
+        end
+        unless metadata.directory?
+          raise Error.render(
+            "refusing to write `#{display}`; a destination parent is not a directory"
+          )
+        end
+      rescue Errno::ENOENT
+        next
+      end
+    end
+
+    def create_bytes(path, display, bytes)
+      open_path(path, display, File::WRONLY | File::CREAT | File::EXCL) do |file|
+        file.write(bytes)
+      end
+    end
+
+    def update_bytes(path, display, bytes, authorized_hash)
+      open_regular(path, display, File::RDWR) do |file|
+        on_disk = file.read
+        return if on_disk.b == bytes.b
+
+        unless Hashing.sha256_prefixed(on_disk) == authorized_hash
+          raise Error.render("refusing to overwrite `#{display}`; it was provisioned and then edited")
+        end
+        file.rewind
+        file.truncate(0)
+        file.write(bytes)
+      end
+    end
+
+    def read_regular_bytes(path, display)
+      open_regular(path, display, File::RDONLY) { |file| file.read }
+    end
+
+    def decode_utf8(bytes, display)
+      text = bytes.dup.force_encoding(Encoding::UTF_8)
+      return text if text.valid_encoding?
+
+      raise Error.render("rendered destination `#{display}` is not valid UTF-8")
+    end
+
+    def open_regular(path, display, flags)
+      open_path(path, display, flags) do |file|
+        unless file.stat.file?
+          raise Error.render("refusing to write `#{display}`; destination is not a regular file")
+        end
+        yield file
+      end
+    end
+
+    def open_path(path, display, flags)
+      no_follow = File.const_defined?(:NOFOLLOW) ? File::NOFOLLOW : 0
+      File.open(path, flags | no_follow) { |file| yield file }
+    rescue Errno::ELOOP
+      raise Error.render("refusing to write `#{display}` because it is a symbolic link")
+    end
+
+    def destination_kind(path)
+      return :symlink if File.symlink?(path)
+      return :missing unless File.exist?(path)
+      return :regular if File.file?(path)
+
+      :other
+    end
+  end
+end

data/lib/pray/render_layout.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Pray
+  module Render
+    module_function
+
+    def layout_rendered_targets(project, rendered)
+      rendered.map do |target|
+        PathSafety.validate_destination_path!(target.path)
+        RenderDest.ensure_safe_destination_ancestors!(project.project_root, target.path, target.path)
+        path = File.join(project.project_root, target.path)
+        content = RenderDest.layout_rendered_content(path, target.path, target.content)
+        RenderedTarget.new(
+          path: target.path,
+          content: content,
+          managed_spans: RenderPatch.relocate_managed_spans(content, target.managed_spans)
+        )
+      end
+    end
+  end
+end

data/lib/pray/render_patch.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+module Pray
+  module RenderPatch
+    module_function
+
+    def patch_rendered_content(existing, fresh)
+      existing_segments = split_segments(existing)
+      fresh_segments = split_segments(fresh)
+      fresh_managed = fresh_segments.filter_map do |segment|
+        [segment[:id], segment[:body]] if segment[:kind] == :managed
+      end.to_h
+      overlap = existing_segments.any? do |segment|
+        segment[:kind] == :managed && fresh_managed.key?(segment[:id])
+      end
+      return fresh unless overlap
+
+      used = {}
+      output = existing_segments.map do |segment|
+        next segment[:text] if segment[:kind] == :text
+
+        used[segment[:id]] = true
+        managed_segment(segment[:id], fresh_managed.fetch(segment[:id], segment[:body]))
+      end.join
+      fresh_segments.each do |segment|
+        next unless segment[:kind] == :managed
+        next if used[segment[:id]]
+
+        output << managed_segment(segment[:id], segment[:body])
+      end
+      output.end_with?("\n") ? output : "#{output}\n"
+    end
+
+    def relocate_managed_spans(content, spans)
+      positions = marker_positions(lines_of(content))
+      spans.map do |span|
+        position = positions[span.id]
+        next span unless position
+
+        span.dup.tap do |relocated|
+          relocated.open_line = position[0]
+          relocated.close_line = position[1]
+        end
+      end
+    end
+
+    def split_segments(content)
+      lines = lines_of(content)
+      segments = []
+      text = +""
+      index = 0
+      while index < lines.length
+        identifier = marker_id(lines[index])
+        close = find_closing_marker(lines, index + 1, identifier) if identifier
+        if identifier && close
+          segments << {kind: :text, text: text} unless text.empty?
+          text = +""
+          body_lines = lines[(index + 1)...close]
+          body = body_lines.empty? ? "" : "#{body_lines.join("\n")}\n"
+          segments << {kind: :managed, id: identifier, body: body}
+          index = close + 1
+        else
+          text << "#{lines[index]}\n"
+          index += 1
+        end
+      end
+      segments << {kind: :text, text: text} unless text.empty?
+      segments
+    end
+
+    def lines_of(content)
+      content.lines(chomp: true)
+    end
+
+    def find_closing_marker(lines, start, identifier)
+      (start...lines.length).find { |index| marker_id(lines[index]) == identifier }
+    end
+
+    def marker_positions(lines)
+      positions = {}
+      active = nil
+      lines.each_with_index do |line, index|
+        identifier = marker_id(line)
+        next unless identifier&.match?(/\A[a-z0-9]+\z/)
+
+        if active.nil?
+          active = [identifier, index + 1]
+        elsif active[0] == identifier
+          positions[identifier] = [active[1], index + 1]
+          active = nil
+        end
+      end
+      positions
+    end
+
+    def marker_id(line)
+      match = line.strip.match(/\A<!-- pray:(.+) -->\z/)
+      identifier = match && match[1]
+      identifier unless identifier == "0 ignore-comments"
+    end
+
+    def managed_segment(identifier, body)
+      content = body.empty? ? "" : "#{body.sub(/\n+\z/, "")}\n"
+      "<!-- pray:#{identifier} -->\n#{content}<!-- pray:#{identifier} -->\n"
+    end
+  end
+end

data/lib/pray/resolve.rb

@@ -87,6 +87,7 @@ module Pray
         errors << "#{declaration.name}: #{error.message}"
       end
       raise Error.resolution(errors.join("\n")) unless errors.empty?
+      PackageSpec.warn_resolved_deprecations(packages)
 
       local_files = []
       local_errors = []
@@ -187,16 +188,16 @@ module Pray
         return [File.expand_path(local_path, project_root), nil]
       end
       return [File.expand_path(declaration.path, project_root), nil] if declaration.path
+      source_name = ResolveSource.implied_source_name(declaration, sources)
+      if source_name
+        source = sources[source_name]
+        raise Error.resolution("unknown source: #{source_name}") unless source
 
-      if declaration.source
-        source = sources[declaration.source]
-        raise Error.resolution("unknown source: #{declaration.source}") unless source
-
-        if (local_path = user_config.local.source[declaration.source])
+        if (local_path = user_config.local.source[source_name])
           source_root = File.expand_path(local_path, project_root)
           resolved = Registry.resolve_local_registry_package_root(
             project_root,
-            "local:#{declaration.source}",
+            "local:#{source_name}",
             source_root,
             declaration,
             preferred_version: lockfile_preferred_version(lockfile, declaration.name),
@@ -298,40 +299,11 @@ module Pray
     end
 
     def select_exports(declaration, spec)
-      unless declaration.exports.empty?
-        declaration.exports.each do |export|
-          unless spec.exports.key?(export)
-            raise Error.resolution("package #{declaration.name} does not export #{export}")
-          end
-        end
-        return declaration.exports
-      end
-
-      roles = declaration.roles || []
-      return spec.exports.keys.sort if roles.empty? && declaration.file.nil?
-
-      effective_roles = roles.dup
-      effective_roles << "file" if declaration.file && !effective_roles.include?("file")
+      ResolveExports.select_exports(declaration, spec)
+    end
 
-      selected = []
-      effective_roles.each do |role|
-        compatible = spec.exports.filter_map do |name, export|
-          name if Destination.export_kind_matches_role?(export.kind, role)
-        end
-        case compatible.length
-        when 1
-          selected << compatible.first unless selected.include?(compatible.first)
-        when 0
-          raise Error.resolution(
-            "package #{declaration.name} has no export compatible with #{role}"
-          )
-        else
-          raise Error.resolution(
-            "package #{declaration.name} has multiple exports compatible with #{role}; set export: \"name\""
-          )
-        end
-      end
-      selected
+    def load_export_bodies(file_bytes, spec, selected_exports)
+      ResolveExports.load_export_bodies(file_bytes, spec, selected_exports)
     end
 
     def load_package_file_bytes(root, spec)
@@ -346,21 +318,6 @@ module Pray
       file_bytes
     end
 
-    def load_export_bodies(file_bytes, spec, selected_exports)
-      export_bodies = {}
-      selected_exports.each do |export_name|
-        entry = spec.exports[export_name]
-        raise Error.resolution("package #{spec.name} is missing export #{export_name}") unless entry
-        next unless entry.kind == "fragment"
-
-        bytes = file_bytes[entry.path]
-        raise Error.integrity("package file missing for export #{export_name}: #{entry.path}") unless bytes
-
-        export_bodies[export_name] = Hashing.normalize_line_endings(bytes.force_encoding(Encoding::UTF_8))
-      end
-      export_bodies
-    end
-
     def build_skill_file_index(spec)
       index = {}
       spec.exports.each do |export_name, export|

data/lib/pray/resolve_exports.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+module Pray
+  module ResolveExports
+    module_function
+
+    def select_exports(declaration, spec)
+      unless declaration.exports.empty?
+        declaration.exports.each do |export|
+          unless spec.exports.key?(export)
+            raise Error.resolution("package #{declaration.name} does not export #{export}")
+          end
+        end
+        return declaration.exports
+      end
+
+      roles = declaration.roles || []
+      return spec.exports.keys.sort if roles.empty? && declaration.file.nil?
+
+      effective_roles = roles.dup
+      effective_roles << "file" if declaration.file && !effective_roles.include?("file")
+
+      selected = []
+      effective_roles.each do |role|
+        compatible = if role == "fragment"
+          fragment_role_exports(spec)
+        else
+          spec.exports.filter_map do |name, export|
+            name if Destination.export_kind_matches_role?(export.kind, role)
+          end
+        end
+        case compatible.length
+        when 1
+          selected << compatible.first unless selected.include?(compatible.first)
+        when 0
+          raise Error.resolution(
+            "package #{declaration.name} has no export compatible with #{role}"
+          )
+        else
+          raise Error.resolution(
+            "package #{declaration.name} has multiple exports compatible with #{role}; set export: \"name\""
+          )
+        end
+      end
+      selected
+    end
+
+    def load_export_bodies(file_bytes, spec, selected_exports)
+      export_bodies = {}
+      selected_exports.each do |export_name|
+        entry = spec.exports[export_name]
+        raise Error.resolution("package #{spec.name} is missing export #{export_name}") unless entry
+        next unless %w[fragment file].include?(entry.kind)
+
+        bytes = file_bytes[entry.path]
+        unless bytes
+          raise Error.integrity("package file missing for export #{export_name}: #{entry.path}")
+        end
+
+        text = bytes.dup.force_encoding(Encoding::UTF_8)
+        unless text.valid_encoding?
+          if entry.kind == "fragment"
+            raise Error.integrity("package file is not valid utf-8 for export #{export_name}")
+          end
+          next
+        end
+        export_bodies[export_name] = Hashing.normalize_line_endings(text)
+      end
+      export_bodies
+    end
+
+    def fragment_role_exports(spec)
+      fragments = spec.exports.filter_map { |name, export| name if export.kind == "fragment" }
+      return fragments unless fragments.empty?
+
+      spec.exports.filter_map { |name, export| name if export.kind == "file" }
+    end
+  end
+end

data/lib/pray/resolve_source.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Pray
+  module ResolveSource
+    module_function
+
+    def implied_source_name(declaration, sources)
+      return declaration.source if declaration.source
+
+      namespace = package_namespace(declaration.name)
+      return namespace if namespace && sources.key?(namespace)
+
+      case sources.length
+      when 0 then nil
+      when 1 then sources.keys.first
+      else
+        raise Error.resolution(
+          "package #{declaration.name} requires source: when multiple sources are declared " \
+          "and the package namespace does not match a source"
+        )
+      end
+    end
+
+    def package_namespace(name)
+      name.include?("/") ? name.split("/", 2).first : nil
+    end
+  end
+end

data/lib/pray/resource_limits.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Pray
+  module ResourceLimits
+    MAX_ARCHIVE_ENTRIES = 10_000
+    MAX_ARCHIVE_ENTRY_BYTES = 32 * 1024 * 1024
+    MAX_ARCHIVE_TOTAL_BYTES = 64 * 1024 * 1024
+    MAX_HTTP_RESPONSE_BYTES = 64 * 1024 * 1024
+    MAX_SERVE_BODY_BYTES = 16 * 1024 * 1024
+    MAX_SERVE_HEADER_BYTES = 64 * 1024
+    MAX_SERVE_CONCURRENT_CONNECTIONS = 32
+    SERVE_SOCKET_TIMEOUT_SECONDS = 30
+  end
+end

data/lib/pray/serve.rb

@@ -4,12 +4,14 @@ require "socket"
 require "fileutils"
 require "pathname"
 require "json"
+require "timeout"
 require_relative "path_safety"
+require_relative "resource_limits"
 require_relative "serve_federation"
 
 module Pray
   module Serve
-    DEFAULT_MAX_CONNECTIONS = 8
+    DEFAULT_MAX_CONNECTIONS = ResourceLimits::MAX_SERVE_CONCURRENT_CONNECTIONS
 
     module_function
 
@@ -39,7 +41,14 @@ module Pray
     end
 
     def serve_connection(root, socket, connection_slots)
-      handle_connection(root, socket)
+      Timeout.timeout(ResourceLimits::SERVE_SOCKET_TIMEOUT_SECONDS) do
+        handle_connection(root, socket)
+      end
+    rescue Timeout::Error
+      socket.print(request_timeout) unless socket.closed?
+    rescue Error => error
+      response = error.message.include?("request body exceeds") ? payload_too_large : bad_request
+      socket.print(response) unless socket.closed?
     ensure
       connection_slots << true unless connection_slots.closed?
       socket.close unless socket.closed?
@@ -51,7 +60,9 @@ module Pray
 
       method, path, = request_line.split
       headers = read_headers(socket)
-      body_length = headers["content-length"].to_i
+      body_length = Integer(headers.fetch("content-length", "0"), exception: false)
+      raise Error.parse("request", "invalid content length") unless body_length
+      validate_body_length!(body_length)
       body = body_length.positive? ? socket.read(body_length) : ""
 
       response = dispatch_request(root, method, path, body)
@@ -60,9 +71,14 @@ module Pray
 
     def read_headers(socket)
       headers = {}
+      header_bytes = 0
       loop do
         line = socket.gets
         break if line.nil? || line.strip.empty?
+        header_bytes += line.bytesize
+        if header_bytes > ResourceLimits::MAX_SERVE_HEADER_BYTES
+          raise Error.parse("request", "request headers exceed server limit")
+        end
 
         name, value = line.split(":", 2)
         headers[name.strip.downcase] = value.strip if name && value
@@ -70,10 +86,20 @@ module Pray
       headers
     end
 
+    def validate_body_length!(body_length)
+      return if body_length <= ResourceLimits::MAX_SERVE_BODY_BYTES
+
+      raise Error.unsupported(
+        "request body exceeds #{ResourceLimits::MAX_SERVE_BODY_BYTES} bytes"
+      )
+    end
+
     def dispatch_request(root, method, path, body = "")
       path = path.split("?", 2).first
 
       case [method, path]
+      when ["GET", "/health"]
+        return ok_response("text/plain", "ok")
       when ["GET", "/.well-known/pray-federation.json"]
         return ServeFederation.discovery_response(root)
       when ["GET", "/v1/sync/index"]
@@ -89,12 +115,13 @@ module Pray
       return not_found unless method == "GET"
 
       if path == "/"
-        return html_response("<h1>Pray distribution</h1><p>Root: #{root}</p>")
+        return html_response("<h1>Pray distribution</h1>")
       end
 
       file_path = PathSafety.join_under_root(root, path.delete_prefix("/"))
       return not_found unless file_path
       return not_found unless File.file?(file_path)
+      return payload_too_large if File.size(file_path) > ResourceLimits::MAX_HTTP_RESPONSE_BYTES
 
       content_type = content_type_for(file_path)
       file_body = File.binread(file_path)
@@ -126,5 +153,20 @@ module Pray
       body = "too many connections"
       "HTTP/1.1 503 Service Unavailable\r\nContent-Type: text/plain\r\nContent-Length: #{body.bytesize}\r\nConnection: close\r\n\r\n#{body}"
     end
+
+    def request_timeout
+      body = "request timed out"
+      "HTTP/1.1 408 Request Timeout\r\nContent-Type: text/plain\r\nContent-Length: #{body.bytesize}\r\nConnection: close\r\n\r\n#{body}"
+    end
+
+    def payload_too_large
+      body = "request exceeds server limit"
+      "HTTP/1.1 413 Payload Too Large\r\nContent-Type: text/plain\r\nContent-Length: #{body.bytesize}\r\nConnection: close\r\n\r\n#{body}"
+    end
+
+    def bad_request
+      body = "bad request"
+      "HTTP/1.1 400 Bad Request\r\nContent-Type: text/plain\r\nContent-Length: #{body.bytesize}\r\nConnection: close\r\n\r\n#{body}"
+    end
   end
 end