mt-lang 0.2.0

data/lib/milk_tea/packages/source_resolver.rb

@@ -0,0 +1,389 @@
+# frozen_string_literal: true
+
+require "digest"
+
+module MilkTea
+  class PackageSourceResolverError < StandardError; end
+
+  class PackageSourceResolver
+    attr_reader :resolved_registry_versions
+
+    RegistryDependencyKey = Data.define(:parent_source_key, :dependency_name)
+
+    class SourceIdentity
+      def kind
+        raise NotImplementedError, "#{self.class} must implement #kind"
+      end
+
+      def lock_attributes
+        raise NotImplementedError, "#{self.class} must implement #lock_attributes"
+      end
+
+      def cache_key
+        nil
+      end
+
+      def cacheable?
+        !cache_key.nil?
+      end
+    end
+
+    class PathIdentity < SourceIdentity
+      attr_reader :path
+
+      def initialize(path)
+        @path = File.expand_path(path)
+      end
+
+      def kind
+        :path
+      end
+
+      def lock_attributes
+        {
+          "source_path" => @path,
+        }
+      end
+    end
+
+    class RegistryIdentity < SourceIdentity
+      attr_reader :package_name, :version
+
+      def initialize(package_name:, version:)
+        @package_name = normalize_required(package_name, "registry package name")
+        @version = normalize_required(version, "registry package version")
+      end
+
+      def kind
+        :registry
+      end
+
+      def lock_attributes
+        {
+          "registry_package" => @package_name,
+          "registry_version" => @version,
+        }
+      end
+
+      def cache_key
+        "registry/#{@package_name}@#{@version}"
+      end
+
+      private
+
+      def normalize_required(value, label)
+        text = value.to_s.strip
+        raise PackageSourceResolverError, "#{label} cannot be empty" if text.empty?
+        if text.include?(File::SEPARATOR) || (File::ALT_SEPARATOR && text.include?(File::ALT_SEPARATOR))
+          raise PackageSourceResolverError, "#{label} cannot contain path separators"
+        end
+
+        text
+      end
+    end
+
+    class GitIdentity < SourceIdentity
+      attr_reader :url, :revision, :subdir
+
+      def initialize(url:, revision:, subdir: nil)
+        @url = normalize_required(url, "git source url")
+        @revision = normalize_required(revision, "git source revision")
+        @subdir = normalize_optional(subdir)
+      end
+
+      def kind
+        :git
+      end
+
+      def lock_attributes
+        attributes = {
+          "git_url" => @url,
+          "git_rev" => @revision,
+        }
+        attributes["git_subdir"] = @subdir if @subdir
+        attributes
+      end
+
+      def cache_key
+        digest = Digest::SHA256.hexdigest([@url, @revision, @subdir].join("\0"))
+        "git/#{digest}"
+      end
+
+      private
+
+      def normalize_required(value, label)
+        text = value.to_s.strip
+        raise PackageSourceResolverError, "#{label} cannot be empty" if text.empty?
+
+        text
+      end
+
+      def normalize_optional(value)
+        text = value.to_s.strip
+        return nil if text.empty?
+        raise PackageSourceResolverError, "git source subdir cannot be absolute" if text.start_with?("/", "\\")
+        raise PackageSourceResolverError, "git source subdir cannot be absolute" if text.match?(/\A[A-Za-z]:[\\\/]/)
+
+        segments = text.split(/[\\\/]+/).reject(&:empty?)
+        normalized_segments = []
+        segments.each do |segment|
+          next if segment == "."
+          raise PackageSourceResolverError, "git source subdir cannot escape the repository root" if segment == ".."
+
+          normalized_segments << segment
+        end
+
+        normalized = normalized_segments.join("/")
+        normalized.empty? ? nil : normalized
+      end
+    end
+
+    Source = Data.define(:identity, :local_root) do
+      def kind
+        identity.kind
+      end
+
+      def lock_attributes
+        identity.lock_attributes
+      end
+
+      def cache_key
+        identity.cache_key
+      end
+
+      def cacheable?
+        identity.cacheable?
+      end
+    end
+
+    ResolvedPackage = Data.define(:manifest, :source)
+
+    def self.source_key_for_identity(identity)
+      return "path:#{File.expand_path(identity.path)}" if identity.is_a?(PathIdentity)
+      return "#{identity.kind}:#{identity.cache_key}" if identity.cache_key
+
+      raise PackageSourceResolverError, "unsupported source identity #{identity.class} for registry dependency key"
+    end
+
+    def self.registry_dependency_key(parent_source_key:, dependency_name:)
+      RegistryDependencyKey.new(
+        parent_source_key: parent_source_key.to_s,
+        dependency_name: dependency_name.to_s,
+      )
+    end
+
+    def initialize(source_cache: PackageSourceCache.new, remote_resolution: :reject, source_fetcher: nil, resolved_registry_versions: nil)
+      @source_cache = source_cache
+      @remote_resolution = remote_resolution
+      @source_fetcher = source_fetcher
+      @resolved_registry_versions = normalize_resolved_registry_versions(resolved_registry_versions)
+    end
+
+    def supports_dependency_solving?
+      @remote_resolution != :reject
+    end
+
+    def with_resolved_registry_versions(resolved_registry_versions)
+      self.class.new(
+        source_cache: @source_cache,
+        remote_resolution: @remote_resolution,
+        source_fetcher: @source_fetcher,
+        resolved_registry_versions:,
+      )
+    end
+
+    def source_for_manifest(manifest)
+      Source.new(
+        identity: PathIdentity.new(manifest.root_dir),
+        local_root: manifest.root_dir,
+      )
+    end
+
+    def identity_from_lock(entry, lock_path)
+      source_kind = entry["source_kind"].to_s
+
+      case source_kind
+      when "path"
+        source_path = entry["source_path"].to_s
+        raise PackageSourceResolverError, "package.lock missing source_path in #{lock_path}" if source_path.empty?
+
+        PathIdentity.new(source_path)
+      when "registry"
+        RegistryIdentity.new(
+          package_name: entry["registry_package"],
+          version: entry["registry_version"],
+        )
+      when "git"
+        GitIdentity.new(
+          url: entry["git_url"],
+          revision: entry["git_rev"],
+          subdir: entry["git_subdir"],
+        )
+      else
+        raise PackageSourceResolverError,
+              "unsupported package source_kind #{source_kind.inspect} in #{lock_path}; supported kinds are path, registry, and git"
+      end
+    end
+
+    def resolve(dependency, parent_manifest:, parent_source: nil)
+      if dependency.path
+        manifest = PackageManifest.load(dependency.path)
+        validate_fixed_dependency_version_requirement!(dependency, manifest, parent_manifest:)
+        return ResolvedPackage.new(manifest:, source: source_for_manifest(manifest))
+      end
+
+      return resolve_registry_dependency(dependency, parent_manifest:, parent_source:) if dependency.registry?
+      return resolve_git_dependency(dependency, parent_manifest:) if dependency.git
+
+      raise PackageSourceResolverError,
+            "dependency #{dependency.name} in #{parent_manifest.manifest_path} has no supported source resolver"
+    end
+
+    def source_from_lock(entry, lock_path)
+      identity = identity_from_lock(entry, lock_path)
+      source_for_identity(identity, lock_path)
+    rescue PackageSourceCacheError => e
+      raise PackageSourceResolverError, e.message
+    end
+
+    private
+
+    def resolve_registry_dependency(dependency, parent_manifest:, parent_source: nil)
+      if dependency.exact_registry_version?
+        identity = RegistryIdentity.new(package_name: dependency.name, version: dependency.version)
+        return resolve_cache_backed_dependency(identity, dependency:, parent_manifest:)
+      end
+
+      parent_source_key = registry_dependency_parent_source_key(parent_manifest:, parent_source:)
+      dependency_key = self.class.registry_dependency_key(parent_source_key:, dependency_name: dependency.name)
+      resolved_version = @resolved_registry_versions[dependency_key] || @resolved_registry_versions[dependency.name]
+      if resolved_version
+        unless dependency.version_req.matches?(resolved_version)
+          raise PackageSourceResolverError,
+                "dependency #{dependency.name} in #{parent_manifest.manifest_path} resolved version #{resolved_version.inspect}, which does not satisfy #{dependency.version_req}"
+        end
+
+        identity = RegistryIdentity.new(package_name: dependency.name, version: resolved_version)
+        return resolve_cache_backed_dependency(identity, dependency:, parent_manifest:)
+      end
+
+      raise PackageSourceResolverError,
+        "dependency #{dependency.name} in #{parent_manifest.manifest_path} uses version requirement #{dependency.version_req}, but registry dependency solving is not implemented yet; use an exact version for now"
+    end
+
+    def resolve_git_dependency(dependency, parent_manifest:)
+      identity = GitIdentity.new(
+        url: dependency.git,
+        revision: dependency.git_rev,
+        subdir: dependency.git_subdir,
+      )
+
+      resolve_cache_backed_dependency(identity, dependency:, parent_manifest:)
+    end
+
+    def resolve_cache_backed_dependency(identity, dependency:, parent_manifest:)
+      context_label = "dependency #{dependency.name} in #{parent_manifest.manifest_path}"
+      source = case @remote_resolution
+               when :reject
+                 raise PackageSourceResolverError,
+                       "#{context_label} uses #{identity.kind} resolution, but live dependency resolution only supports local path dependencies; run mtc deps lock and then use --locked or --frozen"
+               when :cache
+                 source_for_identity(identity, context_label)
+               when :materialize
+                 unless @source_fetcher
+                   raise PackageSourceResolverError,
+                         "#{context_label} requested #{identity.kind} materialization without a package source fetcher"
+                 end
+
+                 @source_fetcher.materialize_identity(package_name: dependency.name, identity:)
+                 source_for_identity(identity, context_label)
+               else
+                 raise PackageSourceResolverError, "unknown remote resolution mode #{@remote_resolution.inspect}"
+               end
+
+      manifest = PackageManifest.load(source.local_root)
+      if manifest.package_name != dependency.name
+        raise PackageSourceResolverError,
+              "#{context_label} resolved package #{manifest.package_name} at #{manifest.manifest_path}"
+      end
+
+      if identity.is_a?(RegistryIdentity) && manifest.package_version != identity.version
+        raise PackageSourceResolverError,
+              "#{context_label} resolved version #{manifest.package_version.inspect} at #{manifest.manifest_path}; expected #{identity.version.inspect}"
+      end
+
+      ResolvedPackage.new(manifest:, source:)
+    rescue PackageManifestError => e
+      raise PackageSourceResolverError, e.message
+    end
+
+    def validate_fixed_dependency_version_requirement!(dependency, manifest, parent_manifest:)
+      return unless dependency.version_req
+
+      package_version = manifest.package_version
+      if package_version.nil?
+        raise PackageSourceResolverError,
+              "dependency #{dependency.name} in #{parent_manifest.manifest_path} requires version #{dependency.version_req}, but #{manifest.manifest_path} does not declare package.version"
+      end
+
+      return if dependency.version_req.matches?(package_version)
+
+      raise PackageSourceResolverError,
+            "dependency #{dependency.name} in #{parent_manifest.manifest_path} resolved version #{package_version.inspect} at #{manifest.manifest_path}, which does not satisfy #{dependency.version_req}"
+    end
+
+    def source_for_identity(identity, context_label)
+      local_root = if identity.is_a?(PathIdentity)
+                     identity.path
+                   else
+                     manifest_path = @source_cache.manifest_path_for(identity)
+                     unless File.file?(manifest_path)
+                       raise PackageSourceResolverError,
+                             "package source_kind #{identity.kind.inspect} in #{context_label} is not materialized in the source cache: expected #{manifest_path}"
+                     end
+
+                     @source_cache.materialized_root_for(identity)
+                   end
+
+      Source.new(
+        identity:,
+        local_root:,
+      )
+    end
+
+    def registry_dependency_parent_source_key(parent_manifest:, parent_source:)
+      identity = if parent_source
+                   parent_source.identity
+                 else
+                   source_for_manifest(parent_manifest).identity
+                 end
+
+      self.class.source_key_for_identity(identity)
+    end
+
+    def normalize_resolved_registry_versions(versions)
+      return {} unless versions
+
+      versions.each_with_object({}) do |(key, version), resolved|
+        normalized_key = case key
+                         when RegistryDependencyKey
+                           self.class.registry_dependency_key(
+                             parent_source_key: key.parent_source_key,
+                             dependency_name: key.dependency_name,
+                           )
+                         when Array
+                           if key.length != 2
+                             raise PackageSourceResolverError,
+                                   "registry dependency key must contain parent manifest path and dependency name"
+                           end
+
+                           self.class.registry_dependency_key(parent_source_key: key[0], dependency_name: key[1])
+                         else
+                           key.to_s
+                         end
+
+        resolved[normalized_key] = version.to_s
+      end
+    end
+  end
+end

data/lib/milk_tea/packages/version.rb

@@ -0,0 +1,164 @@
+# frozen_string_literal: true
+
+module MilkTea
+  class PackageVersionError < StandardError; end
+
+  class PackageVersion
+    include Comparable
+
+    VERSION_PATTERN = /\A(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)\z/
+
+    attr_reader :major, :minor, :patch
+
+    def self.parse(value, label: "package version")
+      return value if value.is_a?(self)
+
+      text = value.to_s.strip
+      raise PackageVersionError, "#{label} cannot be empty" if text.empty?
+
+      match = VERSION_PATTERN.match(text)
+      unless match
+        raise PackageVersionError,
+              "#{label} #{text.inspect} must use semantic version format MAJOR.MINOR.PATCH"
+      end
+
+      new(match[1].to_i, match[2].to_i, match[3].to_i)
+    end
+
+    def initialize(major, minor, patch)
+      @major = Integer(major)
+      @minor = Integer(minor)
+      @patch = Integer(patch)
+    end
+
+    def <=>(other)
+      return nil unless other.is_a?(PackageVersion)
+
+      [major, minor, patch] <=> [other.major, other.minor, other.patch]
+    end
+
+    def to_s
+      "#{major}.#{minor}.#{patch}"
+    end
+  end
+
+  class PackageVersionReq
+    Constraint = Data.define(:operator, :version) do
+      def matches?(candidate)
+        case operator
+        when :eq
+          candidate == version
+        when :gt
+          candidate > version
+        when :gte
+          candidate >= version
+        when :lt
+          candidate < version
+        when :lte
+          candidate <= version
+        else
+          raise PackageVersionError, "unsupported package version operator #{operator.inspect}"
+        end
+      end
+
+      def to_s
+        prefix = case operator
+                 when :eq then "="
+                 when :gt then ">"
+                 when :gte then ">="
+                 when :lt then "<"
+                 when :lte then "<="
+                 else
+                   raise PackageVersionError, "unsupported package version operator #{operator.inspect}"
+                 end
+
+        "#{prefix}#{version}"
+      end
+    end
+
+    attr_reader :constraints
+
+    def self.parse(value, label: "package version requirement")
+      text = value.to_s.strip
+      raise PackageVersionError, "#{label} cannot be empty" if text.empty?
+
+      segments = text.split(",", -1).map(&:strip)
+      if segments.empty? || segments.any?(&:empty?)
+        raise PackageVersionError, "#{label} #{text.inspect} contains an empty constraint"
+      end
+
+      constraints = segments.flat_map do |segment|
+        parse_segment(segment, label:)
+      end
+
+      new(constraints)
+    end
+
+    def self.parse_segment(segment, label:)
+      case segment
+      when /\A\^(.+)\z/
+        version = PackageVersion.parse(Regexp.last_match(1), label:)
+        [Constraint.new(:gte, version), Constraint.new(:lt, caret_upper_bound(version))]
+      when /\A~(.+)\z/
+        version = PackageVersion.parse(Regexp.last_match(1), label:)
+        [Constraint.new(:gte, version), Constraint.new(:lt, tilde_upper_bound(version))]
+      when /\A(>=|<=|>|<|=)\s*(.+)\z/
+        version = PackageVersion.parse(Regexp.last_match(2), label:)
+        [Constraint.new(operator_for(Regexp.last_match(1)), version)]
+      else
+        [Constraint.new(:eq, PackageVersion.parse(segment, label:))]
+      end
+    end
+    private_class_method :parse_segment
+
+    def self.operator_for(token)
+      case token
+      when "=" then :eq
+      when ">" then :gt
+      when ">=" then :gte
+      when "<" then :lt
+      when "<=" then :lte
+      else
+        raise PackageVersionError, "unsupported package version operator #{token.inspect}"
+      end
+    end
+    private_class_method :operator_for
+
+    def self.caret_upper_bound(version)
+      if version.major.positive?
+        PackageVersion.new(version.major + 1, 0, 0)
+      elsif version.minor.positive?
+        PackageVersion.new(0, version.minor + 1, 0)
+      else
+        PackageVersion.new(0, 0, version.patch + 1)
+      end
+    end
+    private_class_method :caret_upper_bound
+
+    def self.tilde_upper_bound(version)
+      PackageVersion.new(version.major, version.minor + 1, 0)
+    end
+    private_class_method :tilde_upper_bound
+
+    def initialize(constraints)
+      @constraints = constraints.freeze
+    end
+
+    def exact?
+      constraints.length == 1 && constraints.first.operator == :eq
+    end
+
+    def exact_version
+      constraints.first.version if exact?
+    end
+
+    def matches?(version)
+      candidate = PackageVersion.parse(version)
+      constraints.all? { |constraint| constraint.matches?(candidate) }
+    end
+
+    def to_s
+      constraints.map(&:to_s).join(", ")
+    end
+  end
+end

data/lib/milk_tea/packages.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require_relative "packages/version"
+require_relative "packages/manifest"
+require_relative "packages/source_cache"
+require_relative "packages/registry_store"
+require_relative "packages/source_resolver"
+require_relative "packages/source_fetcher"
+require_relative "packages/registry_metadata_provider"
+require_relative "packages/dependency_solver"
+require_relative "packages/graph"
+require_relative "packages/atomic_write"
+require_relative "packages/lock"
+require_relative "packages/manifest_editor"
+require_relative "packages/services"
+require_relative "packages/manager_cli"

data/lib/milk_tea/tooling/asset_pack.rb

@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "tempfile"
+
+module MilkTea
+  class AssetPackError < StandardError; end
+
+  class AssetPack
+    MAGIC = "MTAP".b.freeze
+    VERSION = 1
+    HEADER_FLAGS = 0
+    ENTRY_FLAGS_RAW = 0
+    HEADER_FORMAT = "a4vvVQ<Q<".freeze
+    ENTRY_PREFIX_FORMAT = "VVQ<Q<Q<".freeze
+    HEADER_SIZE = [MAGIC, VERSION, HEADER_FLAGS, 0, 0, 0].pack(HEADER_FORMAT).bytesize
+    ENTRY_PREFIX_SIZE = [0, ENTRY_FLAGS_RAW, 0, 0, 0].pack(ENTRY_PREFIX_FORMAT).bytesize
+
+    SourceEntry = Data.define(:source_path, :logical_path, :stored_size, :unpacked_size, :flags)
+    Entry = Data.define(:source_path, :logical_path, :data_offset, :stored_size, :unpacked_size, :flags)
+
+    def self.write(output_path, source_paths)
+      new(output_path, source_paths).write
+    end
+
+    def initialize(output_path, source_paths)
+      @output_path = File.expand_path(output_path)
+      @source_paths = source_paths.map { |source_path| File.expand_path(source_path) }
+    end
+
+    def write
+      ensure_output_does_not_overlap_sources!
+
+      source_entries = collect_source_entries
+      index_size = source_entries.sum { |entry| ENTRY_PREFIX_SIZE + entry.logical_path.bytesize }
+      data_offset = HEADER_SIZE + index_size
+      entries = assign_offsets(source_entries, data_offset)
+
+      FileUtils.mkdir_p(File.dirname(@output_path))
+      Tempfile.create(["milk-tea-asset-pack", ".bin"], File.dirname(@output_path)) do |file|
+        file.binmode
+        write_header(file, entries.length, index_size, data_offset)
+        write_index(file, entries)
+        write_data(file, entries)
+        file.flush
+        file.close
+
+        FileUtils.rm_f(@output_path)
+        FileUtils.mv(file.path, @output_path)
+      end
+
+      @output_path
+    end
+
+    private
+
+    def ensure_output_does_not_overlap_sources!
+      @source_paths.each do |source_path|
+        if File.directory?(source_path)
+          next unless path_within?(@output_path, source_path)
+
+          raise AssetPackError, "asset pack output would be written inside source tree: #{@output_path}"
+        end
+
+        next unless @output_path == source_path
+
+        raise AssetPackError, "asset pack output would overwrite source file: #{@output_path}"
+      end
+    end
+
+    def collect_source_entries
+      @source_paths.flat_map do |source_path|
+        collect_source_path(source_path, File.basename(source_path))
+      end.sort_by(&:logical_path)
+    end
+
+    def collect_source_path(source_path, logical_path)
+      stat = File.lstat(source_path)
+
+      if stat.directory?
+        Dir.children(source_path).sort.flat_map do |child|
+          collect_source_path(File.join(source_path, child), join_logical_path(logical_path, child))
+        end
+      elsif stat.file?
+        file_size = File.size(source_path)
+        [SourceEntry.new(source_path:, logical_path:, stored_size: file_size, unpacked_size: file_size, flags: ENTRY_FLAGS_RAW)]
+      else
+        raise AssetPackError, "asset pack supports only regular files and directories: #{source_path}"
+      end
+    rescue Errno::ENOENT
+      raise AssetPackError, "asset pack source not found: #{source_path}"
+    end
+
+    def assign_offsets(source_entries, first_data_offset)
+      next_offset = first_data_offset
+
+      source_entries.map do |entry|
+        packed_entry = Entry.new(
+          source_path: entry.source_path,
+          logical_path: entry.logical_path,
+          data_offset: next_offset,
+          stored_size: entry.stored_size,
+          unpacked_size: entry.unpacked_size,
+          flags: entry.flags,
+        )
+        next_offset += entry.stored_size
+        packed_entry
+      end
+    end
+
+    def write_header(io, entry_count, index_size, data_offset)
+      io.write([MAGIC, VERSION, HEADER_FLAGS, entry_count, index_size, data_offset].pack(HEADER_FORMAT))
+    end
+
+    def write_index(io, entries)
+      entries.each do |entry|
+        path_bytes = entry.logical_path.b
+        io.write([path_bytes.bytesize, entry.flags, entry.data_offset, entry.stored_size, entry.unpacked_size].pack(ENTRY_PREFIX_FORMAT))
+        io.write(path_bytes)
+      end
+    end
+
+    def write_data(io, entries)
+      entries.each do |entry|
+        File.open(entry.source_path, "rb") do |source_file|
+          IO.copy_stream(source_file, io)
+        end
+      end
+    end
+
+    def join_logical_path(left, right)
+      "#{left}/#{right}"
+    end
+
+    def path_within?(path, root)
+      normalized_path = File.expand_path(path)
+      normalized_root = File.expand_path(root)
+      normalized_path == normalized_root || normalized_path.start_with?(normalized_root + File::SEPARATOR)
+    end
+  end
+end