xccache 0.0.1 → 1.0.0.rc15341775774

data/lib/xccache/installer.rb

@@ -1,9 +1,11 @@
 require "xccache/spm"
+require "xccache/installer/integration"
 Dir["#{__dir__}/#{File.basename(__FILE__, '.rb')}/*.rb"].sort.each { |f| require f }
 
 module XCCache
   class Installer
     include PkgMixin
+    include IntegrationMixin
 
     def initialize(options = {})
       ctx = options[:ctx]
@@ -14,29 +16,33 @@ module XCCache
     end
 
     def perform_install
+      verify_projects!
+      projects.each { |project| migrate_umbrella_to_proxy(project) }
       UI.message("Using cache dir: #{config.spm_cache_dir}")
       config.in_installation = true
-      if @umbrella_pkg.nil?
-        sync_lockfile
-        umbrella_pkg.prepare(**@install_options)
-      end
+      sync_lockfile
+      proxy_pkg.prepare(@install_options)
+
+      yield if block_given?
 
-      yield
-      umbrella_pkg.write_manifest
-      umbrella_pkg.gen_xcconfigs
+      gen_supporting_files
       projects.each do |project|
         add_xccache_refs_to_project(project)
         inject_xcconfig_to_project(project)
       end
-      umbrella_pkg.gen_cachemap_viz
+      gen_cachemap_viz
     end
 
     def sync_lockfile
       UI.info("Syncing lockfile")
+      known_dependencies = lockfile.known_product_dependencies
       update_projects do |project|
-        lockfile.deep_merge!(project.display_name => lockfile_hash_for_project(project))
+        lockfile.deep_merge!(
+          project.display_name => lockfile_hash_for_project(project, known_dependencies)
+        )
       end
       lockfile.save
+      lockfile.verify!
     end
 
     def lockfile
@@ -53,7 +59,6 @@ module XCCache
     end
 
     def update_projects
-      verify_projects!
       projects.each do |project|
         yield project if block_given?
         project.save
@@ -62,17 +67,29 @@ module XCCache
 
     private
 
-    def lockfile_hash_for_project(project)
+    def lockfile_hash_for_project(project, known_dependencies)
       deps_by_targets = project.targets.to_h do |target|
-        deps = target.non_xccache_pkg_product_dependencies.select(&:pkg).map { |d| "#{d.pkg.slug}/#{d.product_name}" }
-        [target.name, deps]
+        deps = target.non_xccache_pkg_product_dependencies.map do |dep|
+          next dep.full_name unless dep.pkg.nil?
+          known = known_dependencies.find { |x| File.basename(x) == dep.product_name }
+          UI.warn("-> Assuming #{known} for #{dep.full_name}".dark) if known
+          known || dep.full_name
+        end
+        [target.name, deps.sort]
       end
       {
         "packages" => project.non_xccache_pkgs.map(&:to_h),
         "dependencies" => deps_by_targets,
+        "platforms" => platforms_for_project(project),
       }
     end
 
+    def platforms_for_project(project)
+      project
+        .targets.map { |t| [t.platform_name.to_s, t.deployment_target] }
+        .sort.reverse.to_h # sort descendingly -> min value is picked for the hash
+    end
+
     def verify_projects!
       raise "No projects detected. Are you running on the correct project directory?" if projects.empty?
     end
@@ -80,7 +97,7 @@ module XCCache
     def add_xccache_refs_to_project(project)
       group = project.xccache_config_group
       add_file = proc { |p| group[p.basename.to_s] || group.new_file(p) }
-      add_file.call(config.spm_umbrella_sandbox / "Package.swift")
+      add_file.call(config.spm_proxy_sandbox / "Package.swift")
       add_file.call(config.lockfile.path)
       add_file.call(config.path) if config.path.exist?
       group.ensure_synced_group(name: "local-packages", path: config.spm_local_pkgs_dir)
@@ -110,5 +127,21 @@ module XCCache
         end
       end
     end
+
+    def migrate_umbrella_to_proxy(project)
+      return unless project.xccache_pkg&.slug == "umbrella"
+
+      UI.info <<~DESC
+        Migrating from umbrella to proxy for project #{project.display_name}
+        You should notice changes in project files from xccache/package/umbrella -> xccache/package/proxy.
+        Don't worry, this is expected.
+      DESC
+        .yellow
+
+      project.xccache_pkg.relative_path = "xccache/packages/proxy"
+      if (group = project.xccache_config_group) && (ref = group["Package.swift"])
+        ref.path = "xccache/packages/proxy/Package.swift"
+      end
+    end
   end
 end

data/lib/xccache/spm/build.rb

@@ -35,7 +35,7 @@ module XCCache
           cmd << "-Xswiftc" << "-emit-module-interface"
           cmd << "-Xswiftc" << "-no-verify-emitted-module-interface"
         end
-        Sh.run(cmd, suppress_err: /(dependency '.*' is not used by any target|unable to create symbolic link)/)
+        Sh.run(cmd)
       end
 
       def swift_build_args

data/lib/xccache/spm/desc/base.rb

@@ -57,10 +57,7 @@ module XCCache
         end
 
         def src_dir
-          @src_dir ||= begin
-            path = root.raw.fetch("packageKind", {}).fetch("root", [])[0]
-            Pathname.new(path) unless path.nil?
-          end
+          @src_dir ||= Pathname(root.raw["path"]).parent
         end
       end
     end

data/lib/xccache/spm/desc/desc.rb

@@ -5,33 +5,9 @@ module XCCache
     class Package
       class Description < BaseObject
         include Cacheable
-        cacheable :resolve_recursive_dependencies
 
-        def self.descs_in_metadata_dir
-          descs = Config.instance.spm_metadata_dir.glob("*.json").map { |p| Description.new(p) }
-          [descs, combine_descs(descs)]
-        end
-
-        def self.in_dir(dir, save_to_dir: nil)
-          path = save_to_dir / "#{dir.basename}.json" unless save_to_dir.nil?
-          begin
-            raw = JSON.parse(Sh.capture_output("swift package dump-package --package-path #{dir}"))
-            Description.new(path, raw: raw)
-          rescue StandardError => e
-            UI.error("Failed to dump package in #{dir}. Error: #{e}")
-          end
-        end
-
-        def self.descs_in_dir(root_dir, save_to_dir: nil)
-          dirs = [root_dir] + root_dir.glob(".build/checkouts/*").reject { |p| p.glob("Package*.swift").empty? }
-          descs = dirs.parallel_map do |dir|
-            desc = Description.in_dir(dir, save_to_dir: save_to_dir)
-            unless save_to_dir.nil?
-              desc.save
-              desc.save(to: desc.path.parent / "#{desc.name}.json") if desc.name != dir.basename.to_s
-            end
-            desc
-          end
+        def self.descs_in_metadata_dir(dir)
+          descs = dir.glob("*.json").map { |p| Description.new(p) }
           [descs, combine_descs(descs)]
         end
 
@@ -77,10 +53,6 @@ module XCCache
             .flat_map { |p| targets.select { |t| p.target_names.include?(t.name) } }
         end
 
-        def resolve_recursive_dependencies(platform: nil)
-          products.to_h { |p| [p, p.recursive_targets(platform: platform)] }
-        end
-
         def local?
           # Workaround: If the pkg dir is under the build checkouts dir -> remote
           !src_dir.to_s.start_with?((config.spm_build_dir / "checkouts").to_s)

data/lib/xccache/spm/mixin.rb

@@ -3,10 +3,11 @@ module XCCache
     include Config::Mixin
 
     def umbrella_pkg
-      @umbrella_pkg ||= SPM::Package::Umbrella.new(
-        root_dir: config.spm_umbrella_sandbox,
-        warn_if_not_direct_target: false,
-      )
+      proxy_pkg.umbrella
+    end
+
+    def proxy_pkg
+      @proxy_pkg ||= SPM::Package::Proxy.new(root_dir: config.spm_proxy_sandbox)
     end
   end
 end

data/lib/xccache/spm/pkg/base.rb

@@ -2,11 +2,15 @@ require "json"
 require "xccache/spm/xcframework/slice"
 require "xccache/spm/xcframework/xcframework"
 require "xccache/spm/xcframework/metadata"
+require "xccache/spm/pkg/proxy"
 require "xccache/swift/sdk"
 
 module XCCache
   module SPM
     class Package
+      include Config::Mixin
+      include Proxy::Mixin
+
       include Cacheable
       cacheable :pkg_desc_of_target
 
@@ -14,7 +18,6 @@ module XCCache
 
       def initialize(options = {})
         @root_dir = Pathname(options[:root_dir] || ".").expand_path
-        @warn_if_not_direct_target = options.fetch(:warn_if_not_direct_target, true)
       end
 
       def build(options = {})
@@ -33,7 +36,10 @@ module XCCache
       end
 
       def build_target(target: nil, sdks: nil, config: nil, out_dir: nil, **options)
-        target_pkg_desc = pkg_desc_of_target(target, skip_resolving_dependencies: options[:skip_resolving_dependencies])
+        target_pkg_desc = pkg_desc_of_target(
+          target,
+          ensure_exist: true,
+        )
         if target_pkg_desc.binary_targets.any? { |t| t.name == target }
           return UI.warn("Target #{target} is a binary target -> no need to build")
         end
@@ -42,33 +48,44 @@ module XCCache
 
         out_dir = Pathname(out_dir || ".")
         out_dir /= target.name if options[:checksum]
+        ext = target.macro? ? ".macro" : ".xcframework"
         basename = options[:checksum] ? "#{target.name}-#{target.checksum}" : target.name
-        basename += target.macro? ? ".macro" : ".xcframework"
+        binary_path = out_dir / "#{basename}#{ext}"
 
-        Dir.create_tmpdir do |_tmpdir|
+        Dir.create_tmpdir do |tmpdir|
           cls = target.macro? ? Macro : XCFramework
           cls.new(
             name: target.name,
             pkg_dir: root_dir,
             config: config,
             sdks: sdks,
-            path: out_dir / basename,
-            tmpdir: Dir.create_tmpdir,
+            path: binary_path,
+            tmpdir: tmpdir,
             pkg_desc: target_pkg_desc,
             library_evolution: options[:library_evolution],
           ).build(**options)
         end
+        return if (symlinks_dir = options[:symlinks_dir]).nil?
+        binary_path.symlink_to(symlinks_dir / target.name / "#{target.name}#{ext}")
       end
 
-      def resolve(force: false)
-        return if @resolved && !force
-
-        UI.section("Resolving package dependencies (package: #{root_dir.basename})", timing: true) do
-          Sh.run("swift package resolve --package-path #{root_dir} 2>&1")
-        end
+      def resolve
+        return if @resolved
+        xccache_proxy.run("resolve --pkg #{root_dir} --metadata #{metadata_dir}")
         @resolved = true
       end
 
+      def pkg_desc_of_target(name, **options)
+        resolve
+        desc = descs.find { |d| d.has_target?(name) }
+        raise GeneralError, "Cannot find package with the given target #{name}" if options[:ensure_exist] && desc.nil?
+        desc
+      end
+
+      def get_target(name)
+        pkg_desc_of_target(name)&.get_target(name)
+      end
+
       private
 
       def validate!
@@ -76,31 +93,20 @@ module XCCache
         raise GeneralError, "No Package.swift in #{root_dir}. Are you sure you're running on a package dir?"
       end
 
-      def pkg_desc_of_target(name, skip_resolving_dependencies: false)
-        # The current package contains the given target
-        return pkg_desc if pkg_desc.has_target?(name)
+      def metadata_dir
+        config.in_installation? ? config.spm_metadata_dir : root_dir / ".build/metadata"
+      end
 
-        if @warn_if_not_direct_target
-          UI.message(
-            "#{name.yellow.dark} is not a direct target of package #{root_dir.basename.to_s.dark} " \
-            "-> trigger from dependencies"
-          )
-        end
-        # Otherwise, it's inside one of the dependencies. Need to resolve then find it
-        resolve unless skip_resolving_dependencies
-
-        @descs ||= if Config.instance.in_installation?
-                   then Description.descs_in_metadata_dir[0]
-                   else
-                     Description.descs_in_dir(Pathname(".").expand_path)[0]
-                   end
-        desc = @descs.find { |d| d.has_target?(name) }
-        return desc unless desc.nil?
-        raise GeneralError, "Cannot find package with the given target #{name}"
+      def descs
+        @descs ||= load_descs[0]
+      end
+
+      def descs_by_name
+        @descs_by_name ||= load_descs[1]
       end
 
-      def pkg_desc
-        @pkg_desc ||= Description.in_dir(root_dir)
+      def load_descs
+        @descs, @descs_by_name = Description.descs_in_metadata_dir(metadata_dir)
       end
     end
   end

data/lib/xccache/spm/pkg/proxy.rb

@@ -0,0 +1,60 @@
+Dir["#{__dir__}/#{File.basename(__FILE__, '.rb')}/*.rb"].sort.each { |f| require f }
+require_relative "proxy_executable"
+
+module XCCache
+  module SPM
+    class Package
+      class Proxy < Package
+        module Mixin
+          def xccache_proxy
+            @xccache_proxy ||= Executable.new
+          end
+        end
+
+        include Mixin
+
+        def umbrella
+          @umbrella ||= Package.new(root_dir: config.spm_umbrella_sandbox)
+        end
+
+        def prepare(options = {})
+          xccache_proxy.run("gen-umbrella")
+          umbrella.resolve
+          invalidate_cache(sdks: options[:sdks])
+          gen_proxy
+        end
+
+        def gen_proxy
+          xccache_proxy.run("gen-proxy")
+          config.cachemap.update_from_graph(graph.reload)
+        end
+
+        def invalidate_cache(sdks: [])
+          UI.message("Invalidating cache (sdks: #{sdks.map(&:name)})")
+
+          config.spm_cache_dir.glob("*/*.{xcframework,macro}").each do |p|
+            cmps = p.basename(".*").to_s.split("-")
+            name, checksum = cmps[...-1].join("-"), cmps[-1]
+            p_without_checksum = config.spm_binaries_dir / name / "#{name}#{p.extname}"
+            accept_cache = proc { p.symlink_to(p_without_checksum) }
+            reject_cache = proc { p_without_checksum.rmtree if p_without_checksum.exist? }
+            next reject_cache.call if (target = umbrella.get_target(name)).nil?
+            next reject_cache.call if target.checksum != checksum
+            # For macro, we just need the tool binary to exist
+            next accept_cache.call if target.macro?
+
+            # For regular targets, the xcframework must satisfy the sdk constraints (ie. containing all the slices)
+            metadata = XCFramework::Metadata.new(p / "Info.plist")
+            expected_triples = sdks.map { |sdk| sdk.triple(without_vendor: true) }
+            missing_triples = expected_triples - metadata.triples
+            missing_triples.empty? ? accept_cache.call : reject_cache.call
+          end
+        end
+
+        def graph
+          @graph ||= JSONRepresentable.new(root_dir / "graph.json")
+        end
+      end
+    end
+  end
+end

data/lib/xccache/spm/pkg/proxy_executable.rb

@@ -0,0 +1,51 @@
+module XCCache
+  module SPM
+    class Package
+      class Proxy < Package
+        class Executable
+          VERSION = "0.0.1rc3".freeze
+
+          def run(cmd)
+            env = { "FORCE_OUTPUT" => "console", "FORCE_COLOR" => "1" } if Config.instance.ansi?
+            cmd = cmd.is_a?(Array) ? [bin_path.to_s] + cmd : [bin_path.to_s, cmd]
+            Sh.run(cmd, env: env)
+          end
+
+          def bin_path
+            @bin_path ||= lookup
+          end
+
+          private
+
+          def lookup
+            [
+              local_bin_path,
+              default_bin_path,
+            ].find(&:exist?) || download
+          end
+
+          def download
+            UI.section("Downloading xccache-proxy binary from remote...".magenta) do
+              Dir.create_tmpdir do |dir|
+                url = "https://github.com/trinhngocthuyen/xccache-proxy/releases/download/#{VERSION}/xccache-proxy.zip"
+                default_bin_path.parent.mkpath
+                tmp_path = dir / File.basename(url)
+                Sh.run("curl -fSL -o #{tmp_path} #{url} && unzip -d #{default_bin_path.parent} #{tmp_path}")
+                FileUtils.chmod("+x", default_bin_path)
+              end
+            end
+            default_bin_path
+          end
+
+          def default_bin_path
+            @default_bin_path ||= LIBEXEC / ".download" / "xccache-proxy-#{VERSION}" / "xccache-proxy"
+          end
+
+          def local_bin_path
+            @local_bin_path ||= LIBEXEC / ".local" / "xccache-proxy"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/xccache/utils/template.rb

@@ -7,7 +7,7 @@ module XCCache
 
     def initialize(name)
       @name = name
-      @path = Gem.find_files("xccache/assets/templates/#{name}.template").first
+      @path = ROOT / "lib/xccache/assets/templates/#{name}.template"
     end
 
     def render(hash = {}, save_to: nil)

data/lib/xccache/xcodeproj/pkg.rb

@@ -17,7 +17,7 @@ module Xcodeproj
         end
 
         def xccache_pkg?
-          local? && id == "xccache/packages/umbrella"
+          local? && ["xccache/packages/umbrella", "xccache/packages/proxy"].include?(id)
         end
 
         def non_xccache_pkg?
@@ -50,9 +50,7 @@ module Xcodeproj
 
         def to_h
           {
-            "relative_path" => relative_path,
-            "path" => path,
-            "path_from_root" => absolute_path.relative_path_from(Pathname(".").expand_path).to_s,
+            "path_from_root" => absolute_path.relative_path_from(Pathname.pwd).to_s,
           }
         end
 

data/lib/xccache/xcodeproj/pkg_product_dependency.rb

@@ -5,13 +5,26 @@ module Xcodeproj
     module Object
       class XCSwiftPackageProductDependency
         def full_name
-          "#{pkg.slug}/#{product_name}"
+          @full_name ||= "#{pkg&.slug || '__unknown__'}/#{product_name}"
         end
 
         def pkg
-          return package unless package.nil?
+          return package if package
+          return if @warned_missing_pkg
+          @warned_missing_pkg = true
+          Log.warn("Missing pkg of product dependency #{uuid}: #{to_hash}")
+        end
 
-          Log.warn("Missing pkg for #{inspect}")
+        def remove_alongside_related
+          target = referrers.find { |x| x.is_a?(PBXNativeTarget) }
+          Log.info(
+            "(-) Remove #{product_name.red} from product dependencies of target #{target.display_name.bold}"
+          )
+          target.dependencies.each { |x| x.remove_from_project if x.product_ref == self }
+          target.build_phases.each do |phase|
+            phase.files.select { |f| f.remove_from_project if f.product_ref == self }
+          end
+          remove_from_project
         end
       end
     end

data/lib/xccache/xcodeproj/project.rb

@@ -29,8 +29,8 @@ module Xcodeproj
     end
 
     def has_pkg?(hash)
-      id = hash[pkg_key_in_hash(hash)]
-      pkgs.any? { |p| p.id == id }
+      pkg_hash = XCCache::Lockfile::Pkg.from_h(hash)
+      pkgs.any? { |p| p.id == pkg_hash.id }
     end
 
     def has_xccache_pkg?
@@ -38,26 +38,26 @@ module Xcodeproj
     end
 
     def add_pkg(hash)
-      key = pkg_key_in_hash(hash)
-      is_local = ["relative_path", "path"].include?(key)
+      pkg_hash = XCCache::Lockfile::Pkg.from_h(hash)
+      pkg_hash["relative_path"] = pkg_hash.relative_path_from_dir(dir).to_s if pkg_hash.key == "path_from_root"
 
-      Log.message("Add package #{hash[key].bold} to project #{display_name.bold}")
-      cls = is_local ? XCLocalSwiftPackageReference : XCRemoteSwiftPackageReference
+      Log.info("Add package #{pkg_hash.id.bold} to project #{display_name.bold}")
+      cls = pkg_hash.local? ? XCLocalSwiftPackageReference : XCRemoteSwiftPackageReference
       ref = new(cls)
       custom_keys = ["path_from_root"]
-      hash.each { |k, v| ref.send("#{k}=", v) unless custom_keys.include?(k) }
+      pkg_hash.each { |k, v| ref.send("#{k}=", v) unless custom_keys.include?(k) }
       root_object.package_references << ref
       ref
     end
 
     def add_xccache_pkg
-      sandbox_path = XCCache::Config.instance.spm_umbrella_sandbox
+      sandbox_path = XCCache::Config.instance.spm_proxy_sandbox
       add_pkg("relative_path" => sandbox_path.relative_path_from(path.parent).to_s)
     end
 
     def remove_pkgs(&block)
       pkgs.select(&block).each do |pkg|
-        XCCache::UI.info("(-) Remove #{pkg.display_name.red} from package refs of project #{display_name.bold}")
+        Log.info("(-) Remove #{pkg.display_name.red} from package refs of project #{display_name.bold}")
         pkg.remove_from_project
       end
     end
@@ -73,11 +73,5 @@ module Xcodeproj
     def xccache_config_group
       self["xccache.config"] || new_group("xccache.config")
     end
-
-    private
-
-    def pkg_key_in_hash(hash)
-      ["repositoryURL", "relative_path", "path"].find { |k| hash.key?(k) }
-    end
   end
 end

data/lib/xccache/xcodeproj/target.rb

@@ -19,32 +19,22 @@ module Xcodeproj
         end
 
         def add_pkg_product_dependency(name)
-          Log.message("(+) Add dependency #{name.blue} to target #{display_name.bold}")
+          Log.info("(+) Add dependency #{name.blue} to target #{display_name.bold}")
           pkg_name, product_name = name.split("/")
           pkg = project.get_pkg(pkg_name)
           pkg_product_dependencies << pkg.create_target_dependency_ref(product_name).product_ref
         end
 
         def add_xccache_product_dependency
-          add_pkg_product_dependency("umbrella/#{name}.xccache")
+          add_pkg_product_dependency("proxy/#{name}.xccache")
         end
 
         def remove_xccache_product_dependencies
-          remove_pkg_product_dependencies { |d| d.pkg.xccache_pkg? }
+          remove_pkg_product_dependencies { |d| d.pkg&.xccache_pkg? }
         end
 
         def remove_pkg_product_dependencies(&block)
-          package_product_dependencies.select(&block).each do |d|
-            XCCache::UI.info(
-              "(-) Remove #{d.product_name.red} from product dependencies of target #{display_name.bold}"
-            )
-            build_phases.each do |phase|
-              phase.files.select { |f| f.remove_from_project if f.product_ref == d }
-            end
-            # Remove it from target dependencies
-            dependencies.each { |x| x.remove_from_project if x.product_ref == d }
-            d.remove_from_project
-          end
+          package_product_dependencies.select(&block).each(&:remove_alongside_related)
         end
       end
     end

data/lib/xccache.rb

@@ -1 +1,6 @@
 require "xccache/main"
+
+module XCCache
+  ROOT = Pathname(__dir__).parent
+  LIBEXEC = ROOT / "libexec"
+end