xccache 0.0.1a2 → 0.0.1a3

data/lib/xccache/core/log.rb

@@ -9,11 +9,21 @@ module XCCache
       include Config::Mixin
       attr_accessor :indent
 
-      def section(title)
+      def section(title, timing: false)
+        start = Time.new if timing
         UI.puts(title)
         self.indent += 2
         res = yield if block_given?
         self.indent -= 2
+        if timing
+          duration = (Time.new - start).to_i
+          duration = if duration < 60 then "#{duration}s"
+                     elsif duration < 60 * 60 then "#{duration / 60}m"
+                     else
+                       "#{duration / 3600}h"
+                     end
+          UI.puts("-> Finished: #{title.dark} (#{duration})")
+        end
         res
       end
 

data/lib/xccache/core/parallel.rb

@@ -0,0 +1,10 @@
+require "parallel"
+
+class Array
+  def parallel_map(options = {})
+    # By default, use in_threads (IO-bound tasks)
+    default = {}
+    default[:in_threads] = Parallel.processor_count unless options.key?(:in_processes)
+    Parallel.map(self, { **default, **options }) { |x| yield x if block_given? }
+  end
+end

data/lib/xccache/core/sh.rb

@@ -15,8 +15,8 @@ module XCCache
         run(cmd, capture: true, log_cmd: false)[0].strip
       end
 
-      def run(cmd, options = {})
-        cmd = cmd.join(" ") if cmd.is_a?(Array)
+      def run(*args, env: nil, **options)
+        cmd = args.join(" ")
         UI.message("$ #{cmd}".cyan.dark) if config.verbose? && options[:log_cmd] != false
 
         out, err = [], []
@@ -35,7 +35,8 @@ module XCCache
           end
         end
 
-        Open3.popen3(cmd) do |_stdin, stdout, stderr, wait_thr|
+        popen3_args = env ? [env, cmd] : [cmd]
+        Open3.popen3(*popen3_args) do |_stdin, stdout, stderr, wait_thr|
           stdout_thread = Thread.new { stdout.each { |l| handle_out.call(l) } }
           stderr_thread = Thread.new { stderr.each { |l| handle_err.call(l) } }
           [stdout_thread, stderr_thread].each(&:join)

data/lib/xccache/core/syntax/plist.rb

@@ -0,0 +1,17 @@
+require "cfpropertylist"
+require_relative "hash"
+
+module XCCache
+  class PlistRepresentable < HashRepresentable
+    def load
+      plist = CFPropertyList::List.new(file: path)
+      CFPropertyList.native_types(plist.value)
+    rescue StandardError
+      {}
+    end
+
+    def save(to: nil)
+      raise NotImplementedError
+    end
+  end
+end

data/lib/xccache/core/system.rb

@@ -1,4 +1,5 @@
 require "digest"
+require "mkmf"
 
 class String
   def c99extidentifier
@@ -6,6 +7,12 @@ class String
   end
 end
 
+class File
+  def self.which(bin)
+    find_executable0(bin)
+  end
+end
+
 class Dir
   def self.prepare(dir)
     dir = Pathname(dir)
@@ -36,6 +43,7 @@ class Pathname
   def copy(to: nil, to_dir: nil)
     dst = to || (Pathname(to_dir) / basename)
     dst.rmtree if dst.exist? || dst.symlink?
+    dst.parent.mkpath
     FileUtils.copy_entry(self, dst)
     dst
   end

data/lib/xccache/installer/build.rb

@@ -6,19 +6,15 @@ module XCCache
       def initialize(options = {})
         super
         @targets = options[:targets]
-        @sdk = options[:sdk]
-        @recursive = options[:recursive]
       end
 
       def install!
         perform_install do
           umbrella_pkg.build(
             targets: @targets,
-            sdk: @sdk,
-            out_dir: config.spm_binaries_frameworks_dir,
+            out_dir: config.spm_repo_dir,
             checksum: true,
-            recursive: @recursive,
-            skip_resolve: @skip_resolving_dependencies,
+            **@build_options,
           )
         end
       end

data/lib/xccache/installer/rollback.rb

@@ -30,6 +30,7 @@ module XCCache
 
         # Remove .binary product from the project
         project.targets.each(&:remove_xccache_product_dependencies)
+        project.xccache_pkg&.remove_from_project
       end
     end
   end

data/lib/xccache/installer/use.rb

@@ -4,8 +4,8 @@ module XCCache
   class Installer
     class Use < Installer
       def install!
-        perform_install do
-          update_projects do |project|
+        update_projects do |project|
+          perform_install do
             UI.section("Using cache for project #{project.display_name}".bold.green) do
               replace_binaries_for_project(project)
             end
@@ -21,6 +21,7 @@ module XCCache
           target.add_xccache_product_dependency unless target.has_xccache_product_dependency?
           target.remove_pkg_product_dependencies { |d| !d.pkg.xccache_pkg? }
         end
+        project.remove_pkgs(&:non_xccache_pkg?) unless config.keep_pkgs_in_project?
       end
     end
   end

data/lib/xccache/installer.rb

@@ -6,16 +6,28 @@ module XCCache
     include PkgMixin
 
     def initialize(options = {})
+      ctx = options[:ctx]
+      raise GeneralError, "Missing context (Command) for #{self.class}" if ctx.nil?
       @umbrella_pkg = options[:umbrella_pkg]
-      @skip_resolving_dependencies = options[:skip_resolving_dependencies]
+      @install_options = ctx.install_options
+      @build_options = ctx.build_options
     end
 
     def perform_install
+      config.in_installation = true
       verify_projects!
-      sync_lockfile if @umbrella_pkg.nil?
-      umbrella_pkg.prepare(skip_resolve: @skip_resolving_dependencies) if @umbrella_pkg.nil?
+      if @umbrella_pkg.nil?
+        sync_lockfile
+        umbrella_pkg.prepare(**@install_options)
+      end
+
       yield
       umbrella_pkg.write_manifest
+      umbrella_pkg.gen_xcconfigs
+      projects.each do |project|
+        add_xccache_refs_to_project(project)
+        inject_xcconfig_to_project(project)
+      end
       umbrella_pkg.gen_cachemap_viz
     end
 
@@ -51,7 +63,7 @@ module XCCache
 
     def lockfile_hash_for_project(project)
       deps_by_targets = project.targets.to_h do |target|
-        deps = target.non_xccache_pkg_product_dependencies.map { |d| "#{d.pkg.slug}/#{d.product_name}" }
+        deps = target.non_xccache_pkg_product_dependencies.select(&:pkg).map { |d| "#{d.pkg.slug}/#{d.product_name}" }
         [target.name, deps]
       end
       {
@@ -63,5 +75,39 @@ module XCCache
     def verify_projects!
       raise "No projects detected. Are you running on the correct project directory?" if projects.empty?
     end
+
+    def add_xccache_refs_to_project(project)
+      group = project.xccache_config_group
+      add_file = proc { |p| group[p.basename.to_s] || group.new_file(p) }
+      add_file.call(config.spm_umbrella_sandbox / "Package.swift")
+      add_file.call(config.lockfile.path)
+      add_file.call(config.path) if config.path.exist?
+      group.ensure_synced_group(name: "local-packages", path: config.spm_local_pkgs_dir)
+    end
+
+    def inject_xcconfig_to_project(project)
+      group = project.xccache_config_group.ensure_synced_group(name: "xcconfigs", path: config.spm_xcconfig_dir)
+      project.targets.each do |target|
+        xcconfig_path = config.spm_xcconfig_dir / "#{target.name}.xcconfig"
+        target.build_configurations.each do |build_config|
+          if (existing = build_config.base_configuration_xcconfig)
+            next if existing.path == xcconfig_path
+
+            relative_path = xcconfig_path.relative_path_from(existing.path.parent)
+            next if existing.includes.include?(relative_path.to_s)
+
+            UI.info("Injecting base configuration for #{target} (#{build_config}) (at: #{existing.path})")
+            existing.path.write <<~DESC
+              #include "#{relative_path}" // Injected by xccache, for prebuilt macros support
+              #{existing.path.read.strip}
+            DESC
+          else
+            UI.info("Setting base configuration #{target} (#{build_config}) as #{xcconfig_path}")
+            build_config.base_configuration_reference_anchor = group
+            build_config.base_configuration_reference_relative_path = xcconfig_path.basename.to_s
+          end
+        end
+      end
+    end
   end
 end

data/lib/xccache/spm/build.rb

@@ -0,0 +1,53 @@
+module XCCache
+  module SPM
+    class Buildable
+      attr_reader :name, :module_name, :pkg_dir, :pkg_desc, :sdk, :sdks, :config, :path, :tmpdir, :library_evolution
+      alias library_evolution? library_evolution
+
+      def initialize(options = {})
+        @name = options[:name]
+        @module_name = @name.c99extidentifier
+        @pkg_dir = Pathname(options[:pkg_dir] || ".").expand_path
+        @pkg_desc = options[:pkg_desc]
+        @sdks = options[:sdks] || []
+        @sdk = options[:sdk] || @sdks&.first
+        @config = options[:config] || "debug"
+        @path = options[:path]
+        @tmpdir = options[:tmpdir]
+        @library_evolution = options[:library_evolution]
+      end
+
+      def build(options = {})
+        raise NotImplementedError
+      end
+
+      def swift_build(target: nil)
+        cmd = ["swift", "build"] + swift_build_args
+        cmd << "--package-path" << pkg_dir
+        cmd << "--target" << (target || name)
+        cmd << "--sdk" << sdk.sdk_path
+        sdk.swiftc_args.each { |arg| cmd << "-Xswiftc" << arg }
+        if library_evolution?
+          # Workaround for swiftinterface emission
+          # https://github.com/swiftlang/swift/issues/64669#issuecomment-1535335601
+          cmd << "-Xswiftc" << "-enable-library-evolution"
+          cmd << "-Xswiftc" << "-alias-module-names-in-module-interface"
+          cmd << "-Xswiftc" << "-emit-module-interface"
+          cmd << "-Xswiftc" << "-no-verify-emitted-module-interface"
+        end
+        Sh.run(cmd, suppress_err: /(dependency '.*' is not used by any target|unable to create symbolic link)/)
+      end
+
+      def swift_build_args
+        [
+          "--configuration", config,
+          "--triple", sdk.triple,
+        ]
+      end
+
+      def pkg_target
+        @pkg_target ||= pkg_desc.get_target(name)
+      end
+    end
+  end
+end

data/lib/xccache/spm/desc/base.rb

@@ -6,7 +6,8 @@ module XCCache
       class BaseObject < JSONRepresentable
         include Config::Mixin
 
-        attr_accessor :root, :retrieve_pkg_desc
+        ATTRS = %i[root retrieve_pkg_desc].freeze
+        attr_accessor(*ATTRS)
 
         def name
           raw["name"]
@@ -28,12 +29,18 @@ module XCCache
           "<#{self.class} name=#{display_name}>"
         end
 
+        def cast_to(cls)
+          o = cls.new(path, raw: raw)
+          ATTRS.each { |sym| o.send("#{sym}=", send(sym.to_s)) }
+          o
+        end
+
         def pkg_name
           @pkg_name ||= root.name
         end
 
         def pkg_slug
-          @pkg_slug ||= root.path.basename(".json").to_s
+          @pkg_slug ||= src_dir.basename.to_s
         end
 
         def fetch(key, dtype)
@@ -51,7 +58,7 @@ module XCCache
 
         def src_dir
           @src_dir ||= begin
-            path = raw.fetch("packageKind", {}).fetch("root", [])[0]
+            path = root.raw.fetch("packageKind", {}).fetch("root", [])[0]
             Pathname.new(path) unless path.nil?
           end
         end

data/lib/xccache/spm/desc/desc.rb

@@ -7,18 +7,34 @@ module XCCache
         include Cacheable
         cacheable :resolve_recursive_dependencies
 
-        def self.in_dir(dir, save_to_dir: nil, checksum: true)
+        def self.descs_in_metadata_dir
+          descs = Config.instance.spm_metadata_dir.glob("*.json").map { |p| Description.new(p) }
+          [descs, combine_descs(descs)]
+        end
+
+        def self.in_dir(dir, save_to_dir: nil)
           path = save_to_dir / "#{dir.basename}.json" unless save_to_dir.nil?
           begin
             raw = JSON.parse(Sh.capture_output("swift package dump-package --package-path #{dir}"))
-            this = Description.new(path, raw: raw)
-            this.calc_checksum if checksum
-            this
+            Description.new(path, raw: raw)
           rescue StandardError => e
             UI.error("Failed to dump package in #{dir}. Error: #{e}")
           end
         end
 
+        def self.descs_in_dir(root_dir, save_to_dir: nil)
+          dirs = [root_dir] + root_dir.glob(".build/checkouts/*").reject { |p| p.glob("Package*.swift").empty? }
+          descs = dirs.parallel_map do |dir|
+            desc = Description.in_dir(dir, save_to_dir: save_to_dir)
+            unless save_to_dir.nil?
+              desc.save
+              desc.save(to: desc.path.parent / "#{desc.name}.json") if desc.name != dir.basename.to_s
+            end
+            desc
+          end
+          [descs, combine_descs(descs)]
+        end
+
         def root
           self
         end
@@ -27,10 +43,6 @@ module XCCache
           raw["_metadata"] ||= {}
         end
 
-        def checksum
-          metadata["checksum"]
-        end
-
         def dependencies
           @dependencies ||= fetch("dependencies", Dependency)
         end
@@ -44,7 +56,7 @@ module XCCache
         end
 
         def targets
-          @targets ||= fetch("targets", Target)
+          @targets ||= fetch("targets", Target).map(&:downcast)
         end
 
         def binary_targets
@@ -74,10 +86,6 @@ module XCCache
           !src_dir.to_s.start_with?((config.spm_build_dir / "checkouts").to_s)
         end
 
-        def calc_checksum
-          metadata["checksum"] = git.nil? ? src_dir.checksum : git.sha
-        end
-
         def traverse
           nodes, edges, parents = [], [], {}
           to_visit = targets.dup
@@ -89,6 +97,10 @@ module XCCache
             visited << cur
             nodes << cur
             yield cur if block_given?
+
+            # For macro impl, we don't need their dependencies, just the tool binary
+            # So, no need to care about swift-syntax dependencies
+            next if cur.macro?
             cur.direct_dependency_targets.each do |t|
               to_visit << t
               edges << [cur, t]
@@ -99,11 +111,15 @@ module XCCache
           [nodes, edges, parents]
         end
 
-        private
-
         def git
           @git ||= Git.new(src_dir) if Dir.git?(src_dir)
         end
+
+        def self.combine_descs(descs)
+          descs_by_name = descs.flat_map { |d| [[d.name, d], [d.pkg_slug, d]] }.to_h
+          descs.each { |d| d.retrieve_pkg_desc = proc { |name| descs_by_name[name] } }
+          descs_by_name
+        end
       end
     end
   end

data/lib/xccache/spm/desc/target/binary.rb

@@ -0,0 +1,8 @@
+module XCCache
+  module SPM
+    class Package
+      class BinaryTarget < Target
+      end
+    end
+  end
+end

data/lib/xccache/spm/desc/target/macro.rb

@@ -0,0 +1,8 @@
+module XCCache
+  module SPM
+    class Package
+      class MacroTarget < Target
+      end
+    end
+  end
+end

data/lib/xccache/spm/desc/target.rb

@@ -7,15 +7,33 @@ module XCCache
         include Cacheable
         cacheable :recursive_targets, :direct_dependency_targets, :direct_dependencies
 
+        Dir["#{__dir__}/#{File.basename(__FILE__, '.rb')}/*.rb"].sort.each { |f| require f }
+
         def xccache?
           name.end_with?(".xccache")
         end
 
+        def xccache_id
+          macro? ? "#{full_name}.macro" : full_name
+        end
+
         def type
           @type ||= raw["type"].to_sym
         end
 
-        def bundle_name
+        def downcast
+          cls = {
+            :binary => BinaryTarget,
+            :macro => MacroTarget,
+          }[type]
+          cls.nil? ? self : cast_to(cls)
+        end
+
+        def module_name
+          name.c99extidentifier
+        end
+
+        def resource_bundle_name
           "#{pkg_name}_#{name}.bundle"
         end
 
@@ -78,7 +96,7 @@ module XCCache
 
         def recursive_targets(platform: nil)
           children = direct_dependency_targets(platform: platform)
-          children += children.flat_map { |t| t.recursive_targets(platform: platform) }
+          children += children.flat_map { |t| t.macro? ? [t] : t.recursive_targets(platform: platform) }
           children.uniq
         end
 
@@ -102,6 +120,10 @@ module XCCache
           true # FIXME: Handle this
         end
 
+        def macro?
+          type == :macro
+        end
+
         def binary?
           type == :binary
         end
@@ -114,6 +136,10 @@ module XCCache
           binary_path if binary? && root.local?
         end
 
+        def checksum
+          @checksum ||= root.git&.sha || sources_path.checksum
+        end
+
         private
 
         def find_deps(name, pkg_name, dep_type)

data/lib/xccache/spm/macro.rb

@@ -0,0 +1,44 @@
+require_relative "build"
+
+module XCCache
+  module SPM
+    class Macro < Buildable
+      attr_reader :macosx_sdk
+
+      def initialize(options = {})
+        super
+        @library_evolution = false # swift-syntax is not compatible with library evolution
+        @macosx_sdk = Swift::Sdk.new(:macosx)
+      end
+
+      def build(_options = {})
+        # NOTE: Building macro binary is tricky...
+        # --------------------------------------------------------------------------------
+        # Consider this manifest config: .target(Macro) -> .macro(MacroImpl)
+        #   where `.target(Macro)` contains the interfaces
+        #   and `.target(MacroImpl)` contains the implementation
+        # --------------------------------------------------------------------------------
+        # Building `.macro(MacroImpl)` does not produce the tool binary (MacroImpl-tool)... Only `.o` files.
+        # Yet, linking those files are exhaustive due to many dependencies in swift-syntax
+        # Luckily, building `.target(Macro)` does produce the tool binary.
+        # -> WORKAROUND: Find the associated regular target and build it, then collect the tool binary
+        # ---------------------------------------------------------------------------------
+        associated_target = pkg_desc.targets.find { |t| t.direct_dependency_targets.include?(pkg_target) }
+        UI.message(
+          "#{name.yellow.dark} is a macro target. " \
+          "Will build the associated target #{associated_target.name.dark} to get the tool binary."
+        )
+        swift_build(target: associated_target.name)
+        binary_path = products_dir / "#{module_name}-tool"
+        raise GeneralError, "Tool binary not exist at: #{binary_path}" unless binary_path.exist?
+        binary_path.copy(to: path)
+        FileUtils.chmod("+x", path)
+        UI.info("-> Macro binary: #{path.to_s.dark}")
+      end
+
+      def products_dir
+        @products_dir ||= pkg_dir / ".build" / macosx_sdk.triple / config
+      end
+    end
+  end
+end

data/lib/xccache/spm/mixin.rb

@@ -3,7 +3,10 @@ module XCCache
     include Config::Mixin
 
     def umbrella_pkg
-      @umbrella_pkg ||= SPM::Package::Umbrella.new(root_dir: config.spm_umbrella_sandbox)
+      @umbrella_pkg ||= SPM::Package::Umbrella.new(
+        root_dir: config.spm_umbrella_sandbox,
+        warn_if_not_direct_target: false,
+      )
     end
   end
 end