xccache 0.0.2 → 1.0.0.rc15341775774

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4201459bc9efce46e4eee06f18117749de0003131e383d3726ca082abbc6686f
-  data.tar.gz: c39f19dc60040641aaa518b192449ec86feee2b4e22c056de528904c04437a4a
+  metadata.gz: 9f7b0d46b9a50e667904aedb052da8fa0605932bcd56c518bb308b9802622d74
+  data.tar.gz: 3b98c6610a1f5db2223f93747533a5b8c27f35a163b0e2b6d480a264b7a9ffd0
 SHA512:
-  metadata.gz: 114d1169b34312b251857b5bd93346d7e23ce266c1c05b6df9e9301d472e3995ee3fb8f200a086e3066e2688507a43ad43c3cf584b8505f8c7e2af91d7bc9710
-  data.tar.gz: 74bf01cc687ae222fc6a9cc4b2e75ac67faec16dd503f6ca4420d09824720a87f4afbe5ff26340b2abfa289fc72321d99026c2261a95e9cc22c17f34dc713f63
+  metadata.gz: 00d1a9e5103c69e770c96b78012ae25ee9dbfd6dbacfee4d2d5b2962d52a116663dfae34e3b7263ead13928470883c555cf11e1d0b17d8c332939d1d8938a1b1
+  data.tar.gz: efb6d6d67964fe7edd78a08985682101c691d38837f92a322c5e85a395808a88e248db6cce3c01aadc4b9dc8b2f97a4d6340be129c61ef0bd2a8cfe7558b8171

data/lib/xccache/cache/cachemap.rb

@@ -11,14 +11,6 @@ module XCCache
         raw["cache"] ||= {}
       end
 
-      def manifest_data
-        raw["manifest"] ||= {
-          "targets" => {},
-          "deps" => {},
-          "macros" => {},
-        }
-      end
-
       def missed?(name)
         missed.include?(name)
       end
@@ -36,20 +28,45 @@ module XCCache
       end
 
       def print_stats
-        hit, missed, ignored = %i[hit missed ignore].map { |type| get_cache_data(type) }
+        hit, missed, ignored = %i[hit missed ignored].map { |type| get_cache_data(type) }
         total_count = cache_data.count
         UI.message <<~DESC
           -------------------------------------------------------------------
           Cache stats
           • Hit (#{hit.count}/#{total_count}): #{hit.to_s.green.dark}
           • Missed (#{missed.count}/#{total_count}): #{missed.to_s.yellow.dark}
-          • Ignored (#{ignored.count}/#{total_count}): #{ignored.to_s.yellow.dark}
+          • Ignored (#{ignored.count}/#{total_count}): #{ignored.to_s.dark}
           -------------------------------------------------------------------
         DESC
       end
 
       def get_cache_data(type)
-        cache_data.select { |_, v| v == type }.keys
+        cache_data.select { |k, v| v == type && !k.end_with?(".xccache") }.keys
+      end
+
+      def update_from_graph(graph)
+        cache_data = graph["cache"].to_h do |k, v|
+          next [k, :hit] if v
+          next [k, :ignored] if Config.instance.ignore?(k)
+          [k, :missed]
+        end
+
+        deps = graph["deps"]
+        edges = deps.flat_map { |k, xs| xs.map { |v| { :source => k, :target => v } } }
+        nodes = deps.keys.map do |k|
+          {
+            :id => k,
+            :cache => cache_data[k],
+            :type => ("agg" if k.end_with?(".xccache")),
+            :binary => graph["cache"][k],
+          }
+        end
+        self.raw = {
+          "cache" => cache_data,
+          "depgraph" => { "nodes" => nodes, "edges" => edges },
+        }
+        save
+        print_stats
       end
     end
   end

data/lib/xccache/command/base.rb

@@ -5,9 +5,6 @@ module XCCache
     class Options
       SDK = ["--sdk=foo,bar", "SDKs (iphonesimulator, iphoneos, etc.)"].freeze
       CONFIG = ["--config=foo", "Configuration (debug, release) (default: debug)"].freeze
-      SKIP_RESOLVING_DEPENDENCIES = [
-        "--skip-resolving-dependencies", "Skip resolving package dependencies",
-      ].freeze
       MERGE_SLICES = [
         "--merge-slices/--no-merge-slices",
         "Whether to merge with existing slices/sdks in the xcframework (default: true)",
@@ -18,7 +15,7 @@ module XCCache
       ].freeze
 
       def self.install_options
-        [SDK, CONFIG, SKIP_RESOLVING_DEPENDENCIES]
+        [SDK, CONFIG]
       end
 
       def self.build_options

data/lib/xccache/command/build.rb

@@ -8,7 +8,6 @@ module XCCache
       def self.options
         [
           *Options.build_options,
-          ["--integrate/no-integrate", "Whether to integrate after building target (default: true)"],
           ["--recursive", "Whether to build their recursive targets if cache-missed (default: false)"],
         ].concat(super)
       end
@@ -19,22 +18,10 @@ module XCCache
       def initialize(argv)
         super
         @targets = argv.arguments!
-        @should_integrate = argv.flag?("integrate", true)
       end
 
       def run
-        installer = Installer::Build.new(
-          ctx: self,
-          targets: @targets,
-        )
-        installer.install!
-
-        # Reuse umbrella_pkg from previous installers
-        return unless @should_integrate
-        Installer::Use.new(
-          ctx: self,
-          umbrella_pkg: installer.umbrella_pkg,
-        ).install!
+        Installer::Build.new(ctx: self, targets: @targets).install!
       end
     end
   end

data/lib/xccache/command/off.rb

@@ -0,0 +1,26 @@
+require "xccache/installer"
+require_relative "base"
+
+module XCCache
+  class Command
+    class Off < Command
+      self.summary = "Force-switch to source mode for specific targets"
+      self.arguments = [
+        CLAide::Argument.new("TARGET", false, true),
+      ]
+
+      def initialize(argv)
+        super
+        @targets = argv.arguments!
+      end
+
+      def run
+        return if @targets.empty?
+
+        UI.info("Will force-use source mode for targets: #{@targets}")
+        @targets.each { |t| config.ignore_list << "*/#{t}" }
+        Installer::Use.new(ctx: self).install!
+      end
+    end
+  end
+end

data/lib/xccache/command.rb

@@ -15,12 +15,12 @@ module XCCache
 
     def initialize(argv)
       super
+      set_ansi_mode
       config.verbose = verbose unless verbose.nil?
       config.install_config = argv.option("config", "debug")
       @install_options = {
         :sdks => str_to_sdks(argv.option("sdk")),
         :config => config.install_config,
-        :skip_resolving_dependencies => argv.flag?("skip-resolving-dependencies"),
       }
       @build_options = {
         **@install_options,
@@ -33,5 +33,14 @@ module XCCache
     def str_to_sdks(str)
       (str || config.default_sdk).split(",").map { |s| Swift::Sdk.new(s) }
     end
+
+    private
+
+    def set_ansi_mode
+      config.ansi = ansi_output?
+      return if ansi_output?
+      Colored2.disable!
+      String.send(:define_method, :colorize) { |s, _| s }
+    end
   end
 end

data/lib/xccache/core/config.rb

@@ -13,8 +13,9 @@ module XCCache
       @instance ||= new(Pathname("xccache.yml").expand_path)
     end
 
-    attr_accessor :verbose
+    attr_accessor :verbose, :ansi
     alias verbose? verbose
+    alias ansi? ansi
 
     # To distinguish if it's within an installation, or standalone like `xccache pkg build`
     attr_accessor :in_installation
@@ -35,11 +36,11 @@ module XCCache
     end
 
     def spm_local_pkgs_dir
-      @spm_local_pkgs_dir ||= Dir.prepare(spm_sandbox / "local")
+      @spm_local_pkgs_dir ||= Dir.prepare(spm_sandbox / "local", clean: true)
     end
 
     def spm_xcconfig_dir
-      @spm_xcconfig_dir ||= Dir.prepare(spm_sandbox / "xcconfigs")
+      @spm_xcconfig_dir ||= Dir.prepare(spm_sandbox / "xcconfigs", clean: true)
     end
 
     def spm_cache_dir
@@ -47,7 +48,7 @@ module XCCache
     end
 
     def spm_binaries_dir
-      @spm_binaries_dir ||= Dir.prepare(spm_umbrella_sandbox / "binaries")
+      @spm_binaries_dir ||= Dir.prepare(spm_sandbox / "binaries", clean: true)
     end
 
     def spm_build_dir
@@ -58,6 +59,10 @@ module XCCache
       @spm_artifacts_dir ||= spm_build_dir / "artifacts"
     end
 
+    def spm_proxy_sandbox
+      @spm_proxy_sandbox ||= Dir.prepare(spm_sandbox / "proxy")
+    end
+
     def spm_umbrella_sandbox
       @spm_umbrella_sandbox ||= Dir.prepare(spm_sandbox / "umbrella")
     end
@@ -71,6 +76,7 @@ module XCCache
     end
 
     def cachemap
+      require "xccache/cache/cachemap"
       @cachemap ||= Cache::Cachemap.new(sandbox / "cachemap.json")
     end
 

data/lib/xccache/core/lockfile.rb

@@ -27,6 +27,10 @@ module XCCache
         return id if key == "relative_path"
         (Pathname.pwd / id).relative_path_from(dir) if key == "path_from_root"
       end
+
+      def local_absolute_path
+        Pathname.pwd / self["path_from_root"] if local?
+      end
     end
 
     def hash_for_project(project)

data/lib/xccache/core/sh.rb

@@ -18,27 +18,13 @@ module XCCache
       def run(*args, env: nil, **options)
         cmd = args.join(" ")
         UI.message("$ #{cmd}".cyan.dark) if config.verbose? && options[:log_cmd] != false
+        return system(cmd) || (raise GeneralError, "Command '#{cmd}' failed") unless options[:capture]
 
         out, err = [], []
-        handle_out = proc do |line|
-          if options[:capture]
-            out << line
-          else
-            UI.puts line
-          end
-        end
-        handle_err = proc do |line|
-          if options[:capture]
-            err << line
-          else
-            UI.puts line.strip.yellow unless options[:suppress_err]&.match(line)
-          end
-        end
-
         popen3_args = env ? [env, cmd] : [cmd]
         Open3.popen3(*popen3_args) do |_stdin, stdout, stderr, wait_thr|
-          stdout_thread = Thread.new { stdout.each { |l| handle_out.call(l) } }
-          stderr_thread = Thread.new { stderr.each { |l| handle_err.call(l) } }
+          stdout_thread = Thread.new { stdout.each { |l| out << l } }
+          stderr_thread = Thread.new { stderr.each { |l| err << l } }
           [stdout_thread, stderr_thread].each(&:join)
           result = wait_thr.value
           result.exitstatus

data/lib/xccache/core/syntax/hash.rb

@@ -8,6 +8,10 @@ module XCCache
       @raw = raw || load || {}
     end
 
+    def reload
+      @raw = load || {}
+    end
+
     def load
       raise NotImplementedError
     end

data/lib/xccache/core/system.rb

@@ -14,8 +14,9 @@ class File
 end
 
 class Dir
-  def self.prepare(dir)
+  def self.prepare(dir, clean: false)
     dir = Pathname(dir)
+    dir.rmtree if clean && dir.exist?
     dir.mkpath
     dir
   end

data/lib/xccache/installer/build.rb

@@ -10,12 +10,14 @@ module XCCache
 
       def install!
         perform_install do
-          umbrella_pkg.build(
+          build(
             targets: @targets,
             out_dir: config.spm_cache_dir,
+            symlinks_dir: config.spm_binaries_dir,
             checksum: true,
             **@build_options,
           )
+          proxy_pkg.gen_proxy # Regenerate proxy to apply new cache after build
         end
       end
     end

data/lib/xccache/installer/integration/build.rb

@@ -0,0 +1,28 @@
+module XCCache
+  class Installer
+    module BuildIntegrationMixin
+      def build(options = {})
+        to_build = targets_to_build(options)
+        return UI.warn("Detected no targets to build among cache-missed targets") if to_build.empty?
+
+        UI.info("-> Targets to build: #{to_build.to_s.bold}")
+        umbrella_pkg.build(**options, targets: to_build)
+      end
+
+      def targets_to_build(options)
+        items = options[:targets] || []
+        items = config.cachemap.missed.map { |x| File.basename(x) } if items.empty?
+        targets = items.map { |x| umbrella_pkg.get_target(x) }
+
+        if options[:recursive]
+          UI.message("Will include cache-missed recursive targets")
+          targets += targets.flat_map do |t|
+            t.recursive_targets.select { |x| config.cachemap.missed?(x.full_name) }
+          end
+        end
+        # TODO: Sort by number of dependents
+        targets.map(&:full_name).uniq
+      end
+    end
+  end
+end

data/lib/xccache/installer/integration/descs.rb

@@ -0,0 +1,27 @@
+module XCCache
+  class Installer
+    module DescsIntegrationMixin
+      def xccache_desc
+        @xccache_desc ||= desc_of("xccache")
+      end
+
+      def targets_of_products(products)
+        products = [products] if products.is_a?(String)
+        products.flat_map { |x| desc_of(x).targets_of_products(File.basename(x)) }
+      end
+
+      def dependency_targets_of_products(products)
+        products = [products] if products.is_a?(String)
+        products.flat_map { |p| @dependency_targets_by_products[p] || [p] }.uniq
+      end
+
+      def desc_of(d)
+        descs_by_name[d.split("/").first]
+      end
+
+      def binary_targets
+        descs_by_name.values.flatten.uniq.flat_map(&:binary_targets)
+      end
+    end
+  end
+end

data/lib/xccache/installer/integration/supporting_files.rb

@@ -0,0 +1,29 @@
+module XCCache
+  class Installer
+    module SupportingFilesIntegrationMixin
+      def gen_supporting_files
+        UI.section("Generating supporting files") do
+          gen_xcconfigs
+        end
+      end
+
+      private
+
+      def gen_xcconfigs
+        macros_config_by_targets.each do |target, hash|
+          xcconfig_path = config.spm_xcconfig_dir / "#{target}.xcconfig"
+          UI.message("XCConfig of target #{target} at: #{xcconfig_path}")
+          Xcodeproj::Config.new(hash).save_as(xcconfig_path)
+        end
+      end
+
+      def macros_config_by_targets
+        proxy_pkg.graph["macros"].to_h do |target, paths|
+          swift_flags = paths.map { |p| "-load-plugin-executable #{p}##{File.basename(p, '.*')}" }
+          hash = { "OTHER_SWIFT_FLAGS" => "$(inherited) #{swift_flags.join(' ')}" }
+          [File.basename(target, ".*"), hash]
+        end
+      end
+    end
+  end
+end

data/lib/xccache/installer/integration/viz.rb

@@ -0,0 +1,38 @@
+module XCCache
+  class Installer
+    module VizIntegrationMixin
+      def gen_cachemap_viz
+        stats = config.cachemap.stats
+        html_path = config.sandbox / "cachemap.html"
+        js_path = Dir.prepare(config.sandbox / "assets") / "cachemap.js"
+        css_path = config.sandbox / "assets" / "style.css"
+
+        root_dir = Pathname(".").expand_path
+        to_relative = proc do |p|
+          p.to_s.start_with?(root_dir.to_s) ? p.relative_path_from(root_dir).to_s : p.to_s
+        end
+
+        UI.info("Cachemap visualization: #{html_path}")
+        Template.new("cachemap.html").render(
+          {
+            :root_dir => root_dir.to_s,
+            :root_dir_short => root_dir.basename.to_s,
+            :lockfile_path => config.lockfile.path.to_s,
+            :lockfile_path_short => to_relative.call(config.lockfile.path),
+            :binaries_dir => config.spm_binaries_dir.to_s,
+            :binaries_dir_short => to_relative.call(config.spm_binaries_dir),
+            :desc_hit => stats[:hit],
+            :desc_missed => stats[:missed],
+            :desc_ignored => stats[:ignored],
+          },
+          save_to: html_path
+        )
+        Template.new("cachemap.js").render(
+          { :json => JSON.pretty_generate(config.cachemap.depgraph_data) },
+          save_to: js_path
+        )
+        Template.new("cachemap.style.css").render(save_to: css_path)
+      end
+    end
+  end
+end

data/lib/xccache/installer/integration.rb

@@ -0,0 +1,12 @@
+Dir["#{__dir__}/#{File.basename(__FILE__, '.rb')}/*.rb"].sort.each { |f| require f }
+
+module XCCache
+  class Installer
+    module IntegrationMixin
+      include VizIntegrationMixin
+      include DescsIntegrationMixin
+      include BuildIntegrationMixin
+      include SupportingFilesIntegrationMixin
+    end
+  end
+end

data/lib/xccache/installer.rb

@@ -1,9 +1,11 @@
 require "xccache/spm"
+require "xccache/installer/integration"
 Dir["#{__dir__}/#{File.basename(__FILE__, '.rb')}/*.rb"].sort.each { |f| require f }
 
 module XCCache
   class Installer
     include PkgMixin
+    include IntegrationMixin
 
     def initialize(options = {})
       ctx = options[:ctx]
@@ -15,21 +17,20 @@ module XCCache
 
     def perform_install
       verify_projects!
+      projects.each { |project| migrate_umbrella_to_proxy(project) }
       UI.message("Using cache dir: #{config.spm_cache_dir}")
       config.in_installation = true
-      if @umbrella_pkg.nil?
-        sync_lockfile
-        umbrella_pkg.prepare(**@install_options)
-      end
+      sync_lockfile
+      proxy_pkg.prepare(@install_options)
+
+      yield if block_given?
 
-      yield
-      umbrella_pkg.write_manifest
-      umbrella_pkg.gen_xcconfigs
+      gen_supporting_files
       projects.each do |project|
         add_xccache_refs_to_project(project)
         inject_xcconfig_to_project(project)
       end
-      umbrella_pkg.gen_cachemap_viz
+      gen_cachemap_viz
     end
 
     def sync_lockfile
@@ -79,9 +80,16 @@ module XCCache
       {
         "packages" => project.non_xccache_pkgs.map(&:to_h),
         "dependencies" => deps_by_targets,
+        "platforms" => platforms_for_project(project),
       }
     end
 
+    def platforms_for_project(project)
+      project
+        .targets.map { |t| [t.platform_name.to_s, t.deployment_target] }
+        .sort.reverse.to_h # sort descendingly -> min value is picked for the hash
+    end
+
     def verify_projects!
       raise "No projects detected. Are you running on the correct project directory?" if projects.empty?
     end
@@ -89,7 +97,7 @@ module XCCache
     def add_xccache_refs_to_project(project)
       group = project.xccache_config_group
       add_file = proc { |p| group[p.basename.to_s] || group.new_file(p) }
-      add_file.call(config.spm_umbrella_sandbox / "Package.swift")
+      add_file.call(config.spm_proxy_sandbox / "Package.swift")
       add_file.call(config.lockfile.path)
       add_file.call(config.path) if config.path.exist?
       group.ensure_synced_group(name: "local-packages", path: config.spm_local_pkgs_dir)
@@ -119,5 +127,21 @@ module XCCache
         end
       end
     end
+
+    def migrate_umbrella_to_proxy(project)
+      return unless project.xccache_pkg&.slug == "umbrella"
+
+      UI.info <<~DESC
+        Migrating from umbrella to proxy for project #{project.display_name}
+        You should notice changes in project files from xccache/package/umbrella -> xccache/package/proxy.
+        Don't worry, this is expected.
+      DESC
+        .yellow
+
+      project.xccache_pkg.relative_path = "xccache/packages/proxy"
+      if (group = project.xccache_config_group) && (ref = group["Package.swift"])
+        ref.path = "xccache/packages/proxy/Package.swift"
+      end
+    end
   end
 end

data/lib/xccache/spm/build.rb

@@ -35,7 +35,7 @@ module XCCache
           cmd << "-Xswiftc" << "-emit-module-interface"
           cmd << "-Xswiftc" << "-no-verify-emitted-module-interface"
         end
-        Sh.run(cmd, suppress_err: /(dependency '.*' is not used by any target|unable to create symbolic link)/)
+        Sh.run(cmd)
       end
 
       def swift_build_args

data/lib/xccache/spm/desc/base.rb

@@ -57,10 +57,7 @@ module XCCache
         end
 
         def src_dir
-          @src_dir ||= begin
-            path = root.raw.fetch("packageKind", {}).fetch("root", [])[0]
-            Pathname.new(path) unless path.nil?
-          end
+          @src_dir ||= Pathname(root.raw["path"]).parent
         end
       end
     end

data/lib/xccache/spm/desc/desc.rb

@@ -5,33 +5,9 @@ module XCCache
     class Package
       class Description < BaseObject
         include Cacheable
-        cacheable :resolve_recursive_dependencies
 
-        def self.descs_in_metadata_dir
-          descs = Config.instance.spm_metadata_dir.glob("*.json").map { |p| Description.new(p) }
-          [descs, combine_descs(descs)]
-        end
-
-        def self.in_dir(dir, save_to_dir: nil)
-          path = save_to_dir / "#{dir.basename}.json" unless save_to_dir.nil?
-          begin
-            raw = JSON.parse(Sh.capture_output("swift package dump-package --package-path #{dir}"))
-            Description.new(path, raw: raw)
-          rescue StandardError => e
-            UI.error("Failed to dump package in #{dir}. Error: #{e}")
-          end
-        end
-
-        def self.descs_in_dir(root_dir, save_to_dir: nil)
-          dirs = [root_dir] + root_dir.glob(".build/checkouts/*").reject { |p| p.glob("Package*.swift").empty? }
-          descs = dirs.parallel_map do |dir|
-            desc = Description.in_dir(dir, save_to_dir: save_to_dir)
-            unless save_to_dir.nil?
-              desc.save
-              desc.save(to: desc.path.parent / "#{desc.name}.json") if desc.name != dir.basename.to_s
-            end
-            desc
-          end
+        def self.descs_in_metadata_dir(dir)
+          descs = dir.glob("*.json").map { |p| Description.new(p) }
           [descs, combine_descs(descs)]
         end
 
@@ -77,10 +53,6 @@ module XCCache
             .flat_map { |p| targets.select { |t| p.target_names.include?(t.name) } }
         end
 
-        def resolve_recursive_dependencies(platform: nil)
-          products.to_h { |p| [p, p.recursive_targets(platform: platform)] }
-        end
-
         def local?
           # Workaround: If the pkg dir is under the build checkouts dir -> remote
           !src_dir.to_s.start_with?((config.spm_build_dir / "checkouts").to_s)

data/lib/xccache/spm/mixin.rb

@@ -3,10 +3,11 @@ module XCCache
     include Config::Mixin
 
     def umbrella_pkg
-      @umbrella_pkg ||= SPM::Package::Umbrella.new(
-        root_dir: config.spm_umbrella_sandbox,
-        warn_if_not_direct_target: false,
-      )
+      proxy_pkg.umbrella
+    end
+
+    def proxy_pkg
+      @proxy_pkg ||= SPM::Package::Proxy.new(root_dir: config.spm_proxy_sandbox)
     end
   end
 end