xccache 0.0.1 → 1.0.0.rc15341775774

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b10dd2ba3e0a83b400776d8dde8116be6632a9806714ee85526f3788f6364cab
-  data.tar.gz: 789518eadb39c223a5044788122897e7b38c62d0e487733736602b22cb17ff8b
+  metadata.gz: 9f7b0d46b9a50e667904aedb052da8fa0605932bcd56c518bb308b9802622d74
+  data.tar.gz: 3b98c6610a1f5db2223f93747533a5b8c27f35a163b0e2b6d480a264b7a9ffd0
 SHA512:
-  metadata.gz: 349bbbf1f5101dfc4e4555ff8e9546fa870660ad2f600ea57b28fdbb7a8f7d6007d9fd7aab37937c61659f181fc85d19358c7bc00b3408118d0f4a8e8e6c989e
-  data.tar.gz: f77646c655bdfbcb413a5112f1935545dc2b6c042f92122c0181b9f4b9761874dcc75cad783d645cad5f332eeb49f71181aa9766b84d9f498486ccd7e50e6a3d
+  metadata.gz: 00d1a9e5103c69e770c96b78012ae25ee9dbfd6dbacfee4d2d5b2962d52a116663dfae34e3b7263ead13928470883c555cf11e1d0b17d8c332939d1d8938a1b1
+  data.tar.gz: efb6d6d67964fe7edd78a08985682101c691d38837f92a322c5e85a395808a88e248db6cce3c01aadc4b9dc8b2f97a4d6340be129c61ef0bd2a8cfe7558b8171

data/lib/xccache/cache/cachemap.rb

@@ -11,14 +11,6 @@ module XCCache
         raw["cache"] ||= {}
       end
 
-      def manifest_data
-        raw["manifest"] ||= {
-          "targets" => {},
-          "deps" => {},
-          "macros" => {},
-        }
-      end
-
       def missed?(name)
         missed.include?(name)
       end
@@ -36,20 +28,45 @@ module XCCache
       end
 
       def print_stats
-        hit, missed, ignored = %i[hit missed ignore].map { |type| get_cache_data(type) }
+        hit, missed, ignored = %i[hit missed ignored].map { |type| get_cache_data(type) }
         total_count = cache_data.count
         UI.message <<~DESC
           -------------------------------------------------------------------
           Cache stats
           • Hit (#{hit.count}/#{total_count}): #{hit.to_s.green.dark}
           • Missed (#{missed.count}/#{total_count}): #{missed.to_s.yellow.dark}
-          • Ignored (#{ignored.count}/#{total_count}): #{ignored.to_s.yellow.dark}
+          • Ignored (#{ignored.count}/#{total_count}): #{ignored.to_s.dark}
           -------------------------------------------------------------------
         DESC
       end
 
       def get_cache_data(type)
-        cache_data.select { |_, v| v == type }.keys
+        cache_data.select { |k, v| v == type && !k.end_with?(".xccache") }.keys
+      end
+
+      def update_from_graph(graph)
+        cache_data = graph["cache"].to_h do |k, v|
+          next [k, :hit] if v
+          next [k, :ignored] if Config.instance.ignore?(k)
+          [k, :missed]
+        end
+
+        deps = graph["deps"]
+        edges = deps.flat_map { |k, xs| xs.map { |v| { :source => k, :target => v } } }
+        nodes = deps.keys.map do |k|
+          {
+            :id => k,
+            :cache => cache_data[k],
+            :type => ("agg" if k.end_with?(".xccache")),
+            :binary => graph["cache"][k],
+          }
+        end
+        self.raw = {
+          "cache" => cache_data,
+          "depgraph" => { "nodes" => nodes, "edges" => edges },
+        }
+        save
+        print_stats
       end
     end
   end

data/lib/xccache/command/base.rb

@@ -5,9 +5,6 @@ module XCCache
     class Options
       SDK = ["--sdk=foo,bar", "SDKs (iphonesimulator, iphoneos, etc.)"].freeze
       CONFIG = ["--config=foo", "Configuration (debug, release) (default: debug)"].freeze
-      SKIP_RESOLVING_DEPENDENCIES = [
-        "--skip-resolving-dependencies", "Skip resolving package dependencies",
-      ].freeze
       MERGE_SLICES = [
         "--merge-slices/--no-merge-slices",
         "Whether to merge with existing slices/sdks in the xcframework (default: true)",
@@ -18,7 +15,7 @@ module XCCache
       ].freeze
 
       def self.install_options
-        [SDK, CONFIG, SKIP_RESOLVING_DEPENDENCIES]
+        [SDK, CONFIG]
       end
 
       def self.build_options

data/lib/xccache/command/build.rb

@@ -8,7 +8,6 @@ module XCCache
       def self.options
         [
           *Options.build_options,
-          ["--integrate/no-integrate", "Whether to integrate after building target (default: true)"],
           ["--recursive", "Whether to build their recursive targets if cache-missed (default: false)"],
         ].concat(super)
       end
@@ -19,22 +18,10 @@ module XCCache
       def initialize(argv)
         super
         @targets = argv.arguments!
-        @should_integrate = argv.flag?("integrate", true)
       end
 
       def run
-        installer = Installer::Build.new(
-          ctx: self,
-          targets: @targets,
-        )
-        installer.install!
-
-        # Reuse umbrella_pkg from previous installers
-        return unless @should_integrate
-        Installer::Use.new(
-          ctx: self,
-          umbrella_pkg: installer.umbrella_pkg,
-        ).install!
+        Installer::Build.new(ctx: self, targets: @targets).install!
       end
     end
   end

data/lib/xccache/command/off.rb

@@ -0,0 +1,26 @@
+require "xccache/installer"
+require_relative "base"
+
+module XCCache
+  class Command
+    class Off < Command
+      self.summary = "Force-switch to source mode for specific targets"
+      self.arguments = [
+        CLAide::Argument.new("TARGET", false, true),
+      ]
+
+      def initialize(argv)
+        super
+        @targets = argv.arguments!
+      end
+
+      def run
+        return if @targets.empty?
+
+        UI.info("Will force-use source mode for targets: #{@targets}")
+        @targets.each { |t| config.ignore_list << "*/#{t}" }
+        Installer::Use.new(ctx: self).install!
+      end
+    end
+  end
+end

data/lib/xccache/command.rb

@@ -15,12 +15,12 @@ module XCCache
 
     def initialize(argv)
       super
+      set_ansi_mode
       config.verbose = verbose unless verbose.nil?
       config.install_config = argv.option("config", "debug")
       @install_options = {
         :sdks => str_to_sdks(argv.option("sdk")),
         :config => config.install_config,
-        :skip_resolving_dependencies => argv.flag?("skip-resolving-dependencies"),
       }
       @build_options = {
         **@install_options,
@@ -33,5 +33,14 @@ module XCCache
     def str_to_sdks(str)
       (str || config.default_sdk).split(",").map { |s| Swift::Sdk.new(s) }
     end
+
+    private
+
+    def set_ansi_mode
+      config.ansi = ansi_output?
+      return if ansi_output?
+      Colored2.disable!
+      String.send(:define_method, :colorize) { |s, _| s }
+    end
   end
 end

data/lib/xccache/core/config.rb

@@ -13,8 +13,9 @@ module XCCache
       @instance ||= new(Pathname("xccache.yml").expand_path)
     end
 
-    attr_accessor :verbose
+    attr_accessor :verbose, :ansi
     alias verbose? verbose
+    alias ansi? ansi
 
     # To distinguish if it's within an installation, or standalone like `xccache pkg build`
     attr_accessor :in_installation
@@ -35,11 +36,11 @@ module XCCache
     end
 
     def spm_local_pkgs_dir
-      @spm_local_pkgs_dir ||= Dir.prepare(spm_sandbox / "local")
+      @spm_local_pkgs_dir ||= Dir.prepare(spm_sandbox / "local", clean: true)
     end
 
     def spm_xcconfig_dir
-      @spm_xcconfig_dir ||= Dir.prepare(spm_sandbox / "xcconfigs")
+      @spm_xcconfig_dir ||= Dir.prepare(spm_sandbox / "xcconfigs", clean: true)
     end
 
     def spm_cache_dir
@@ -47,7 +48,7 @@ module XCCache
     end
 
     def spm_binaries_dir
-      @spm_binaries_dir ||= Dir.prepare(spm_umbrella_sandbox / "binaries")
+      @spm_binaries_dir ||= Dir.prepare(spm_sandbox / "binaries", clean: true)
     end
 
     def spm_build_dir
@@ -58,6 +59,10 @@ module XCCache
       @spm_artifacts_dir ||= spm_build_dir / "artifacts"
     end
 
+    def spm_proxy_sandbox
+      @spm_proxy_sandbox ||= Dir.prepare(spm_sandbox / "proxy")
+    end
+
     def spm_umbrella_sandbox
       @spm_umbrella_sandbox ||= Dir.prepare(spm_sandbox / "umbrella")
     end
@@ -71,6 +76,7 @@ module XCCache
     end
 
     def cachemap
+      require "xccache/cache/cachemap"
       @cachemap ||= Cache::Cachemap.new(sandbox / "cachemap.json")
     end
 

data/lib/xccache/core/hash.rb

@@ -1,12 +1,12 @@
 class Hash
-  def deep_merge(other, uniq_block: nil, &block)
-    dup.deep_merge!(other, uniq_block: uniq_block, &block)
+  def deep_merge(other, uniq_block: nil, sort_block: nil, &block)
+    dup.deep_merge!(other, uniq_block: uniq_block, sort_block: sort_block, &block)
   end
 
-  def deep_merge!(other, uniq_block: nil, &block)
+  def deep_merge!(other, uniq_block: nil, sort_block: nil, &block)
     merge!(other) do |key, this_val, other_val|
       result = if this_val.is_a?(Hash) && other_val.is_a?(Hash)
-                 this_val.deep_merge(other_val, uniq_block: uniq_block, &block)
+                 this_val.deep_merge(other_val, uniq_block: uniq_block, sort_block: sort_block, &block)
                elsif this_val.is_a?(Array) && other_val.is_a?(Array)
                  this_val + other_val
                elsif block_given?
@@ -14,8 +14,11 @@ class Hash
                else
                  other_val
                end
-      next result if uniq_block.nil? || !result.is_a?(Array)
-      result.reverse.uniq(&uniq_block).reverse # prefer updates
+
+      # uniq by block, prefer updates
+      result = result.reverse.uniq(&uniq_block).reverse if uniq_block && result.is_a?(Array)
+      result = result.sort_by(&sort_block) if sort_block && result.is_a?(Array)
+      result
     end
   end
 end

data/lib/xccache/core/lockfile.rb

@@ -2,6 +2,37 @@ require "xccache/core/syntax/json"
 
 module XCCache
   class Lockfile < JSONRepresentable
+    class Pkg < Hash
+      def self.from_h(h)
+        Pkg.new.merge(h)
+      end
+
+      def key
+        @key ||= ["repositoryURL", "path_from_root", "relative_path"].find { |x| key?(x) }
+      end
+
+      def id
+        self[key]
+      end
+
+      def local?
+        key != "repositoryURL"
+      end
+
+      def slug
+        @slug ||= File.basename(id, ".*")
+      end
+
+      def relative_path_from_dir(dir)
+        return id if key == "relative_path"
+        (Pathname.pwd / id).relative_path_from(dir) if key == "path_from_root"
+      end
+
+      def local_absolute_path
+        Pathname.pwd / self["path_from_root"] if local?
+      end
+    end
+
     def hash_for_project(project)
       raw[project.display_name] ||= {}
     end
@@ -11,22 +42,31 @@ module XCCache
     end
 
     def deep_merge!(hash)
-      keys = ["repositoryURL", "path_from_root", "relative_path"]
-      raw.deep_merge!(hash, uniq_block: proc do |h|
-        h.is_a?(Hash) && (k = keys.find { |k| h.key?(k) }) ? h[k] : h
-      end)
+      raw.deep_merge!(
+        hash,
+        uniq_block: proc { |h| h.is_a?(Hash) ? Pkg.from_h(h).id || h : h },
+        sort_block: proc { |x| x.to_s.downcase },
+      )
+      # After deep_merge, clear property cache
+      (instance_variables - %i[@path @raw]).each do |ivar|
+        remove_instance_variable(ivar)
+      end
     end
 
     def pkgs
-      @pkgs ||= raw.values.flat_map { |h| h["packages"] || [] }
+      @pkgs ||= raw.values.flat_map { |h| h["packages"] || [] }.map { |h| Pkg.from_h(h) }
     end
 
     def local_pkgs
-      @local_pkgs ||= pkgs.select { |h| h.key?("relative_path") || h.key?("path") }
+      @local_pkgs ||= pkgs.select(&:local?).uniq
     end
 
     def local_pkg_slugs
-      @local_pkg_slugs ||= local_pkgs.map { |h| File.basename(h["relative_path"] || h["path"]) }.uniq
+      @local_pkg_slugs ||= local_pkgs.map(&:slug).uniq
+    end
+
+    def known_product_dependencies
+      raw.empty? ? [] : product_dependencies.reject { |d| File.dirname(d) == "__unknown__" }
     end
 
     def product_dependencies
@@ -36,5 +76,20 @@ module XCCache
     def targets_data
       @targets_data ||= product_dependencies_by_targets.transform_keys { |k| "#{k}.xccache" }
     end
+
+    def verify!
+      known_slugs = pkgs.map(&:slug)
+      unknown = product_dependencies.reject { |d| known_slugs.include?(File.dirname(d)) }
+      return if unknown.empty?
+
+      UI.error! <<~DESC
+        Unknown product dependencies at #{path}:
+
+        #{unknown.sort.map { |d| "  • #{d}" }.join("\n")}
+
+        Refer to this doc for how to resolve this issue:
+          https://github.com/trinhngocthuyen/xccache/blob/main/docs/troubleshooting.md#unknown-product-dependencies
+      DESC
+    end
   end
 end

data/lib/xccache/core/log.rb

@@ -43,6 +43,11 @@ module XCCache
         UI.puts("[ERROR] #{message}".red)
       end
 
+      def error!(message)
+        error(message)
+        raise GeneralError, message
+      end
+
       def puts(message)
         $stdout.puts("#{' ' * self.indent}#{message}")
       end

data/lib/xccache/core/sh.rb

@@ -18,27 +18,13 @@ module XCCache
       def run(*args, env: nil, **options)
         cmd = args.join(" ")
         UI.message("$ #{cmd}".cyan.dark) if config.verbose? && options[:log_cmd] != false
+        return system(cmd) || (raise GeneralError, "Command '#{cmd}' failed") unless options[:capture]
 
         out, err = [], []
-        handle_out = proc do |line|
-          if options[:capture]
-            out << line
-          else
-            UI.puts line
-          end
-        end
-        handle_err = proc do |line|
-          if options[:capture]
-            err << line
-          else
-            UI.puts line.strip.yellow unless options[:suppress_err]&.match(line)
-          end
-        end
-
         popen3_args = env ? [env, cmd] : [cmd]
         Open3.popen3(*popen3_args) do |_stdin, stdout, stderr, wait_thr|
-          stdout_thread = Thread.new { stdout.each { |l| handle_out.call(l) } }
-          stderr_thread = Thread.new { stderr.each { |l| handle_err.call(l) } }
+          stdout_thread = Thread.new { stdout.each { |l| out << l } }
+          stderr_thread = Thread.new { stderr.each { |l| err << l } }
           [stdout_thread, stderr_thread].each(&:join)
           result = wait_thr.value
           result.exitstatus

data/lib/xccache/core/syntax/hash.rb

@@ -8,6 +8,10 @@ module XCCache
       @raw = raw || load || {}
     end
 
+    def reload
+      @raw = load || {}
+    end
+
     def load
       raise NotImplementedError
     end

data/lib/xccache/core/system.rb

@@ -14,8 +14,9 @@ class File
 end
 
 class Dir
-  def self.prepare(dir)
+  def self.prepare(dir, clean: false)
     dir = Pathname(dir)
+    dir.rmtree if clean && dir.exist?
     dir.mkpath
     dir
   end

data/lib/xccache/installer/build.rb

@@ -10,12 +10,14 @@ module XCCache
 
       def install!
         perform_install do
-          umbrella_pkg.build(
+          build(
             targets: @targets,
             out_dir: config.spm_cache_dir,
+            symlinks_dir: config.spm_binaries_dir,
             checksum: true,
             **@build_options,
           )
+          proxy_pkg.gen_proxy # Regenerate proxy to apply new cache after build
         end
       end
     end

data/lib/xccache/installer/integration/build.rb

@@ -0,0 +1,28 @@
+module XCCache
+  class Installer
+    module BuildIntegrationMixin
+      def build(options = {})
+        to_build = targets_to_build(options)
+        return UI.warn("Detected no targets to build among cache-missed targets") if to_build.empty?
+
+        UI.info("-> Targets to build: #{to_build.to_s.bold}")
+        umbrella_pkg.build(**options, targets: to_build)
+      end
+
+      def targets_to_build(options)
+        items = options[:targets] || []
+        items = config.cachemap.missed.map { |x| File.basename(x) } if items.empty?
+        targets = items.map { |x| umbrella_pkg.get_target(x) }
+
+        if options[:recursive]
+          UI.message("Will include cache-missed recursive targets")
+          targets += targets.flat_map do |t|
+            t.recursive_targets.select { |x| config.cachemap.missed?(x.full_name) }
+          end
+        end
+        # TODO: Sort by number of dependents
+        targets.map(&:full_name).uniq
+      end
+    end
+  end
+end

data/lib/xccache/installer/integration/descs.rb

@@ -0,0 +1,27 @@
+module XCCache
+  class Installer
+    module DescsIntegrationMixin
+      def xccache_desc
+        @xccache_desc ||= desc_of("xccache")
+      end
+
+      def targets_of_products(products)
+        products = [products] if products.is_a?(String)
+        products.flat_map { |x| desc_of(x).targets_of_products(File.basename(x)) }
+      end
+
+      def dependency_targets_of_products(products)
+        products = [products] if products.is_a?(String)
+        products.flat_map { |p| @dependency_targets_by_products[p] || [p] }.uniq
+      end
+
+      def desc_of(d)
+        descs_by_name[d.split("/").first]
+      end
+
+      def binary_targets
+        descs_by_name.values.flatten.uniq.flat_map(&:binary_targets)
+      end
+    end
+  end
+end

data/lib/xccache/installer/integration/supporting_files.rb

@@ -0,0 +1,29 @@
+module XCCache
+  class Installer
+    module SupportingFilesIntegrationMixin
+      def gen_supporting_files
+        UI.section("Generating supporting files") do
+          gen_xcconfigs
+        end
+      end
+
+      private
+
+      def gen_xcconfigs
+        macros_config_by_targets.each do |target, hash|
+          xcconfig_path = config.spm_xcconfig_dir / "#{target}.xcconfig"
+          UI.message("XCConfig of target #{target} at: #{xcconfig_path}")
+          Xcodeproj::Config.new(hash).save_as(xcconfig_path)
+        end
+      end
+
+      def macros_config_by_targets
+        proxy_pkg.graph["macros"].to_h do |target, paths|
+          swift_flags = paths.map { |p| "-load-plugin-executable #{p}##{File.basename(p, '.*')}" }
+          hash = { "OTHER_SWIFT_FLAGS" => "$(inherited) #{swift_flags.join(' ')}" }
+          [File.basename(target, ".*"), hash]
+        end
+      end
+    end
+  end
+end

data/lib/xccache/installer/integration/viz.rb

@@ -0,0 +1,38 @@
+module XCCache
+  class Installer
+    module VizIntegrationMixin
+      def gen_cachemap_viz
+        stats = config.cachemap.stats
+        html_path = config.sandbox / "cachemap.html"
+        js_path = Dir.prepare(config.sandbox / "assets") / "cachemap.js"
+        css_path = config.sandbox / "assets" / "style.css"
+
+        root_dir = Pathname(".").expand_path
+        to_relative = proc do |p|
+          p.to_s.start_with?(root_dir.to_s) ? p.relative_path_from(root_dir).to_s : p.to_s
+        end
+
+        UI.info("Cachemap visualization: #{html_path}")
+        Template.new("cachemap.html").render(
+          {
+            :root_dir => root_dir.to_s,
+            :root_dir_short => root_dir.basename.to_s,
+            :lockfile_path => config.lockfile.path.to_s,
+            :lockfile_path_short => to_relative.call(config.lockfile.path),
+            :binaries_dir => config.spm_binaries_dir.to_s,
+            :binaries_dir_short => to_relative.call(config.spm_binaries_dir),
+            :desc_hit => stats[:hit],
+            :desc_missed => stats[:missed],
+            :desc_ignored => stats[:ignored],
+          },
+          save_to: html_path
+        )
+        Template.new("cachemap.js").render(
+          { :json => JSON.pretty_generate(config.cachemap.depgraph_data) },
+          save_to: js_path
+        )
+        Template.new("cachemap.style.css").render(save_to: css_path)
+      end
+    end
+  end
+end

data/lib/xccache/installer/integration.rb

@@ -0,0 +1,12 @@
+Dir["#{__dir__}/#{File.basename(__FILE__, '.rb')}/*.rb"].sort.each { |f| require f }
+
+module XCCache
+  class Installer
+    module IntegrationMixin
+      include VizIntegrationMixin
+      include DescsIntegrationMixin
+      include BuildIntegrationMixin
+      include SupportingFilesIntegrationMixin
+    end
+  end
+end

data/lib/xccache/installer/use.rb

@@ -19,7 +19,7 @@ module XCCache
         project.add_xccache_pkg unless project.has_xccache_pkg?
         project.targets.each do |target|
           target.add_xccache_product_dependency unless target.has_xccache_product_dependency?
-          target.remove_pkg_product_dependencies { |d| !d.pkg.xccache_pkg? }
+          target.remove_pkg_product_dependencies { |d| d.pkg.nil? || !d.pkg.xccache_pkg? }
         end
         project.remove_pkgs(&:non_xccache_pkg?) unless config.keep_pkgs_in_project?
       end