xccache 0.0.1a1 → 0.0.1a3

data/lib/xccache/spm/desc/target.rb

@@ -0,0 +1,164 @@
+require "xccache/spm/desc/base"
+
+module XCCache
+  module SPM
+    class Package
+      class Target < BaseObject
+        include Cacheable
+        cacheable :recursive_targets, :direct_dependency_targets, :direct_dependencies
+
+        Dir["#{__dir__}/#{File.basename(__FILE__, '.rb')}/*.rb"].sort.each { |f| require f }
+
+        def xccache?
+          name.end_with?(".xccache")
+        end
+
+        def xccache_id
+          macro? ? "#{full_name}.macro" : full_name
+        end
+
+        def type
+          @type ||= raw["type"].to_sym
+        end
+
+        def downcast
+          cls = {
+            :binary => BinaryTarget,
+            :macro => MacroTarget,
+          }[type]
+          cls.nil? ? self : cast_to(cls)
+        end
+
+        def module_name
+          name.c99extidentifier
+        end
+
+        def resource_bundle_name
+          "#{pkg_name}_#{name}.bundle"
+        end
+
+        def flatten_as_targets
+          [self]
+        end
+
+        def sources_path
+          @sources_path ||= begin
+            path = raw["path"] || "Sources/#{name}"
+            root.src_dir / path
+          end
+        end
+
+        def use_clang?
+          !header_paths.empty?
+        end
+
+        def header_paths(options = {})
+          paths = []
+          paths += public_header_paths if options.fetch(:public, true)
+          paths += header_search_paths if options.fetch(:search, false)
+          paths
+            .flat_map { |p| p.glob("**/*.h*") }
+            .map(&:realpath)
+            .uniq
+        end
+
+        def settings
+          raw["settings"]
+        end
+
+        def header_search_paths
+          @header_search_paths ||=
+            settings
+            .filter_map { |h| h.fetch("kind", {})["headerSearchPath"] }
+            .flat_map(&:values)
+            .map { |p| sources_path / p }
+        end
+
+        def public_header_paths
+          @public_header_paths ||= begin
+            res = []
+            implicit_path = sources_path / "include"
+            res << implicit_path unless implicit_path.glob("**/*.h*").empty?
+            res << (sources_path / raw["publicHeadersPath"]) if raw.key?("publicHeadersPath")
+            res
+          end
+        end
+
+        def resource_paths
+          @resource_paths ||= begin
+            res = raw.fetch("resources", []).map { |h| sources_path / h["path"] }
+            # Refer to the following link for the implicit resources
+            # https://developer.apple.com/documentation/xcode/bundling-resources-with-a-swift-package#Add-resource-files
+            implicit = sources_path.glob("*.{xcassets,xib,storyboard,xcdatamodeld,lproj}")
+            res + implicit
+          end
+        end
+
+        def recursive_targets(platform: nil)
+          children = direct_dependency_targets(platform: platform)
+          children += children.flat_map { |t| t.macro? ? [t] : t.recursive_targets(platform: platform) }
+          children.uniq
+        end
+
+        def direct_dependencies(platform: nil)
+          raw["dependencies"].flat_map do |hash|
+            dep_types = ["byName", "target", "product"]
+            if (dep_type = dep_types.intersection(hash.keys).first).nil?
+              raise GeneralError, "Unexpected dependency type. Must be one of #{dep_types}. Hash: #{hash}"
+            end
+            next [] unless match_platform?(hash[dep_type][-1], platform)
+            pkg_name = hash[dep_type][1] if dep_type == "product"
+            find_deps(hash[dep_type][0], pkg_name, dep_type)
+          end
+        end
+
+        def direct_dependency_targets(platform: nil)
+          direct_dependencies(platform: platform).flat_map(&:flatten_as_targets).uniq
+        end
+
+        def match_platform?(_condition, _platform)
+          true # FIXME: Handle this
+        end
+
+        def macro?
+          type == :macro
+        end
+
+        def binary?
+          type == :binary
+        end
+
+        def binary_path
+          sources_path if binary?
+        end
+
+        def local_binary_path
+          binary_path if binary? && root.local?
+        end
+
+        def checksum
+          @checksum ||= root.git&.sha || sources_path.checksum
+        end
+
+        private
+
+        def find_deps(name, pkg_name, dep_type)
+          # If `dep_type` is `target` -> constrained within current pkg only
+          # If `dep_type` is `product` -> `pkg_name` must be present
+          # If `dep_type` is `byName` -> it's either from this pkg, or its children/dependencies
+          res = []
+          descs = pkg_name.nil? ? [root] + root.uniform_dependencies : [pkg_desc_of(pkg_name)]
+          descs.each do |desc|
+            by_target = -> { desc.targets.select { |t| t.name == name } }
+            by_product = -> { desc.products.select { |t| t.name == name } }
+            return by_target.call if dep_type == "target"
+            return by_product.call if dep_type == "product"
+            return res unless (res = by_target.call).empty?
+            return res unless (res = by_product.call).empty?
+          end
+          []
+        end
+      end
+    end
+  end
+end

data/lib/xccache/spm/desc.rb

@@ -0,0 +1 @@
+Dir["#{__dir__}/#{File.basename(__FILE__, '.rb')}/*.rb"].sort.each { |f| require f }

data/lib/xccache/spm/macro.rb

@@ -0,0 +1,44 @@
+require_relative "build"
+
+module XCCache
+  module SPM
+    class Macro < Buildable
+      attr_reader :macosx_sdk
+
+      def initialize(options = {})
+        super
+        @library_evolution = false # swift-syntax is not compatible with library evolution
+        @macosx_sdk = Swift::Sdk.new(:macosx)
+      end
+
+      def build(_options = {})
+        # NOTE: Building macro binary is tricky...
+        # --------------------------------------------------------------------------------
+        # Consider this manifest config: .target(Macro) -> .macro(MacroImpl)
+        #   where `.target(Macro)` contains the interfaces
+        #   and `.target(MacroImpl)` contains the implementation
+        # --------------------------------------------------------------------------------
+        # Building `.macro(MacroImpl)` does not produce the tool binary (MacroImpl-tool)... Only `.o` files.
+        # Yet, linking those files are exhaustive due to many dependencies in swift-syntax
+        # Luckily, building `.target(Macro)` does produce the tool binary.
+        # -> WORKAROUND: Find the associated regular target and build it, then collect the tool binary
+        # ---------------------------------------------------------------------------------
+        associated_target = pkg_desc.targets.find { |t| t.direct_dependency_targets.include?(pkg_target) }
+        UI.message(
+          "#{name.yellow.dark} is a macro target. " \
+          "Will build the associated target #{associated_target.name.dark} to get the tool binary."
+        )
+        swift_build(target: associated_target.name)
+        binary_path = products_dir / "#{module_name}-tool"
+        raise GeneralError, "Tool binary not exist at: #{binary_path}" unless binary_path.exist?
+        binary_path.copy(to: path)
+        FileUtils.chmod("+x", path)
+        UI.info("-> Macro binary: #{path.to_s.dark}")
+      end
+
+      def products_dir
+        @products_dir ||= pkg_dir / ".build" / macosx_sdk.triple / config
+      end
+    end
+  end
+end

data/lib/xccache/spm/mixin.rb

@@ -0,0 +1,12 @@
+module XCCache
+  module PkgMixin
+    include Config::Mixin
+
+    def umbrella_pkg
+      @umbrella_pkg ||= SPM::Package::Umbrella.new(
+        root_dir: config.spm_umbrella_sandbox,
+        warn_if_not_direct_target: false,
+      )
+    end
+  end
+end

data/lib/xccache/spm/pkg/base.rb

@@ -0,0 +1,107 @@
+require "json"
+require "xccache/spm/xcframework/slice"
+require "xccache/spm/xcframework/xcframework"
+require "xccache/spm/xcframework/metadata"
+require "xccache/swift/sdk"
+
+module XCCache
+  module SPM
+    class Package
+      include Cacheable
+      cacheable :pkg_desc_of_target
+
+      attr_reader :root_dir
+
+      def initialize(options = {})
+        @root_dir = Pathname(options[:root_dir] || ".").expand_path
+        @warn_if_not_direct_target = options.fetch(:warn_if_not_direct_target, true)
+      end
+
+      def build(options = {})
+        validate!
+        targets = options.delete(:targets) || []
+        raise GeneralError, "No targets were specified" if targets.empty?
+
+        targets.map { |t| t.split("/")[-1] }.each_with_index do |t, i|
+          UI.section("\n▶ Building target: #{t} (#{i + 1}/#{targets.count})".bold.magenta) do
+            build_target(**options, target: t)
+          rescue StandardError => e
+            UI.error("Failed to build target: #{t}. Error: #{e}")
+            raise e unless Config.instance.ignore_build_errors?
+          end
+        end
+      end
+
+      def build_target(target: nil, sdks: nil, config: nil, out_dir: nil, **options)
+        target_pkg_desc = pkg_desc_of_target(target, skip_resolving_dependencies: options[:skip_resolving_dependencies])
+        if target_pkg_desc.binary_targets.any? { |t| t.name == target }
+          return UI.warn("Target #{target} is a binary target -> no need to build")
+        end
+
+        target = target_pkg_desc.get_target(target)
+
+        out_dir = Pathname(out_dir || ".")
+        out_dir /= target.name if options[:checksum]
+        basename = options[:checksum] ? "#{target.name}-#{target.checksum}" : target.name
+        basename += target.macro? ? ".macro" : ".xcframework"
+
+        Dir.create_tmpdir do |_tmpdir|
+          cls = target.macro? ? Macro : XCFramework
+          cls.new(
+            name: target.name,
+            pkg_dir: root_dir,
+            config: config,
+            sdks: sdks,
+            path: out_dir / basename,
+            tmpdir: Dir.create_tmpdir,
+            pkg_desc: target_pkg_desc,
+            library_evolution: options[:library_evolution],
+          ).build(**options)
+        end
+      end
+
+      def resolve(force: false)
+        return if @resolved && !force
+
+        UI.section("Resolving package dependencies (package: #{root_dir.basename})", timing: true) do
+          Sh.run("swift package resolve --package-path #{root_dir} 2>&1")
+        end
+        @resolved = true
+      end
+
+      private
+
+      def validate!
+        return unless root_dir.glob("Package*.swift").empty?
+        raise GeneralError, "No Package.swift in #{root_dir}. Are you sure you're running on a package dir?"
+      end
+
+      def pkg_desc_of_target(name, skip_resolving_dependencies: false)
+        # The current package contains the given target
+        return pkg_desc if pkg_desc.has_target?(name)
+
+        if @warn_if_not_direct_target
+          UI.message(
+            "#{name.yellow.dark} is not a direct target of package #{root_dir.basename.to_s.dark} " \
+            "-> trigger from dependencies"
+          )
+        end
+        # Otherwise, it's inside one of the dependencies. Need to resolve then find it
+        resolve unless skip_resolving_dependencies
+
+        @descs ||= if Config.instance.in_installation?
+                   then Description.descs_in_metadata_dir[0]
+                   else
+                     Description.descs_in_dir(Pathname(".").expand_path)[0]
+                   end
+        desc = @descs.find { |d| d.has_target?(name) }
+        return desc unless desc.nil?
+        raise GeneralError, "Cannot find package with the given target #{name}"
+      end
+
+      def pkg_desc
+        @pkg_desc ||= Description.in_dir(root_dir)
+      end
+    end
+  end
+end

data/lib/xccache/spm/pkg/umbrella/build.rb

@@ -0,0 +1,30 @@
+module XCCache
+  module SPM
+    class Package
+      module UmbrellaBuildMixin
+        def build(options = {})
+          to_build = targets_to_build(options)
+          return UI.warn("Detected no targets to build among cache-missed targets") if to_build.empty?
+
+          UI.info("-> Targets to build: #{to_build.to_s.bold}")
+          super(options.merge(:targets => to_build))
+          sync_cachemap(sdks: options[:sdks])
+        end
+
+        def targets_to_build(options)
+          items = options[:targets] || []
+          items = config.cachemap.missed.map { |x| File.basename(x) } if items.empty?
+          targets = @descs.flat_map(&:targets).select { |t| items.include?(t.name) }
+          if options[:recursive]
+            UI.message("Will include cache-missed recursive targets")
+            targets += targets.flat_map do |t|
+              t.recursive_targets.select { |x| config.cachemap.missed?(x.full_name) }
+            end
+          end
+          # TODO: Sort by number of dependents
+          targets.map(&:full_name).uniq
+        end
+      end
+    end
+  end
+end

data/lib/xccache/spm/pkg/umbrella/cachemap.rb

@@ -0,0 +1,110 @@
+module XCCache
+  module SPM
+    class Package
+      module UmbrellaCachemapMixin
+        def sync_cachemap(sdks: [])
+          UI.section("Syncing cachemap (sdks: #{sdks.map(&:name)})")
+          nodes, edges, parents = xccache_desc.traverse
+          cache_data = gen_cache_data(nodes, parents, sdks)
+          targets_data, macros_data, deps_data = {}, {}, {}
+          xccache_desc.targets.each do |agg_target|
+            targets, macros = [], []
+            agg_target.direct_dependencies.each do |d|
+              all_hit = d.recursive_targets.all? { |t| cache_data[t] == :hit }
+              # If any associated targets is missed -> use original product form
+              # Otherwise, replace with recursive targets' binaries
+              deps_data[d.full_name] = d.recursive_targets.map(&:xccache_id)
+              targets << (all_hit ? "#{d.full_name}.binary" : d.full_name)
+              macros += d.recursive_targets.select(&:macro?).map(&:full_name) if all_hit
+            end
+            targets_data[agg_target.name] = targets.uniq.sort_by(&:downcase)
+            macros_data[agg_target.name] = macros.uniq
+          end
+
+          config.cachemap.raw = {
+            "manifest" => {
+              "targets" => targets_data,
+              "macros" => macros_data,
+              "deps" => deps_data,
+            },
+            "cache" => cache_data.transform_keys(&:full_name),
+            "depgraph" => {
+              "nodes" => nodes.map { |x| target_to_cytoscape_node(x, cache_data) },
+              "edges" => edges.map { |x, y| { :source => x.full_name, :target => y.full_name } },
+            },
+          }
+          config.cachemap.save
+          config.cachemap.print_stats
+        end
+
+        private
+
+        def gen_cache_data(nodes, parents, sdks)
+          result = nodes.to_h do |node|
+            res = if config.ignore?(node.full_name) then :ignored
+                  else
+                    verify_binary?(node, sdks) ? :hit : :missed
+                  end
+            [node, res]
+          end
+
+          # Propagate cache miss
+          to_visit = result.select { |_, v| %i[missed ignore].include?(v) }.keys
+          visited = Set.new
+          until to_visit.empty?
+            node = to_visit.pop
+            next if visited.include?(node)
+            visited << node
+            result[node] = :missed if result[node] == :hit
+            to_visit += parents[node] if parents.key?(node)
+          end
+          result.reject { |k, _| k.name.end_with?(".xccache") }
+        end
+
+        def verify_binary?(target, sdks)
+          return true if target.binary?
+
+          bpath = binary_path(target.xccache_id)
+          bpath_with_checksum = binary_path(target.xccache_id, checksum: target.checksum, in_repo: true)
+
+          check = proc do
+            # For macro, we just need the tool binary to exist
+            # For regular targets, the xcframework must satisfy the sdk constraints (ie. containing all the slices)
+            next bpath_with_checksum.exist? if target.macro?
+
+            metadata = XCFramework::Metadata.new(bpath_with_checksum / "Info.plist")
+            expected_triples = sdks.map { |sdk| sdk.triple(without_vendor: true) }
+            missing_triples = expected_triples - metadata.triples
+            missing_triples.empty?
+          end
+
+          # If requirements are meet, create symlink `A-abc123.xcframework` -> `A.framework`
+          # Otherwise, remove symlink `A.xcframework`
+          if check.call
+            bpath_with_checksum.symlink_to(bpath)
+          elsif bpath.exist?
+            bpath.rmtree
+          end
+          bpath.exist?
+        end
+
+        def binary_path(name, checksum: nil, in_repo: false)
+          suffix = checksum.nil? ? "" : "-#{checksum}"
+          ext = File.extname(name) == ".macro" ? ".macro" : ".xcframework"
+          binaries_dir = in_repo ? config.spm_repo_dir : config.spm_binaries_dir
+          p = binaries_dir / File.basename(name, ".*")
+          p / "#{p.basename}#{suffix}#{ext}"
+        end
+
+        def target_to_cytoscape_node(x, cache_data)
+          h = { :id => x.full_name, :cache => cache_data[x] }
+          h[:type] = if x.name.end_with?(".xccache") then "agg"
+                     elsif x.macro? then "macro" end
+          h[:checksum] = x.checksum
+          h[:binary] = binary_path(x.xccache_id) if cache_data[x] == :hit
+          h
+        end
+      end
+    end
+  end
+end

data/lib/xccache/spm/pkg/umbrella/descs.rb

@@ -0,0 +1,35 @@
+module XCCache
+  module SPM
+    class Package
+      module UmbrellaDescsMixin
+        def gen_metadata
+          UI.section("Generating metadata of packages", timing: true) do
+            @descs, @descs_by_name = Description.descs_in_dir(root_dir, save_to_dir: config.spm_metadata_dir)
+          end
+        end
+
+        def xccache_desc
+          @xccache_desc ||= desc_of("xccache")
+        end
+
+        def targets_of_products(products)
+          products = [products] if products.is_a?(String)
+          products.flat_map { |x| desc_of(x).targets_of_products(File.basename(x)) }
+        end
+
+        def dependency_targets_of_products(products)
+          products = [products] if products.is_a?(String)
+          products.flat_map { |p| @dependency_targets_by_products[p] || [p] }.uniq
+        end
+
+        def desc_of(d)
+          @descs_by_name[d.split("/").first]
+        end
+
+        def binary_targets
+          @descs_by_name.values.flatten.uniq.flat_map(&:binary_targets)
+        end
+      end
+    end
+  end
+end

data/lib/xccache/spm/pkg/umbrella/manifest.rb

@@ -0,0 +1,83 @@
+module XCCache
+  module SPM
+    class Package
+      module UmbrellaManifestMixin
+        def write_manifest(no_cache: false)
+          UI.info("Writing Package.swift (package: #{root_dir.basename})")
+          Template.new("umbrella.Package.swift").render(
+            {
+              :timestamp => Time.new.strftime("%F %T"),
+              :json => manifest_targets_json(no_cache: no_cache),
+              :products_to_targets => manifest_products_to_targets_json(no_cache: no_cache),
+              :platforms => manifest_platforms,
+              :dependencies => manifest_pkg_dependencies,
+              :swift_version => Swift::Swiftc.version_without_patch,
+            },
+            save_to: root_dir / "Package.swift",
+          )
+        end
+
+        def manifest_targets_json(no_cache: false)
+          data = no_cache ? config.lockfile.targets_data : config.cachemap.manifest_data["targets"]
+          JSON.pretty_generate(data)
+        end
+
+        def manifest_products_to_targets_json(no_cache: false)
+          data = no_cache ? {} : config.cachemap.manifest_data["deps"]
+          JSON.pretty_generate(data)
+        end
+
+        def manifest_pkg_dependencies
+          decl = proc do |hash|
+            if (path_from_root = hash["path_from_root"])
+              absolute_path = (Pathname(".") / path_from_root).expand_path
+              next ".package(path: \"#{absolute_path}\")"
+            end
+
+            requirement = hash["requirement"]
+            case requirement["kind"]
+            when "upToNextMajorVersion"
+              opt = ".upToNextMajor(from: \"#{requirement['minimumVersion']}\")"
+            when "upToNextMinorVersion"
+              opt = ".upToNextMinor(from: \"#{requirement['minimumVersion']}\")"
+            when "exactVersion"
+              opt = "exact: \"#{requirement['version']}\""
+            when "branch"
+              opt = "branch: \"#{requirement['branch']}\""
+            when "revision"
+              opt = "revision: \"#{requirement['revision']}\""
+            when "versionRange"
+              opt = "\"#{requirement['minimumVersion']}\"..<\"#{requirement['maximumVersion']}\""
+            end
+            ".package(url: \"#{hash['repositoryURL']}\", #{opt})"
+          end
+
+          config.lockfile.pkgs.map { |h| "  #{decl.call(h)}," }.join("\n")
+        end
+
+        def manifest_platforms
+          @manifest_platforms ||= begin
+            to_spm_platform = {
+              :ios => "iOS",
+              :macos => "macOS",
+              :osx => "macOS",
+              :tvos => "tvOS",
+              :watchos => "watchOS",
+              :visionos => "visionOS",
+            }
+            hash = {}
+            config.project_targets.each do |t|
+              platform = to_spm_platform[t.platform_name]
+              hash[platform] ||= []
+              hash[platform] << t.deployment_target.split(".")[0]
+            end
+            hash
+              .transform_values(&:min)
+              .map { |platform, version| "  .#{platform}(.v#{version})," }
+              .join("\n")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/xccache/spm/pkg/umbrella/viz.rb

@@ -0,0 +1,40 @@
+module XCCache
+  module SPM
+    class Package
+      module UmbrellaVizMixin
+        def gen_cachemap_viz
+          stats = config.cachemap.stats
+          html_path = config.sandbox / "cachemap.html"
+          js_path = Dir.prepare(config.sandbox / "assets") / "cachemap.js"
+          css_path = config.sandbox / "assets" / "style.css"
+
+          root_dir = Pathname(".").expand_path
+          to_relative = proc do |p|
+            p.to_s.start_with?(root_dir.to_s) ? p.relative_path_from(root_dir).to_s : p.to_s
+          end
+
+          UI.info("Cachemap visualization: #{html_path}")
+          Template.new("cachemap.html").render(
+            {
+              :root_dir => root_dir.to_s,
+              :root_dir_short => root_dir.basename.to_s,
+              :lockfile_path => config.lockfile.path.to_s,
+              :lockfile_path_short => to_relative.call(config.lockfile.path),
+              :binaries_dir => config.spm_binaries_dir.to_s,
+              :binaries_dir_short => to_relative.call(config.spm_binaries_dir),
+              :desc_hit => stats[:hit],
+              :desc_missed => stats[:missed],
+              :desc_ignored => stats[:ignored],
+            },
+            save_to: html_path
+          )
+          Template.new("cachemap.js").render(
+            { :json => JSON.pretty_generate(config.cachemap.depgraph_data) },
+            save_to: js_path
+          )
+          Template.new("cachemap.style.css").render(save_to: css_path)
+        end
+      end
+    end
+  end
+end

data/lib/xccache/spm/pkg/umbrella/xcconfigs.rb

@@ -0,0 +1,31 @@
+module XCCache
+  module SPM
+    class Package
+      module UmbrellaXCConfigsMixin
+        def gen_xcconfigs
+          UI.section("Generating xcconfigs") do
+            macros_config_by_targets.each do |target, hash|
+              xcconfig_path = config.spm_xcconfig_dir / "#{target}.xcconfig"
+              UI.message("XCConfig of target #{target} at: #{xcconfig_path}")
+              Xcodeproj::Config.new(hash).save_as(xcconfig_path)
+            end
+          end
+        end
+
+        private
+
+        def macros_config_by_targets
+          config.cachemap.manifest_data["macros"].to_h do |target, macros|
+            swift_flags = macros.map do |m|
+              basename = File.basename(m, ".*")
+              binary_path = config.spm_binaries_dir / basename / "#{basename}.macro"
+              "-load-plugin-executable #{binary_path}##{basename}"
+            end
+            hash = { "OTHER_SWIFT_FLAGS" => "$(inherited) #{swift_flags.join(' ')}" }
+            [File.basename(target, ".*"), hash]
+          end
+        end
+      end
+    end
+  end
+end