refinement 0.6.0 → 0.7.0

data/lib/sq/refinement/changeset/file_modification.rb

@@ -0,0 +1,144 @@
+# frozen_string_literal: true
+
+require 'xcodeproj'
+
+module Sq
+  module Refinement
+    class Changeset
+      # Represents a modification to a single file or directory on disk
+      class FileModification
+        # @return [Symbol] the change type for directories
+        DIRECTORY_CHANGE_TYPE = :'had contents change'
+
+        # @return [Pathname] the path to the modified file
+        attr_reader :path
+
+        # @return [Pathname, Nil] the prior path to the modified file, or `nil` if it was not renamed or copied
+        attr_reader :prior_path
+
+        # @return [#to_s] the type of change that happened to this file
+        attr_reader :type
+
+        def initialize(path:, type:,
+                      prior_path: nil,
+                      contents_reader: -> {},
+                      prior_contents_reader: -> {})
+          @path = path
+          @type = type
+          @prior_path = prior_path
+          @contents_reader = contents_reader
+          @prior_contents_reader = prior_contents_reader
+        end
+
+        # @visibility private
+        def to_s
+          case type
+          when DIRECTORY_CHANGE_TYPE
+            "contents of dir `#{path}` changed"
+          else
+            message = "file `#{path}` #{type}"
+            message += " (from #{prior_path})" if prior_path
+            message
+          end
+        end
+
+        # @visibility private
+        def inspect
+          "#<#{self.class} path=#{path.inspect} type=#{type.inspect} prior_path=#{prior_path.inspect} " \
+            "contents=#{contents.inspect} prior_contents=#{prior_contents.inspect}>"
+        end
+
+        # @visibility private
+        def hash
+          [path, type].hash
+        end
+
+        # @visibility private
+        def ==(other)
+          return unless other.is_a?(FileModification)
+
+          (path == other.path) && (type == other.type) && prior_path == other.prior_path
+        end
+
+        # @visibility private
+        def eql?(other)
+          return false unless other.is_a?(FileModification)
+
+          path.eql?(other.path) && type.eql?(other.type) && prior_path.eql?(other.prior_path)
+        end
+
+        # @return [String,Nil] a YAML string representing the diff of the file
+        #  from the prior revision to the current revision at the given keypath
+        #  in the YAML, or `nil` if there is no diff
+        # @param keypath [Array] a list of indices passed to `dig`.
+        #  An empty array is equivalent to the entire YAML document
+        def yaml_diff(keypath)
+          require 'yaml'
+
+          @cached_yaml ||= {}
+
+          dig_yaml = lambda do |yaml, path|
+            return yaml if yaml == DOES_NOT_EXIST
+
+            object = @cached_yaml[path] ||= YAML.safe_load(yaml, permitted_classes: [Symbol], aliases: true)
+            if keypath.empty?
+              object
+            elsif object.respond_to?(:dig)
+              object.dig(*keypath)
+            else # backwards compatibility
+              keypath.reduce(object) do |acc, elem|
+                acc[elem]
+              end
+            end
+          end
+
+          prior = dig_yaml[prior_contents, :prior]
+          current = dig_yaml[contents, :current]
+
+          require 'xcodeproj/differ'
+
+          # rubocop:disable Naming/VariableNumber
+          return unless (diff = Xcodeproj::Differ.diff(
+            prior,
+            current,
+            key_1: 'prior_revision',
+            key_2: 'current_revision'
+          ))
+          # rubocop:enable Naming/VariableNumber
+
+          diff.to_yaml.prepend("#{path} changed at keypath #{keypath.inspect}\n")
+        end
+
+        DOES_NOT_EXIST = Object.new.tap do |o|
+          class << o
+            def to_s
+              'DOES NOT EXISTS'
+            end
+            alias_method :inspect, :to_s
+          end
+        end.freeze
+        private_constant :DOES_NOT_EXIST
+
+        # @return [String] the current contents of the file
+        def contents
+          @contents ||=
+            begin
+              @contents_reader[].tap { @contents_reader = nil } || DOES_NOT_EXIST
+            rescue StandardError
+              DOES_NOT_EXIST
+            end
+        end
+
+        # @return [String] the prior contents of the file
+        def prior_contents
+          @prior_contents ||=
+            begin
+              @prior_contents_reader[].tap { @prior_contents_reader = nil } || DOES_NOT_EXIST
+            rescue StandardError
+              DOES_NOT_EXIST
+            end
+        end
+      end
+    end
+  end
+end

data/lib/sq/refinement/changeset.rb

@@ -0,0 +1,225 @@
+# frozen_string_literal: true
+
+require 'cocoapods/executable'
+require 'set'
+
+module Sq
+  module Refinement
+    # Represents a set of changes in a repository between a prior revision and the current state
+    class Changeset
+      # An error that happens when computing a git diff
+      class GitError < Error; end
+      require 'sq/refinement/changeset/file_modification'
+
+      # @return [Pathname] the path to the repository
+      attr_reader :repository
+      # @return [Array<FileModification>] the modifications in the changeset
+      attr_reader :modifications
+      # @return [Hash<Pathname,FileModification>] modifications keyed by relative path
+      attr_reader :modified_paths
+      # @return [Hash<Pathname,FileModification>] modifications keyed by relative path
+      attr_reader :modified_absolute_paths
+      # @return [String] a desciption of the changeset
+      attr_reader :description
+
+      private :modifications, :modified_paths, :modified_absolute_paths
+
+      def initialize(repository:, modifications:, description: nil)
+        @repository = repository
+        @modifications = self.class.add_directories(modifications).uniq.freeze
+        @description = description
+
+        @modified_paths = {}
+        @modifications
+          .each { |mod| @modified_paths[mod.path] = mod }
+          .each { |mod| @modified_paths[mod.prior_path] ||= mod if mod.prior_path }
+        @modified_paths.freeze
+
+        @modified_absolute_paths = {}
+        @modified_paths
+          .each { |path, mod| @modified_absolute_paths[path.expand_path(repository).freeze] = mod }
+        @modified_absolute_paths.freeze
+      end
+
+      # @visibility private
+      # @return [Array<FileModification>] file modifications that include modifications for each
+      #                                   directory that has had a child modified
+      # @param  modifications [Array<FileModification>] The modifications to add directory modifications to
+      def self.add_directories(modifications)
+        dirs = Set.new
+        add = lambda { |path|
+          break unless dirs.add?(path)
+
+          add[path.dirname]
+        }
+        modifications.each do |mod|
+          add[mod.path.dirname]
+          add[mod.prior_path.dirname] if mod.prior_path
+        end
+        modifications +
+          dirs.map { |d| FileModification.new(path: Pathname("#{d}/").freeze, type: FileModification::DIRECTORY_CHANGE_TYPE) }
+      end
+
+      # @return [FileModification,Nil] the changeset for the given absolute path,
+      #   or `nil` if the given path is un-modified
+      # @param absolute_path [Pathname]
+      def find_modification_for_path(absolute_path:)
+        modified_absolute_paths[absolute_path]
+      end
+
+      # @return [Array<String>] An array of patterns converted from a
+      #         {Dir.glob} pattern to patterns that {File.fnmatch} can handle.
+      #         This is used by the {#relative_glob} method to emulate
+      #         {Dir.glob}.
+      #
+      #   The expansion provides support for:
+      #
+      #   - Literals
+      #
+      #       dir_glob_equivalent_patterns('{file1,file2}.{h,m}')
+      #       => ["file1.h", "file1.m", "file2.h", "file2.m"]
+      #
+      #   - Matching the direct children of a directory with `**`
+      #
+      #       dir_glob_equivalent_patterns('Classes/**/file.m')
+      #       => ["Classes/**/file.m", "Classes/file.m"]
+      #
+      # @param [String] pattern   A {Dir#glob} like pattern.
+      #
+      def dir_glob_equivalent_patterns(pattern)
+        pattern = pattern.gsub('/**/', '{/**/,/}')
+        values_by_set = {}
+        pattern.scan(/\{[^}]*\}/) do |set|
+          values = set.gsub(/[{}]/, '').split(',', -1)
+          values_by_set[set] = values
+        end
+
+        if values_by_set.empty?
+          [pattern]
+        else
+          patterns = [pattern]
+          values_by_set.each do |set, values|
+            patterns = patterns.flat_map do |old_pattern|
+              values.map do |value|
+                old_pattern.gsub(set, value)
+              end
+            end
+          end
+          patterns
+        end
+      end
+      private :dir_glob_equivalent_patterns
+
+      # @return [FileModification,Nil] the modification for the given absolute glob,
+      #   or `nil` if no files matching the glob were modified
+      # @note Will only return a single (arbitrary) matching modification, even if there are
+      #   multiple modifications that match the glob
+      # @param absolute_glob [String] a glob pattern for absolute paths, suitable for an invocation of `Dir.glob`
+      def find_modification_for_glob(absolute_glob:)
+        absolute_globs = dir_glob_equivalent_patterns(absolute_glob)
+        _path, modification = modified_absolute_paths.find do |absolute_path, _modification|
+          absolute_globs.any? do |glob|
+            File.fnmatch?(glob, absolute_path, File::FNM_CASEFOLD | File::FNM_PATHNAME)
+          end
+        end
+        modification
+      end
+
+      # @return [FileModification,Nil] a modification and yaml diff for the keypath at the given absolute path,
+      #  or `nil` if the value at the given keypath is un-modified
+      # @param absolute_path [Pathname]
+      # @param keypath [Array]
+      def find_modification_for_yaml_keypath(absolute_path:, keypath:)
+        return unless (file_modification = find_modification_for_path(absolute_path:))
+
+        diff = file_modification.yaml_diff(keypath)
+        return unless diff
+
+        [file_modification, diff]
+      end
+
+      # @return [Changeset] the changes in the given git repository between the given revision and HEAD
+      # @param repository [Pathname]
+      # @param base_revision [String]
+      def self.from_git(repository:, base_revision:)
+        raise ArgumentError, "must be given a Pathname for repository, got #{repository.inspect}" unless repository.is_a?(Pathname)
+        raise ArgumentError, "must be given a String for base_revision, got #{base_revision.inspect}" unless base_revision.is_a?(String)
+
+        merge_base = git!('merge-base', base_revision, 'HEAD', chdir: repository).strip
+        diff = git!('diff', '--raw', '-z', merge_base, chdir: repository)
+        modifications = parse_raw_diff(diff, repository:, base_revision: merge_base).freeze
+
+        new(repository:, modifications:, description: "since #{base_revision}")
+      end
+
+      CHANGE_TYPES = {
+        'was added': 'A',
+        'was copied': 'C',
+        'was deleted': 'D',
+        'was modified': 'M',
+        'was renamed': 'R',
+        'changed type': 'T',
+        'is unmerged': 'U',
+        'changed in an unknown way': 'X'
+      }.freeze
+      private_constant :CHANGE_TYPES
+
+      CHANGE_CHARACTERS = CHANGE_TYPES.invert.freeze
+      private_constant :CHANGE_CHARACTERS
+
+      # Parses the raw diff into FileModification objects
+      # @return [Array<FileModification>]
+      # @param diff [String] a diff generated by `git diff --raw -z`
+      # @param repository [Pathname] the path to the repository
+      # @param base_revision [String] the base revision the diff was constructed agains
+      def self.parse_raw_diff(diff, repository:, base_revision:)
+        # since we're null separating the chunks (to avoid dealing with path escaping) we have to reconstruct
+        # the chunks into individual diff entries. entries always start with a colon so we can use that to signal if
+        # we're on a new entry
+        parsed_lines = diff.split("\0").each_with_object([]) do |chunk, lines|
+          lines << [] if chunk.start_with?(':')
+          lines.last << chunk
+        end
+
+        parsed_lines.map do |split_line|
+          # change chunk (letter + optional similarity percentage) will always be the last part of first line chunk
+          change_chunk = split_line[0].split(/\s/).last
+
+          new_path = Pathname(split_line[2]).freeze if split_line[2]
+          old_path = Pathname(split_line[1]).freeze
+          prior_path = old_path if new_path
+          # new path if one exists, else existing path. new path only exists for rename and copy
+          changed_path = new_path || old_path
+
+          change_character = change_chunk[0]
+          # returns 0 when a similarity percentage isn't specified by git.
+          _similarity = change_chunk[1..3].to_i
+
+          FileModification.new(
+            path: changed_path,
+            type: CHANGE_CHARACTERS[change_character],
+            prior_path:,
+            contents_reader: -> { repository.join(changed_path).read },
+            prior_contents_reader: lambda {
+              git!('show', "#{base_revision}:#{prior_path || changed_path}", chdir: repository)
+            }
+          )
+        end
+      end
+
+      # @return [String] the STDOUT of the git command
+      # @raise [GitError] when running the git command fails
+      # @param command [String] the base git command to run
+      # @param args [String...] arguments to the git command
+      # @param chdir [String,Pathname] the directory to run the git command in
+      def self.git!(command, *args, chdir:)
+        require 'open3'
+        out, err, status = Open3.capture3('git', command, *args, chdir: chdir.to_s)
+        raise GitError, "Running git #{command} failed (#{status.to_s.gsub(/pid \d+\s*/, '')}):\n\n#{err}" unless status.success?
+
+        out
+      end
+      private_class_method :git!
+    end
+  end
+end

data/lib/sq/refinement/cli.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+require 'claide'
+
+module Sq
+  module Refinement
+    # @visibility private
+    class CLI < CLAide::Command
+      self.abstract_command = true
+      self.command = 'refine'
+      self.version = VERSION
+
+      self.summary = 'Generates a list of Xcode targets to build & test as a result of a diff'
+
+      def self.options
+        super + [
+          ['--repository=REPOSITORY', 'Path to repository'],
+          ['--workspace=WORKSPACE_PATH', 'Path to project or workspace'],
+          ['--scheme=SCHEME_PATH', 'Path to scheme to be filtered'],
+          ['--augmenting-paths-yaml-files=PATH1,PATH2...', 'Paths to augmenting yaml files, relative to the repository path'],
+          ['--[no-]print-changes', 'Print the change reason for changed targets'],
+          ['--[no-]print-scheme-changes', 'Print the change reason for targets in the given scheme'],
+          ['--change-level=LEVEL', 'Change level at which a target must have changed in order to be considered changed. ' \
+                                  'One of `full-transitive`, `itself`, or an integer'],
+          ['--filter-scheme-for-build-action=BUILD_ACTION', 'The xcodebuild action the scheme (if given) is filtered for. ' \
+                                                            'One of `building` or `testing`.']
+        ]
+      end
+
+      def initialize(argv)
+        @repository = argv.option('repository', '.')
+        @workspace = argv.option('workspace')
+        @scheme = argv.option('scheme')
+        @augmenting_paths_yaml_files = argv.option('augmenting-paths-yaml-files', '')
+        @print_changes = argv.flag?('print-changes', false)
+        @print_scheme_changes = argv.flag?('print-scheme-changes', false)
+        @change_level = argv.option('change-level', 'full-transitive')
+        @filter_scheme_for_build_action = argv.option('filter-scheme-for-build-action', 'testing').to_sym
+
+        super
+      end
+
+      def run
+        changeset = compute_changeset
+
+        analyzer = Sq::Refinement::Analyzer.new(changeset:,
+                                            workspace_path: @workspace,
+                                            augmenting_paths_yaml_files: @augmenting_paths_yaml_files)
+        analyzer.annotate_targets!
+
+        puts analyzer.format_changes if @print_changes
+
+        return unless @scheme
+
+        analyzer.filtered_scheme(scheme_path: @scheme, log_changes: @print_scheme_changes, filter_scheme_for_build_action: @filter_scheme_for_build_action)
+                .save_as(@scheme.gsub(%r{\.(xcodeproj|xcworkspace)/.+}, '.\1'), File.basename(@scheme, '.xcscheme'), true)
+      end
+
+      def validate!
+        super
+
+        File.directory?(@repository) || help!("Unable to find a repository at #{@repository.inspect}")
+
+        @workspace || help!('Must specify a project or workspace path')
+        File.directory?(@workspace) || help!("Unable to find a project or workspace at #{@workspace.inspect}")
+
+        @augmenting_paths_yaml_files = @augmenting_paths_yaml_files.split(',')
+        @augmenting_paths_yaml_files.each do |yaml_path|
+          yaml_path = File.join(@repository, yaml_path)
+          File.file?(yaml_path) || help!("Unable to find a YAML file at #{yaml_path.inspect}")
+
+          require 'yaml'
+          begin
+            YAML.safe_load_file(yaml_path)
+          rescue StandardError => e
+            help! "Failed to load YAML file at #{yaml_path.inspect} (#{e})"
+          end
+        end
+
+        File.file?(@scheme) || help!("Unabled to find a scheme at #{@scheme.inspect}") if @scheme
+
+        @change_level =
+          case @change_level
+          when 'full-transitive' then :full_transitive
+          when 'itself' then :itself
+          when /\A\d+\z/ then [:at_most_n_away, @change_level.to_i]
+          else help! "Unknown change level #{@change_level.inspect}"
+          end
+      end
+
+      # @visibility private
+      class Git < CLI
+        self.summary = 'Generates a list of Xcode targets to build & test as a result of a git diff'
+
+        def self.options
+          super + [
+            ['--base-revision=SHA', 'Base revision to compute the git diff against']
+          ]
+        end
+
+        def initialize(argv)
+          @base_revision = argv.option('base-revision')
+
+          super
+        end
+
+        def validate!
+          super
+
+          @base_revision || help!('Must specify a base revision')
+        end
+
+        private
+
+        def compute_changeset
+          Sq::Refinement::Changeset.from_git(repository: Pathname(@repository), base_revision: @base_revision)
+        end
+      end
+    end
+  end
+end

data/lib/sq/refinement/cocoapods_post_install_writer.rb

@@ -0,0 +1,129 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module Sq
+  module Refinement
+    # Called after CocoaPods installation to write an augmenting file that
+    # takes into account changes to Pod configuration,
+    # as well as the globs used by podspecs to search for files
+    class CocoaPodsPostInstallWriter
+      attr_reader :aggregate_targets, :pod_targets, :config, :repo, :options
+      private :aggregate_targets, :pod_targets, :config, :repo, :options
+
+      # Initializes a post-install writer with CocoaPods target objects.
+      # @return [CocoaPodsPostInstallWriter] a new instance of CocoaPodsPostInstallWriter
+      # @param aggregate_targets [Array<Pod::AggregateTarget>]
+      # @param pod_targets [Array<Pod::PodTarget>]
+      # @param config [Pod::Config]
+      # @param options [Hash]
+      def initialize(aggregate_targets, pod_targets, config, options)
+        @aggregate_targets = aggregate_targets
+        @pod_targets = pod_targets
+        @config = config
+        @repo = config.installation_root
+        @options = options || {}
+      end
+
+      # Writes the refinement augmenting file to the configured path
+      # @return [Void]
+      def write!
+        write_file options.fetch('output_path', config.sandbox.root.join('pods_refinement.json')), paths_by_target_name
+      end
+
+      private
+
+      def write_file(path, hash)
+        File.open(path, 'w') do |f|
+          f << JSON.generate(hash) << "\n"
+        end
+      end
+
+      def paths_by_target_name
+        targets = {}
+        aggregate_targets.each do |aggregate_target|
+          targets[aggregate_target.label] = paths_for_aggregate_target(aggregate_target)
+        end
+        pod_targets.each do |pod_target|
+          targets.merge! paths_for_pod_targets(pod_target)
+        end
+        targets
+      end
+
+      def paths_for_aggregate_target(aggregate_target)
+        paths = []
+        if (podfile_path = aggregate_target.podfile.defined_in_file)
+          paths << { path: podfile_path.relative_path_from(repo), inclusion_reason: 'Podfile' }
+        end
+        if (user_project_path = aggregate_target.user_project_path)
+          paths << { path: user_project_path.relative_path_from(repo), inclusion_reason: 'user project' }
+        end
+        paths
+      end
+
+      def library_specification?(spec)
+        # Backwards compatibility
+        if spec.respond_to?(:library_specification?, false)
+          spec.library_specification?
+        else
+          !spec.test_specification?
+        end
+      end
+
+      def specification_paths_from_pod_target(pod_target)
+        pod_target
+          .target_definitions
+          .map(&:podfile)
+          .uniq
+          .flat_map do |podfile|
+          podfile
+            .dependencies
+            .select { |d| d.root_name == pod_target.pod_name }
+            .map { |d| (d.external_source || {})[:path] }
+            .compact
+        end.uniq
+      end
+
+      def paths_for_pod_targets(pod_target)
+        file_accessors_by_target_name = pod_target.file_accessors.group_by do |fa|
+          if library_specification?(fa.spec)
+            pod_target.label
+          elsif pod_target.respond_to?(:non_library_spec_label)
+            pod_target.non_library_spec_label(fa.spec)
+          else
+            pod_target.test_target_label(fa.spec)
+          end
+        end
+
+        pod_dir = pod_target.sandbox.pod_dir(pod_target.pod_name).relative_path_from(repo)
+
+        spec_paths = specification_paths_from_pod_target(pod_target)
+
+        file_accessors_by_target_name.each_with_object({}) do |(label, file_accessors), h|
+          paths = [
+            { path: 'Podfile.lock',
+              inclusion_reason: 'CocoaPods lockfile',
+              yaml_keypath: ['SPEC CHECKSUMS', pod_target.pod_name] }
+          ]
+          if pod_target.sandbox.predownloaded?(pod_target.pod_name)
+            paths << { path: 'Podfile.lock', inclusion_reason: 'Dependency external source', yaml_keypath: ['EXTERNAL SOURCES', pod_target.pod_name] }
+            paths << { path: 'Podfile.lock', inclusion_reason: 'Pod checkout options', yaml_keypath: ['CHECKOUT OPTIONS', pod_target.pod_name] }
+          end
+          spec_paths.each { |path| paths << { path:, inclusion_reason: 'podspec' } }
+
+          Pod::Validator::FILE_PATTERNS.each do |pattern|
+            file_accessors.each do |fa|
+              globs = fa.spec_consumer.send(pattern)
+              globs = globs.values.flatten if globs.is_a?(Hash)
+              globs.each do |glob|
+                paths << { glob: pod_dir.join(glob), inclusion_reason: pattern.to_s.tr('_', ' ').chomp('s') }
+              end
+            end
+          end
+
+          h[label] = paths.uniq
+        end
+      end
+    end
+  end
+end

data/lib/sq/refinement/setup.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Sq
+  # Generates a list of Xcode targets to build & test as a result of a git diff.
+  module Refinement
+    class Error < StandardError; end
+
+    # @visibility private
+    # @param enum [Enumerable]
+    # Enumerates through `enum`, and applied the given block to each element.
+    # If the result of calling the block is truthy, the first such result is returned.
+    # If no such result is found, `nil` is returned.
+    def self.map_find(enum)
+      enum.each do |elem|
+        transformed = yield elem
+        return transformed if transformed
+      end
+
+      nil
+    end
+  end
+end