buildkite-builder 1.0.0.beta.1

data/lib/buildkite/builder/commands/files.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Buildkite
+  module Builder
+    module Commands
+      class Files < Abstract
+        private
+
+        self.description = 'Outputs files that match the specified manifest.'
+
+        def run
+          pipeline, manifest = ARGV.first.to_s.split('/')
+          if !pipeline || !manifest
+            raise 'You must specify a pipeline and a manifest (eg "mypipeline/mymanifest")'
+          end
+
+          manifests = Loaders::Manifests.load(pipeline)
+          manifests[manifest].files.each do |file|
+            puts file
+          end
+        end
+      end
+    end
+  end
+end

data/lib/buildkite/builder/commands/preview.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Buildkite
+  module Builder
+    module Commands
+      class Preview < Abstract
+        private
+
+        self.description = 'Outputs the pipeline YAML.'
+
+        def run
+          unless pipeline
+            raise 'You must specify a pipeline'
+          end
+
+          puts Runner.new(pipeline: pipeline).run.to_yaml
+        end
+
+        def pipeline
+          @pipeline ||= ARGV.last || begin
+            if available_pipelines.one?
+              available_pipelines.first
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/buildkite/builder/definition.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Buildkite
+  module Builder
+    module Definition
+      module Helper
+        def load_definition(file, expected)
+          result = eval(file.read, TOPLEVEL_BINDING.dup, file.to_s) # rubocop:disable Security/Eval
+          unless result.is_a?(expected)
+            raise "#{file} must return a valid definition (#{expected}); got #{result.class}"
+          end
+
+          result
+        end
+      end
+
+      class Pipeline < Proc
+      end
+
+      class Template < Proc
+      end
+    end
+  end
+end

data/lib/buildkite/builder/file_resolver.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require 'open3'
+require 'set'
+
+module Buildkite
+  module Builder
+    class FileResolver
+      @cache = true
+
+      attr_reader :modified_files
+
+      class << self
+        attr_accessor :cache
+
+        def resolve(reset = false)
+          @resolve = nil if !cache || reset
+          @resolve ||= new
+        end
+      end
+
+      def initialize
+        @modified_files = SortedSet.new(pull_request? ? files_from_pull_request : files_from_git)
+      end
+
+      private
+
+      def files_from_pull_request
+        Github.pull_request_files.map { |f| f.fetch('filename') }
+      end
+
+      def files_from_git
+        if Buildkite.env
+          changed_files = command("git diff-tree --no-commit-id --name-only -r #{Buildkite.env.commit}")
+        else
+          default_branch = command('git symbolic-ref refs/remotes/origin/HEAD').strip
+          changed_files = command("git diff --name-only #{default_branch}")
+          changed_files << command('git diff --name-only')
+        end
+
+        changed_files.split.uniq.sort
+      end
+
+      def pull_request?
+        Buildkite.env&.pull_request
+      end
+
+      def command(cmd)
+        output, status = Open3.capture2(*cmd.split)
+
+        if status.success?
+          output
+        else
+          raise "Command failed (exit #{status.exitstatus}): #{cmd}"
+        end
+      end
+    end
+  end
+end

data/lib/buildkite/builder/github.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'net/http'
+require 'uri'
+
+module Buildkite
+  module Builder
+    class Github
+      BASE_URI = URI('https://api.github.com').freeze
+      ACCEPT_HEADER = 'application/vnd.github.v3+json'
+      LINK_HEADER = 'link'
+      NEXT_LINK_REGEX = /<(?<uri>.+)>; rel="next"/.freeze
+      REPO_REGEX = /github\.com(?::|\/)(.*)\.git\z/.freeze
+      PER_PAGE = 100
+
+      def self.pull_request_files
+        new.pull_request_files
+      end
+
+      def initialize(env = ENV)
+        @env = env
+      end
+
+      def pull_request_files
+        files = []
+        next_uri = URI.join(BASE_URI, "repos/#{repo}/pulls/#{pull_request_number}/files?per_page=#{PER_PAGE}")
+
+        while next_uri
+          response = request(next_uri)
+          files.concat(JSON.parse(response.body))
+          next_uri = parse_next_uri(response)
+        end
+
+        files
+      end
+
+      private
+
+      def repo
+        Buildkite.env.repo[REPO_REGEX, 1]
+      end
+
+      def token
+        @env.fetch('GITHUB_API_TOKEN')
+      end
+
+      def pull_request_number
+        Buildkite.env.pull_request
+      end
+
+      def request(uri)
+        Net::HTTP.start(uri.hostname, uri.port, use_ssl: true) do |http|
+          request = Net::HTTP::Get.new(uri)
+          request['Authorization'] = "token #{token}"
+          request['Accept'] = ACCEPT_HEADER
+
+          http.request(request)
+        end
+      end
+
+      def parse_next_uri(response)
+        links = response[LINK_HEADER]
+        return unless links
+
+        matches = links.match(NEXT_LINK_REGEX)
+        URI.parse(matches[:uri]) if matches
+      end
+    end
+  end
+end

data/lib/buildkite/builder/loaders.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Buildkite
+  module Builder
+    module Loaders
+      autoload :Abstract, File.expand_path('loaders/abstract', __dir__)
+      autoload :Manifests, File.expand_path('loaders/manifests', __dir__)
+      autoload :Templates, File.expand_path('loaders/templates', __dir__)
+      autoload :Processors, File.expand_path('loaders/processors', __dir__)
+    end
+  end
+end

data/lib/buildkite/builder/loaders/abstract.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Buildkite
+  module Builder
+    module Loaders
+      class Abstract
+        attr_reader :assets
+        attr_reader :pipeline
+
+        def self.load(pipeline)
+          new(pipeline).assets
+        end
+
+        def initialize(pipeline)
+          @pipeline = pipeline
+          @assets = {}
+          load
+        end
+
+        private
+
+        def buildkite_path
+          Buildkite::Builder.root.join('.buildkite')
+        end
+
+        def pipeline_path
+          buildkite_path.join("pipelines/#{pipeline}")
+        end
+
+        def load
+          raise NotImplementedError
+        end
+
+        def add(name, asset)
+          @assets[name.to_s] = asset
+        end
+      end
+    end
+  end
+end

data/lib/buildkite/builder/loaders/manifests.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Buildkite
+  module Builder
+    module Loaders
+      class Manifests < Abstract
+        MANIFESTS_PATH = Pathname.new('manifests').freeze
+
+        def load
+          return unless manifests_path.directory?
+
+          manifests_path.children.map do |file|
+            add(file.basename, Manifest.new(Buildkite::Builder.root, file.readlines))
+          end
+        end
+
+        def manifests_path
+          pipeline_path.join(MANIFESTS_PATH)
+        end
+      end
+    end
+  end
+end

data/lib/buildkite/builder/loaders/processors.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require 'pathname'
+
+module Buildkite
+  module Builder
+    module Loaders
+      class Processors < Abstract
+        PROCESSORS_PATH = Pathname.new('processors').freeze
+
+        def load
+          load_processors_from_path(global_processors_path)
+          load_processors_from_path(pipeline_processors_path)
+        end
+
+        private
+
+        def load_processors_from_path(path)
+          return unless path.directory?
+
+          path.children.map do |file|
+            required_status = require(file)
+            add(file.basename, { required: required_status })
+          end
+        end
+
+        def global_processors_path
+          buildkite_path.join(PROCESSORS_PATH)
+        end
+
+        def pipeline_processors_path
+          pipeline_path.join(PROCESSORS_PATH)
+        end
+      end
+    end
+  end
+end

data/lib/buildkite/builder/loaders/templates.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Buildkite
+  module Builder
+    module Loaders
+      class Templates < Abstract
+        include Definition::Helper
+
+        TEMPLATES_PATH = Pathname.new('templates').freeze
+
+        def load
+          return unless templates_path.directory?
+
+          templates_path.children.sort.each do |file|
+            add(file.basename('.rb'), load_definition(file, Definition::Template))
+          end
+        end
+
+        def templates_path
+          pipeline_path.join(TEMPLATES_PATH)
+        end
+      end
+    end
+  end
+end

data/lib/buildkite/builder/logging_utils.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require 'benchmark'
+
+module Buildkite
+  module Builder
+    module LoggingUtils
+      def benchmark(output, &block)
+        time = Benchmark.realtime(&block)
+        output % [pluralize(time.round(2), 'second')]
+      end
+
+      def pluralize(count, singular, plural = nil)
+        if count == 1
+          "#{count} #{singular}"
+        elsif plural
+          "#{count} #{plural}"
+        else
+          "#{count} #{singular}s"
+        end
+      end
+    end
+  end
+end

data/lib/buildkite/builder/manifest.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+require 'digest/md5'
+require 'pathname'
+require 'set'
+
+module Buildkite
+  module Builder
+    class Manifest
+      autoload :Rule, File.expand_path('manifest/rule', __dir__)
+
+      class << self
+        def resolve(root, patterns)
+          new(root, Array(patterns)).modified?
+        end
+
+        def manifests
+          @manifests ||= {}
+        end
+
+        def [](name)
+          manifests[name.to_s]
+        end
+
+        def []=(name, manifest)
+          name = name.to_s
+          if manifests.key?(name)
+            raise ArgumentError, "manifest #{name} already exists"
+          end
+
+          manifests[name] = manifest
+        end
+      end
+
+      attr_reader :root
+
+      def initialize(root, patterns)
+        @root = Pathname.new(root)
+        @root = Buildkite::Builder.root.join(@root) unless @root.absolute?
+        @patterns = patterns.map(&:to_s)
+      end
+
+      def modified?
+        # DO NOT intersect FileResolver with manifest files. If the manifest is
+        # large, the operation can be expensive. It's always cheaper to loop
+        # through the changed files and compare them against the rules.
+        unless defined?(@modified)
+          @modified = FileResolver.resolve.modified_files.any? do |file|
+            file = Buildkite::Builder.root.join(file)
+            inclusion_rules.any? { |rule| rule.match?(file) } &&
+              exclusion_rules.none? { |rule| rule.match?(file) }
+          end
+        end
+
+        @modified
+      end
+
+      def files
+        @files ||= inclusion_rules.map(&:files).reduce(SortedSet.new, :merge) - exclusion_rules.map(&:files).reduce(SortedSet.new, :merge)
+      end
+
+      def digest
+        @digest ||= begin
+          digests = files.map { |file| Digest::MD5.file(Buildkite::Builder.root.join(file)).hexdigest }
+          Digest::MD5.hexdigest(digests.join)
+        end
+      end
+
+      private
+
+      def rules
+        @rules ||= @patterns.each_with_object([]) do |pattern, rules|
+          pattern = pattern.strip
+          unless pattern.match?(/\A(#|\z)/)
+            rules << Rule.new(root, pattern)
+          end
+        end
+      end
+
+      def inclusion_rules
+        @inclusion_rules ||= rules.reject(&:exclude)
+      end
+
+      def exclusion_rules
+        @exclusion_rules ||= rules.select(&:exclude)
+      end
+    end
+  end
+end