buildkite-builder 2.4.0 → 3.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4c4c2623f11f578f2839830abd97f02de9189aca48838b51af749a2ed63b5d43
-  data.tar.gz: 80ad9118885ca08b62b9d849f5bde06d60ab8ed13095921a4bf017c1b23754f1
+  metadata.gz: bef719c816c5d64756bbcd041ca2a3fabe8fe59bff47cc154948207ce64a672f
+  data.tar.gz: f25762b1422e540f395526df9d6082e1d5908ea0b9b290bf025a365b56c317ff
 SHA512:
-  metadata.gz: faf4b6d95c5ce06432dec04ddc53df0b6ca4a0e8e0986fe5798b02678513847f1bc1dc6cfa4e3a81d603d0aad5db55e6b7f7ece5f21801fb92e52ae8e6895d68
-  data.tar.gz: 8c134596ebac502f43c0da347dbe36f5db527534b50781623cd3ea0b2e69a8ff6ecb9bfc1cd9ec697b616bcfe7c3804bc728a9c6a70d5c71aecfc4d772fa9931
+  metadata.gz: c00f3fba91b34cf907c99ff839612270ef8aecb754c988af52226e979d5e743a508963d1d06d9957b529a2e3ba47cf6ba0951ec33290b8581dc9937b4178033a
+  data.tar.gz: 6e62bff8ecdb50b72bdb3afb8fd6fa1385f17ef1852b607f5273570cab1d396ee65d8908a78604d48ccb7a564ad4cb0f9dbff6b94b7a016cfd9ee881e1d4098a

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+## 3.1.0
+* Add subpipeline support to save triggered pipeline's YML definition beforehand to artifacts and pass down the file to an ENV for pipeline setup.
+
+## 3.0.0
+* Remove manifest features to prevent Github API dependency and simplify the gem to focus on Buildkite features.
+
+## 2.4.1
+* Fix pipeline upload as artifact logic.
+
 ## 2.4.0
 * Upload custom pipeline artifacts in a single command.
 * Only upload the pipeline as an artifact when the pipeline upload fails.

data/README.md

@@ -48,7 +48,7 @@ At its core, BKB is really just a YAML builder. This tool allows you to scale yo
 - Perform pre-build code/diff analysis to determine whether or not to to add a step to the pipeline.
 - Reorder pipeline steps dynamically.
 - Augment your pipeline steps with BKB processors.
-  
+
 ### Pipeline Files
 
 Your repo can contain as many pipeline definitions as you'd like. By convention, pipeline file structure are as such:
@@ -74,9 +74,9 @@ Buildkite::Builder.pipeline do
     label "Rspec", emoji: :rspec
     command "bundle exec rspec"
   end
-  
+
   wait
-  
+
   trigger do
     trigger "deploy-pipeline"
   end
@@ -127,7 +127,7 @@ You can then include the template into the the pipeline once or as many time as
 ```ruby
 Buildkite::Builder.pipeline do
   command(:rspec)
-  
+
   # Reuse and agument templates on the fly.
   command(:rspec) do
     label "Run RSpec again!"
@@ -135,6 +135,36 @@ Buildkite::Builder.pipeline do
 end
 ```
 
+### Subpipeline
+
+While triggering another pipeline, you can predefine subpipeline's steps using `pipeline(NAME_OF_SUBPIPELINE)` in the main pipeline's `pipeline.rb` file and share with main pipeline's plugins and templates definition.
+
+`.buildkite/pipelines/pipeline-triggerer/pipeline.rb`
+
+```ruby
+Buildkite::Builder.pipeline do
+  pipeline('rspec-pipeline') do
+    command do
+      label "Run RSpec in separate pipeline"
+    end
+  end
+end
+```
+
+Inside your Buildkite pipeline setup, you can do the following:
+
+In `https://buildkite.com/your-org/rspec-pipeline/steps`
+
+```yaml
+steps:
+  - label: ":pipeline:"
+    commands:
+      - buildkite-agent artifact download $BKB_SUBPIPELINE_FILE . --build $BUILDKITE_TRIGGERED_FROM_BUILD_ID
+      - buildkite-agent pipeline upload $BKB_SUBPIPELINE_FILE
+```
+
+This will upload the pregenerated `pipeline.yml` to `rspec-pipeline`.
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/VERSION

@@ -1 +1 @@
-2.4.0
+3.1.0

data/lib/buildkite/builder/commands.rb

@@ -5,7 +5,6 @@ module Buildkite
     module Commands
       using Rainbow
       COMMANDS = {
-        'files' => :Files,
         'preview' => :Preview,
         'run' => :Run
       }.freeze

data/lib/buildkite/builder/data.rb

@@ -9,7 +9,7 @@ module Buildkite
         @data.each_with_object({}) do |(key, value), hash|
           value = value.respond_to?(:to_definition) ? value.to_definition : value
 
-          next if value.empty?
+          next if value.nil? || value.empty?
 
           hash[key] = value
         end

data/lib/buildkite/builder/extensions/sub_pipelines.rb

@@ -0,0 +1,96 @@
+require 'securerandom'
+
+module Buildkite
+  module Builder
+    module Extensions
+      class SubPipelines < Extension
+        class Pipeline
+          include Buildkite::Pipelines::Attributes
+
+          attr_reader :data, :name
+
+          attribute :depends_on, append: true
+          attribute :key
+
+          def self.to_sym
+            name.split('::').last.downcase.to_sym
+          end
+
+          def initialize(name, steps, &block)
+            @name = name
+            @data = Data.new
+            @data.steps = StepCollection.new(
+              steps.templates,
+              steps.plugins
+            )
+            @data.notify = []
+            @data.env = {}
+
+            @dsl = Dsl.new(self)
+            @dsl.extend(Extensions::Steps)
+            @dsl.extend(Extensions::Notify)
+            @dsl.extend(Extensions::Env)
+            instance_eval(&block) if block_given?
+            self
+          end
+
+          def to_h
+            attributes = super
+            attributes.merge(data.to_definition)
+          end
+
+          def method_missing(method_name, *args, **kwargs, &_block)
+            @dsl.public_send(method_name, *args, **kwargs, &_block)
+          end
+
+          def pipeline_yml
+            @pipeline_yml ||= "tmp/buildkite-builder/#{SecureRandom.urlsafe_base64}.yml"
+          end
+        end
+
+        def prepare
+          context.data.pipelines = PipelineCollection.new(context.artifacts)
+        end
+
+        dsl do
+          def pipeline(name, template = nil, &block)
+            raise "Subpipeline must have a name" if name.empty?
+            raise "Subpipeline does not allow nested in another Subpipeline" if context.is_a?(Buildkite::Builder::Extensions::SubPipelines::Pipeline)
+            sub_pipeline = Buildkite::Builder::Extensions::SubPipelines::Pipeline.new(name, context.data.steps, &block)
+
+            context.data.pipelines.add(sub_pipeline)
+
+            if template
+              # Use predefined template
+              step = context.data.steps.add(Pipelines::Steps::Trigger, template)
+
+              if step.build.nil?
+                step.build(env: { BKB_SUBPIPELINE_FILE: sub_pipeline.pipeline_yml })
+              else
+                step.build[:env].merge!(BKB_SUBPIPELINE_FILE: sub_pipeline.pipeline_yml)
+              end
+            else
+              # Generic trigger step
+              context.data.steps.add(Pipelines::Steps::Trigger, key: "subpipeline_#{name}_#{context.data.pipelines.count}") do |context|
+                key context[:key]
+                label name.capitalize
+                trigger name
+                build(
+                  message: '${BUILDKITE_MESSAGE}',
+                  commit: '${BUILDKITE_COMMIT}',
+                  branch: '${BUILDKITE_BRANCH}',
+                  env: {
+                    BUILDKITE_PULL_REQUEST: '${BUILDKITE_PULL_REQUEST}',
+                    BUILDKITE_PULL_REQUEST_BASE_BRANCH: '${BUILDKITE_PULL_REQUEST_BASE_BRANCH}',
+                    BUILDKITE_PULL_REQUEST_REPO: '${BUILDKITE_PULL_REQUEST_REPO}',
+                    BKB_SUBPIPELINE_FILE: sub_pipeline.pipeline_yml
+                  }
+                )
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/buildkite/builder/extensions.rb

@@ -6,6 +6,7 @@ module Buildkite
       autoload :Env, File.expand_path('extensions/env', __dir__)
       autoload :Lib, File.expand_path('extensions/lib', __dir__)
       autoload :Notify, File.expand_path('extensions/notify', __dir__)
+      autoload :SubPipelines, File.expand_path('extensions/sub_pipelines', __dir__)
       autoload :Steps, File.expand_path('extensions/steps', __dir__)
       autoload :Use, File.expand_path('extensions/use', __dir__)
     end

data/lib/buildkite/builder/loaders.rb

@@ -4,7 +4,6 @@ module Buildkite
   module Builder
     module Loaders
       autoload :Abstract, File.expand_path('loaders/abstract', __dir__)
-      autoload :Manifests, File.expand_path('loaders/manifests', __dir__)
       autoload :Templates, File.expand_path('loaders/templates', __dir__)
       autoload :Extensions, File.expand_path('loaders/extensions', __dir__)
     end

data/lib/buildkite/builder/pipeline.rb

@@ -37,7 +37,7 @@ module Buildkite
         use(Extensions::Env)
         use(Extensions::Notify)
         use(Extensions::Steps)
-        load_manifests
+        use(Extensions::SubPipelines)
       end
 
       def upload
@@ -52,9 +52,10 @@ module Buildkite
           file.write(contents)
 
           logger.info "+++ :pipeline: Uploading pipeline"
-          unless Buildkite::Pipelines::Command.pipeline!(:upload, file.path)
+          unless Buildkite::Pipelines::Command.pipeline(:upload, file.path)
             logger.info "Pipeline upload failed, saving as artifact…"
             Buildkite::Pipelines::Command.artifact!(:upload, file.path)
+            abort
           end
           logger.info "+++ :toolbox: Setting job meta-data to #{Buildkite.env.job_id.color(:yellow)}"
           Buildkite::Pipelines::Command.meta_data!(:set, Builder::META_DATA.fetch(:job), Buildkite.env.job_id)
@@ -81,12 +82,6 @@ module Buildkite
 
       attr_reader :extensions
 
-      def load_manifests
-        Loaders::Manifests.load(root).each do |name, asset|
-          Manifest[name] = asset
-        end
-      end
-
       def upload_artifacts
         return if artifacts.empty?
 

data/lib/buildkite/builder/pipeline_collection.rb

@@ -0,0 +1,39 @@
+require "forwardable"
+
+module Buildkite
+  module Builder
+    class PipelineCollection
+      extend Forwardable
+
+      attr_reader :pipelines
+
+      def_delegator :@pipelines, :count
+
+      def initialize(artifacts)
+        @artifacts = artifacts
+        @pipelines = []
+      end
+
+      def add(pipeline)
+        unless pipeline.is_a?(Buildkite::Builder::Extensions::SubPipelines::Pipeline)
+          raise "`#{pipeline}` must be a Buildkite::Builder::Extensions::SubPipelines::Pipeline"
+        end
+
+        pipelines << pipeline
+      end
+
+      def to_definition
+        # Instead of generates pipeline.yml, subpipelines save generated file to artifacts
+        pipelines.each do |pipeline|
+          file = Pathname.new(pipeline.pipeline_yml)
+          file.dirname.mkpath
+          file.write(YAML.dump(Pipelines::Helpers.sanitize(pipeline.to_h)))
+
+          @artifacts << file
+        end
+
+        nil
+      end
+    end
+  end
+end

data/lib/buildkite/builder.rb

@@ -13,16 +13,14 @@ module Buildkite
     autoload :Extension, File.expand_path('builder/extension', __dir__)
     autoload :ExtensionManager, File.expand_path('builder/extension_manager', __dir__)
     autoload :Extensions, File.expand_path('builder/extensions', __dir__)
-    autoload :FileResolver, File.expand_path('builder/file_resolver', __dir__)
-    autoload :Github, File.expand_path('builder/github', __dir__)
     autoload :Loaders, File.expand_path('builder/loaders', __dir__)
     autoload :LoggingUtils, File.expand_path('builder/logging_utils', __dir__)
-    autoload :Manifest, File.expand_path('builder/manifest', __dir__)
     autoload :Processors, File.expand_path('builder/processors', __dir__)
     autoload :Rainbow, File.expand_path('builder/rainbow', __dir__)
     autoload :Plugin, File.expand_path('builder/plugin', __dir__)
     autoload :PluginCollection, File.expand_path('builder/plugin_collection', __dir__)
     autoload :StepCollection, File.expand_path('builder/step_collection', __dir__)
+    autoload :PipelineCollection, File.expand_path('builder/pipeline_collection', __dir__)
     autoload :TemplateManager, File.expand_path('builder/template_manager', __dir__)
     autoload :PluginManager, File.expand_path('builder/plugin_manager', __dir__)
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: buildkite-builder
 version: !ruby/object:Gem::Version
-  version: 2.4.0
+  version: 3.1.0
 platform: ruby
 authors:
 - Ngan Pham
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-01-08 00:00:00.000000000 Z
+date: 2022-06-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rainbow
@@ -116,7 +116,6 @@ files:
 - lib/buildkite/builder.rb
 - lib/buildkite/builder/commands.rb
 - lib/buildkite/builder/commands/abstract.rb
-- lib/buildkite/builder/commands/files.rb
 - lib/buildkite/builder/commands/preview.rb
 - lib/buildkite/builder/commands/run.rb
 - lib/buildkite/builder/data.rb
@@ -129,19 +128,16 @@ files:
 - lib/buildkite/builder/extensions/lib.rb
 - lib/buildkite/builder/extensions/notify.rb
 - lib/buildkite/builder/extensions/steps.rb
+- lib/buildkite/builder/extensions/sub_pipelines.rb
 - lib/buildkite/builder/extensions/use.rb
-- lib/buildkite/builder/file_resolver.rb
-- lib/buildkite/builder/github.rb
 - lib/buildkite/builder/group.rb
 - lib/buildkite/builder/loaders.rb
 - lib/buildkite/builder/loaders/abstract.rb
 - lib/buildkite/builder/loaders/extensions.rb
-- lib/buildkite/builder/loaders/manifests.rb
 - lib/buildkite/builder/loaders/templates.rb
 - lib/buildkite/builder/logging_utils.rb
-- lib/buildkite/builder/manifest.rb
-- lib/buildkite/builder/manifest/rule.rb
 - lib/buildkite/builder/pipeline.rb
+- lib/buildkite/builder/pipeline_collection.rb
 - lib/buildkite/builder/plugin.rb
 - lib/buildkite/builder/plugin_collection.rb
 - lib/buildkite/builder/plugin_manager.rb

data/lib/buildkite/builder/commands/files.rb

@@ -1,24 +0,0 @@
-# frozen_string_literal: true
-
-module Buildkite
-  module Builder
-    module Commands
-      class Files < Abstract
-        private
-
-        self.description = 'Outputs files that match the specified manifest.'
-
-        def run
-          manifests = Loaders::Manifests.load(pipeline_path)
-          puts manifests[options[:manifest]].files.sort.join("\n")
-        end
-
-        def parse_options(opts)
-          opts.on('--manifest MANIFEST', 'The manifest to use') do |manifest|
-            options[:manifest] = manifest
-          end
-        end
-      end
-    end
-  end
-end

data/lib/buildkite/builder/file_resolver.rb

@@ -1,59 +0,0 @@
-# frozen_string_literal: true
-
-require 'open3'
-require 'set'
-
-module Buildkite
-  module Builder
-    class FileResolver
-      @cache = true
-
-      attr_reader :modified_files
-
-      class << self
-        attr_accessor :cache
-
-        def resolve(reset = false)
-          @resolve = nil if !cache || reset
-          @resolve ||= new
-        end
-      end
-
-      def initialize
-        @modified_files = Set.new(pull_request? ? files_from_pull_request.sort! : files_from_git.sort!)
-      end
-
-      private
-
-      def files_from_pull_request
-        Github.pull_request_files.map { |f| f.fetch('filename') }
-      end
-
-      def files_from_git
-        if Buildkite.env
-          changed_files = command("git diff-tree --no-commit-id --name-only -r #{Buildkite.env.commit}")
-        else
-          default_branch = command('git symbolic-ref refs/remotes/origin/HEAD').strip
-          changed_files = command("git diff --name-only #{default_branch}")
-          changed_files << command('git diff --name-only')
-        end
-
-        changed_files.split.uniq.sort
-      end
-
-      def pull_request?
-        Buildkite.env&.pull_request
-      end
-
-      def command(cmd)
-        output, status = Open3.capture2(*cmd.split)
-
-        if status.success?
-          output
-        else
-          raise "Command failed (exit #{status.exitstatus}): #{cmd}"
-        end
-      end
-    end
-  end
-end

data/lib/buildkite/builder/github.rb

@@ -1,71 +0,0 @@
-# frozen_string_literal: true
-
-require 'json'
-require 'net/http'
-require 'uri'
-
-module Buildkite
-  module Builder
-    class Github
-      BASE_URI = URI('https://api.github.com').freeze
-      ACCEPT_HEADER = 'application/vnd.github.v3+json'
-      LINK_HEADER = 'link'
-      NEXT_LINK_REGEX = /<(?<uri>.+)>; rel="next"/.freeze
-      REPO_REGEX = /github\.com(?::|\/)(.*?)(?:\.git)?\z/.freeze
-      PER_PAGE = 100
-
-      def self.pull_request_files
-        new.pull_request_files
-      end
-
-      def initialize(env = ENV)
-        @env = env
-      end
-
-      def pull_request_files
-        files = []
-        next_uri = URI.join(BASE_URI, "repos/#{repo}/pulls/#{pull_request_number}/files?per_page=#{PER_PAGE}")
-
-        while next_uri
-          response = request(next_uri)
-          files.concat(JSON.parse(response.body))
-          next_uri = parse_next_uri(response)
-        end
-
-        files
-      end
-
-      private
-
-      def repo
-        Buildkite.env.repo[REPO_REGEX, 1]
-      end
-
-      def token
-        @env.fetch('GITHUB_API_TOKEN')
-      end
-
-      def pull_request_number
-        Buildkite.env.pull_request
-      end
-
-      def request(uri)
-        Net::HTTP.start(uri.hostname, uri.port, use_ssl: true) do |http|
-          request = Net::HTTP::Get.new(uri)
-          request['Authorization'] = "token #{token}"
-          request['Accept'] = ACCEPT_HEADER
-
-          http.request(request)
-        end
-      end
-
-      def parse_next_uri(response)
-        links = response[LINK_HEADER]
-        return unless links
-
-        matches = links.match(NEXT_LINK_REGEX)
-        URI.parse(matches[:uri]) if matches
-      end
-    end
-  end
-end

data/lib/buildkite/builder/loaders/manifests.rb

@@ -1,23 +0,0 @@
-# frozen_string_literal: true
-
-module Buildkite
-  module Builder
-    module Loaders
-      class Manifests < Abstract
-        MANIFESTS_PATH = Pathname.new('manifests').freeze
-
-        def load
-          return unless manifests_path.directory?
-
-          manifests_path.children.map do |file|
-            add(file.basename, Manifest.new(Buildkite::Builder.root, file.readlines))
-          end
-        end
-
-        def manifests_path
-          root.join(MANIFESTS_PATH)
-        end
-      end
-    end
-  end
-end

data/lib/buildkite/builder/manifest/rule.rb

@@ -1,51 +0,0 @@
-# frozen_string_literal: true
-
-require 'set'
-require 'pathname'
-
-module Buildkite
-  module Builder
-    class Manifest::Rule
-      GLOB_OPTIONS = File::FNM_PATHNAME | File::FNM_DOTMATCH | File::FNM_EXTGLOB
-
-      attr_reader :exclude
-      attr_reader :glob
-
-      def initialize(root, pattern)
-        @root = Pathname.new(root)
-        @exclude = false
-        @glob = @root
-
-        if pattern[0] == '!'
-          @exclude = true
-          pattern = pattern[1..-1]
-        end
-
-        if pattern.start_with?('/')
-          pattern = pattern[1..-1]
-        else
-          @glob = @glob.join('**')
-        end
-
-        @glob = @glob.join(pattern).to_s
-      end
-
-      def files
-        @files ||= begin
-          matched = Dir.glob(glob, GLOB_OPTIONS)
-          matched.map! { |file| Pathname.new(file) }
-          matched.reject!(&:directory?)
-          matched.map! { |file| file.relative_path_from(Builder.root) }
-          Set.new(matched.sort!)
-        end
-      end
-
-      def match?(file)
-        file = Pathname.new(file)
-        file = @root.join(file) unless file.absolute?
-
-        File.fnmatch?(glob, file.to_s, GLOB_OPTIONS)
-      end
-    end
-  end
-end

data/lib/buildkite/builder/manifest.rb

@@ -1,88 +0,0 @@
-# frozen_string_literal: true
-
-require 'digest/md5'
-require 'pathname'
-
-module Buildkite
-  module Builder
-    class Manifest
-      autoload :Rule, File.expand_path('manifest/rule', __dir__)
-
-      class << self
-        def resolve(root, patterns)
-          new(root, Array(patterns)).modified?
-        end
-
-        def manifests
-          @manifests ||= {}
-        end
-
-        def [](name)
-          manifests[name.to_s]
-        end
-
-        def []=(name, manifest)
-          name = name.to_s
-          if manifests.key?(name)
-            raise ArgumentError, "manifest #{name} already exists"
-          end
-
-          manifests[name] = manifest
-        end
-      end
-
-      attr_reader :root
-
-      def initialize(root, patterns)
-        @root = Pathname.new(root)
-        @root = Buildkite::Builder.root.join(@root) unless @root.absolute?
-        @patterns = patterns.map(&:to_s)
-      end
-
-      def modified?
-        # DO NOT intersect FileResolver with manifest files. If the manifest is
-        # large, the operation can be expensive. It's always cheaper to loop
-        # through the changed files and compare them against the rules.
-        unless defined?(@modified)
-          @modified = FileResolver.resolve.modified_files.any? do |file|
-            file = Buildkite::Builder.root.join(file)
-            inclusion_rules.any? { |rule| rule.match?(file) } &&
-              exclusion_rules.none? { |rule| rule.match?(file) }
-          end
-        end
-
-        @modified
-      end
-
-      def files
-        @files ||= (inclusion_rules.map(&:files).reduce(Set.new, :merge) - exclusion_rules.map(&:files).reduce(Set.new, :merge)).sort.to_set
-      end
-
-      def digest
-        @digest ||= begin
-          digests = files.map { |file| Digest::MD5.file(Buildkite::Builder.root.join(file)).hexdigest }
-          Digest::MD5.hexdigest(digests.join)
-        end
-      end
-
-      private
-
-      def rules
-        @rules ||= @patterns.each_with_object([]) do |pattern, rules|
-          pattern = pattern.strip
-          unless pattern.match?(/\A(#|\z)/)
-            rules << Rule.new(root, pattern)
-          end
-        end
-      end
-
-      def inclusion_rules
-        @inclusion_rules ||= rules.reject(&:exclude)
-      end
-
-      def exclusion_rules
-        @exclusion_rules ||= rules.select(&:exclude)
-      end
-    end
-  end
-end