buildkite-builder 1.1.0 → 1.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 024ee5aff1b1a819fe74946becf1e3b1941c9f6e8ea86ba822a2d1411209e098
-  data.tar.gz: d9716d08bb4512fd0c9490ad26b1618a2b5e4f896693d5f5bca76024500fef70
+  metadata.gz: 9a3cd9ffee7742ded0fca17e0a11a214bd6383bd05c1c6a8101348d45b938bb1
+  data.tar.gz: f1b32964a3da0acdef53ba9febe034d1e77dc826ccb0b6722ffec1839f0f8126
 SHA512:
-  metadata.gz: 3d8130054bd4276b9d1c4505798985b9032ca406c126de026fa36bbb0fa919541c4c7d4cdd4c510b52e6554febb66c2e8a8bcc8e63aa54061fb727e6cbe8b752
-  data.tar.gz: 2492024dff09bfd31688c07ca712fed1b3191584f9b5917c4294775f687671decfcd3f0e0b20635bb3941c88b7097129e79700706a916b615ca11867e47a75e9
+  metadata.gz: d0ecf15f187b6644cf1e1af05c9bf425f17c8f7e2ecf4054b973e977a9feb202db8a63014286a9177b276687f935cf8e62ffc56ea7933503e0355fcc378db6fb
+  data.tar.gz: 71813fc583c20d14b99f487283f111ffc77bf9ad0f3adfbf97c87d17e13451eb4436f99a53d182952c5e37ee65c20376730d63cf1dec98ae777fbaf49bf8e3be

data/CHANGELOG.md

@@ -0,0 +1,13 @@
+## 1.4.1
+* Fix the Github API Builder to account for Buildkite having both `.git` and no file exention repository URIs (#33)
+
+## 1.4.0
+* Fix the `files` command. You now pass in the manifest with the `--manifest` CLI argument.
+
+## 1.3.1
+* Expose `data` from `StepContext` to `Step`
+
+## 1.3.0
+* Add ability for step to store data in step context
+* Move `upload` from `BuildKite::Builder::Commands::Run` to `BuildKite::Builder::Context`
+* Add ability to set custom artifacts in context and uplaod before `pipeile upload`

data/README.md

@@ -28,7 +28,7 @@ steps:
   - label: ":toolbox:"
     plugins:
       - docker#v3.7.0:
-          image: gusto/buildkite-builder:1.1.0
+          image: gusto/buildkite-builder:1.2.0
           propagate-environment: true
 ```
 

data/VERSION

@@ -1 +1 @@
-1.1.0
+1.4.1

data/lib/buildkite/builder.rb

@@ -14,7 +14,6 @@ module Buildkite
     autoload :Manifest, File.expand_path('builder/manifest', __dir__)
     autoload :Processors, File.expand_path('builder/processors', __dir__)
     autoload :Rainbow, File.expand_path('builder/rainbow', __dir__)
-    autoload :Runner, File.expand_path('builder/runner', __dir__)
 
     BUILDKITE_DIRECTORY_NAME = Pathname.new('.buildkite').freeze
 

data/lib/buildkite/builder/commands/abstract.rb

@@ -6,6 +6,18 @@ module Buildkite
   module Builder
     module Commands
       class Abstract
+        PIPELINES_DIRECTORY = 'pipelines'
+        POSSIBLE_PIPELINE_PATHS = [
+          File.join('.buildkite', Context::PIPELINE_DEFINITION_FILE),
+          File.join('buildkite', Context::PIPELINE_DEFINITION_FILE),
+          File.join(Context::PIPELINE_DEFINITION_FILE)
+        ].freeze
+        POSSIBLE_PIPELINES_PATHS = [
+          File.join('.buildkite', PIPELINES_DIRECTORY),
+          File.join('buildkite', PIPELINES_DIRECTORY),
+          File.join(PIPELINES_DIRECTORY)
+        ].freeze
+
         class << self
           attr_accessor :description
 
@@ -35,6 +47,8 @@ module Buildkite
           if options[:help]
             puts options[:help]
             return
+          elsif !pipeline_path
+            abort "Unable to find pipeline"
           end
 
           run
@@ -42,6 +56,10 @@ module Buildkite
 
         private
 
+        def pipeline_slug
+          ARGV.last || Buildkite.env&.pipeline_slug
+        end
+
         def command_name
           Commands::COMMANDS.key(self.class.name.split('::').last.to_sym)
         end
@@ -51,12 +69,39 @@ module Buildkite
           # Subclasses should override to parse options.
         end
 
-        def available_pipelines
-          @available_pipelines ||= pipelines_path.children.select(&:directory?).map { |dir| dir.basename.to_s }
+        def log
+          @log ||= begin
+            Logger.new($stdout).tap do |logger|
+              logger.formatter = proc do |_severity, _datetime, _progname, msg|
+                "#{msg}\n"
+              end
+            end
+          end
         end
 
-        def pipelines_path
-          Builder.root.join(Builder::BUILDKITE_DIRECTORY_NAME).join(Runner::PIPELINES_PATH)
+        def pipeline_path
+          @pipeline_path ||=
+            find_root_by_main_pipeline ||
+            find_root_by_multi_pipeline
+        end
+
+        def find_root_by_main_pipeline
+          POSSIBLE_PIPELINE_PATHS.map { |path| Builder.root.join(path) }.find(&:exist?)&.dirname
+        end
+
+        def find_root_by_multi_pipeline
+          pipelines_path = POSSIBLE_PIPELINES_PATHS.map { |path| Builder.root.join(path) }.find(&:directory?)
+          
+          if pipelines_path
+            if pipeline_slug
+              path = pipelines_path.join(pipeline_slug)
+              path if path.directory?
+            elsif pipelines_path.children.one?
+              pipelines_path.children.first
+            else
+              raise 'Your project has multiple pipelines, please specify one.'
+            end
+          end
         end
       end
     end

data/lib/buildkite/builder/commands/files.rb

@@ -9,14 +9,13 @@ module Buildkite
         self.description = 'Outputs files that match the specified manifest.'
 
         def run
-          pipeline, manifest = ARGV.first.to_s.split('/')
-          if !pipeline || !manifest
-            raise 'You must specify a pipeline and a manifest (eg "mypipeline/mymanifest")'
-          end
+          manifests = Loaders::Manifests.load(pipeline_path)
+          puts manifests[options[:manifest]].files.sort.join("\n")
+        end
 
-          manifests = Loaders::Manifests.load(pipeline)
-          manifests[manifest].files.each do |file|
-            puts file
+        def parse_options(opts)
+          opts.on('--manifest MANIFEST', 'The manifest to use') do |manifest|
+            options[:manifest] = manifest
           end
         end
       end

data/lib/buildkite/builder/commands/preview.rb

@@ -9,21 +9,7 @@ module Buildkite
         self.description = 'Outputs the pipeline YAML.'
 
         def run
-          pipeline = ARGV.last
-
-          if !pipeline && !root_pipeline?
-            if available_pipelines.one?
-              pipeline = available_pipelines.first
-            else
-              raise 'You must specify a pipeline'
-            end
-          end
-
-          puts Runner.new(pipeline: pipeline).run.to_yaml
-        end
-
-        def root_pipeline?
-          pipelines_path.join(Context::PIPELINE_DEFINITION_FILE).exist?
+          puts Context.build(pipeline_path).pipeline.to_yaml
         end
       end
     end

data/lib/buildkite/builder/commands/run.rb

@@ -5,21 +5,31 @@ module Buildkite
     module Commands
       class Run < Abstract
         private
+        include LoggingUtils
+        using Rainbow
 
         self.description = 'Builds and uploads the generated pipeline.'
 
         def run
+          relative_pipeline_path = pipeline_path.relative_path_from(Builder.root)
+
           # This entrypoint is for running on CI. It expects certain environment
-          # variables to be set.
-          options = {
-            upload: true
-          }
+          # variables to be set. It also uploads the pipeline to Buildkite.
+          log.info "#{'+++ ' if Buildkite.env}🧰 " + 'Buildkite Builder'.color(:springgreen) + " ─ #{relative_pipeline_path.to_s.yellow}"
+          Context.new(pipeline_path, logger: log).upload
+        end
 
-          if available_pipelines.include?(Buildkite.env.pipeline_slug)
-            options[:pipeline] = Buildkite.env.pipeline_slug
-          end
+        private
 
-          Builder::Runner.new(**options).run
+        def pipeline_path
+          pipeline_path_override || super
+        end
+
+        def pipeline_path_override
+          if ENV['BUILDKITE_BUILDER_PIPELINE_PATH']
+            path = Pathname.new(ENV['BUILDKITE_BUILDER_PIPELINE_PATH'])
+            path.absolute? ? path : Builder.root.join(path)
+          end
         end
       end
     end

data/lib/buildkite/builder/context.rb

@@ -1,13 +1,19 @@
+require 'logger'
+require 'tempfile'
+
 module Buildkite
   module Builder
     class Context
       include Definition::Helper
+      include LoggingUtils
+      using Rainbow
 
       PIPELINE_DEFINITION_FILE = Pathname.new('pipeline.rb').freeze
 
       attr_reader :logger
       attr_reader :root
       attr_reader :pipeline
+      attr_reader :artifacts
 
       def self.build(root, logger: nil)
         context = new(root, logger: logger)
@@ -18,22 +24,44 @@ module Buildkite
       def initialize(root, logger: nil)
         @root = root
         @logger = logger || Logger.new(File::NULL)
+        @artifacts = []
       end
 
       def build
-        unless @pipeline
-          @pipeline = Pipelines::Pipeline.new
-
-          load_manifests
-          load_templates
-          load_processors
-          load_pipeline
-          run_processors
+        results = benchmark("\nDone (%s)".color(:springgreen)) do
+          unless @pipeline
+            @pipeline = Pipelines::Pipeline.new
+
+            load_manifests
+            load_templates
+            load_processors
+            load_pipeline
+            run_processors
+          end
         end
+        logger.info(results)
 
         @pipeline
       end
 
+      def upload
+        build unless @pipeline
+
+        logger.info '+++ :paperclip: Uploading artifacts'
+        upload_artifacts
+
+        # Upload the pipeline.
+        Tempfile.create(['pipeline', '.yml']) do |file|
+          file.sync = true
+          file.write(pipeline.to_yaml)
+
+          logger.info '+++ :paperclip: Uploading pipeline.yml as artifact'
+          Buildkite::Pipelines::Command.artifact!(:upload, file.path)
+          logger.info '+++ :pipeline: Uploading pipeline'
+          Buildkite::Pipelines::Command.pipeline!(:upload, file.path)
+        end
+      end
+
       private
 
       def load_manifests
@@ -58,6 +86,16 @@ module Buildkite
         end
       end
 
+      def upload_artifacts
+        return if artifacts.empty?
+
+        artifacts.each do |path|
+          if File.exist?(path)
+            Buildkite::Pipelines::Command.artifact!(:upload, path)
+          end
+        end
+      end
+
       def load_pipeline
         pipeline.instance_eval(&pipeline_definition)
       end

data/lib/buildkite/builder/github.rb

@@ -11,7 +11,7 @@ module Buildkite
       ACCEPT_HEADER = 'application/vnd.github.v3+json'
       LINK_HEADER = 'link'
       NEXT_LINK_REGEX = /<(?<uri>.+)>; rel="next"/.freeze
-      REPO_REGEX = /github\.com(?::|\/)(.*)\.git\z/.freeze
+      REPO_REGEX = /github\.com(?::|\/)(.*?)(?:\.git)?\z/.freeze
       PER_PAGE = 100
 
       def self.pull_request_files

data/lib/buildkite/pipelines/step_context.rb

@@ -5,10 +5,12 @@ module Buildkite
     class StepContext
       attr_reader :step
       attr_reader :args
+      attr_reader :data
 
       def initialize(step, **args)
         @step = step
         @args = args
+        @data = {}
       end
 
       def pipeline

data/lib/buildkite/pipelines/steps/abstract.rb

@@ -1,11 +1,16 @@
 # frozen_string_literal: true
 
+require "forwardable"
+
 module Buildkite
   module Pipelines
     module Steps
       class Abstract
+        extend Forwardable
         include Attributes
 
+        def_delegator :@context, :data
+
         attr_reader :pipeline
         attr_reader :template
 
@@ -16,10 +21,10 @@ module Buildkite
         def initialize(pipeline, template = nil, **args, &block)
           @pipeline = pipeline
           @template = template
-          context = StepContext.new(self, **args)
+          @context = StepContext.new(self, **args)
 
-          instance_exec(context, &template) if template
-          instance_exec(context, &block) if block_given?
+          instance_exec(@context, &template) if template
+          instance_exec(@context, &block) if block_given?
         end
       end
     end

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: buildkite-builder
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.4.1
 platform: ruby
 authors:
 - Ngan Pham
 - Andrew Lee
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-02-16 00:00:00.000000000 Z
+date: 2021-04-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sorted_set
@@ -135,6 +135,7 @@ files:
 - lib/buildkite/builder/commands/run.rb
 - lib/buildkite/builder/context.rb
 - lib/buildkite/builder/definition.rb
+- lib/buildkite/builder/dsl/pipeline.rb
 - lib/buildkite/builder/file_resolver.rb
 - lib/buildkite/builder/github.rb
 - lib/buildkite/builder/loaders.rb
@@ -148,7 +149,6 @@ files:
 - lib/buildkite/builder/processors.rb
 - lib/buildkite/builder/processors/abstract.rb
 - lib/buildkite/builder/rainbow.rb
-- lib/buildkite/builder/runner.rb
 - lib/buildkite/env.rb
 - lib/buildkite/pipelines.rb
 - lib/buildkite/pipelines/api.rb
@@ -172,6 +172,7 @@ files:
 - lib/buildkite/pipelines/steps/abstract.rb
 - lib/buildkite/pipelines/steps/block.rb
 - lib/buildkite/pipelines/steps/command.rb
+- lib/buildkite/pipelines/steps/group.rb
 - lib/buildkite/pipelines/steps/input.rb
 - lib/buildkite/pipelines/steps/skip.rb
 - lib/buildkite/pipelines/steps/trigger.rb
@@ -184,7 +185,7 @@ metadata:
   source_code_uri: https://github.com/Gusto/buildkite-builder
   changelog_uri: https://github.com/Gusto/buildkite-builder/blob/master/CHANGELOG.md
   bug_tracker_uri: https://github.com/Gusto/buildkite-builder/issues
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -200,7 +201,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.2.2
-signing_key: 
+signing_key:
 specification_version: 4
 summary: A gem for programmatically creating Buildkite pipelines.
 test_files: []

data/lib/buildkite/builder/runner.rb

@@ -1,77 +0,0 @@
-# frozen_string_literal: true
-
-require 'bundler'
-require 'pathname'
-require 'tempfile'
-require 'logger'
-
-module Buildkite
-  module Builder
-    class Runner
-      include LoggingUtils
-      using Rainbow
-
-      PIPELINES_PATH = Pathname.new('pipelines').freeze
-
-      attr_reader :options
-
-      def initialize(**options)
-        @options = options
-      end
-
-      def run
-        log.info "#{'+++ ' if Buildkite.env}🧰 " + 'Buildkite Builder'.color(:springgreen) + " ─ #{@options[:pipeline].yellow}"
-        context = Context.new(root, logger: log)
-
-        results = benchmark("\nDone (%s)".color(:springgreen)) do
-          context.build
-        end
-        log.info(results)
-
-        if options[:upload]
-          upload(context.pipeline)
-        end
-
-        # Always return the pipeline.
-        context.pipeline
-      end
-
-      private
-
-      def log
-        @log ||= begin
-          Logger.new($stdout).tap do |logger|
-            logger.formatter = proc do |_severity, _datetime, _progname, msg|
-              "#{msg}\n"
-            end
-          end
-        end
-      end
-
-      def upload(pipeline)
-        Tempfile.create(['pipeline', '.yml']) do |file|
-          file.sync = true
-          file.write(pipeline.to_yaml)
-
-          log.info '+++ :paperclip: Uploading artifact'
-          Buildkite::Pipelines::Command.artifact!(:upload, file.path)
-          log.info '+++ :pipeline: Uploading pipeline'
-          Buildkite::Pipelines::Command.pipeline!(:upload, file.path)
-        end
-      end
-
-      def root
-        @root ||= begin
-          path = Builder.root.join(Builder::BUILDKITE_DIRECTORY_NAME)
-          if options[:pipeline]
-            pipeline_path = path.join(PIPELINES_PATH).join(options[:pipeline])
-            if pipeline_path.directory?
-              path = pipeline_path
-            end
-          end
-          path
-        end
-      end
-    end
-  end
-end