puppet_docker_tools 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 279a8c6d6a95b7d1c5f3f0dc4c045f2b67e363fd
-  data.tar.gz: fa7be0251b3e84ae7fe7be4942c795a578f116e9
+  metadata.gz: 83418018be00481fd020d44eb91dc3e91cbe64d9
+  data.tar.gz: 981bd7c153e5cfec42bf8f97bc6e791278d76dd8
 SHA512:
-  metadata.gz: d1156f61257a15d2bb792ab555d146e6041d0782d072a2467a1766174562f9a10af9486d2b9d82f350ae45f18053467937cfa444664c70ebc0cfaec03c3d51b6
-  data.tar.gz: 5f2e135989bc479afca3e1116c45fff26cb6a43e21c2eacc904a1adf454ae627fb6e67e61d83a248d74fbea348c305f8933e090a2115e14edf2bfe21847629af
+  metadata.gz: 27d23f46ffe35d28e41e5fe9c66493d35eadcfde6adfc97b55ad01159f517b6052e769a99f3da9428b0405a5b533d972856160a5bb55c26902d2017eae594d26
+  data.tar.gz: 18f9ec6d818748ec65ebb58142b9b65832259d679211f8d2a0a0761548b73be7e9c903f5a40a92945cfde5c8f5402e5eb56e4e2f04c99041bf2cc228c4a603e4

data/README.md

@@ -15,11 +15,11 @@ $ puppet-docker help
 Utilities for building and releasing Puppet docker images.
 
 Usage:
-  puppet-docker build [DIRECTORY] [--dockerfile=<dockerfile>] [--repository=<repo>] [--namespace=<namespace>] [--no-cache] [--version=<version] [--build-arg=<buildarg> ...]
+  puppet-docker build [DIRECTORY] [--dockerfile=<dockerfile>] [--repository=<repo>] [--namespace=<namespace>] [--no-cache] [--version=<version] [--build-arg=<buildarg> ...] [--no-latest]
   puppet-docker lint [DIRECTORY] [--dockerfile=<dockerfile>]
   puppet-docker local-lint [DIRECTORY] [--dockerfile=<dockerfile>]
   puppet-docker pull [IMAGE] [--repository=<repo>]
-  puppet-docker push [DIRECTORY] [--dockerfile=<dockerfile>] [--repository=<repo>] [--namespace=<namespace>]
+  puppet-docker push [DIRECTORY] [--dockerfile=<dockerfile>] [--repository=<repo>] [--namespace=<namespace>] [--no-latest]
   puppet-docker rev-labels [DIRECTORY] [--dockerfile=<dockerfile>] [--namespace=<namespace>]
   puppet-docker spec [DIRECTORY]
   puppet-docker test [DIRECTORY] [--dockerfile=<dockerfile>]
@@ -40,6 +40,8 @@ Options:
   --version=<version>        Version to build. This field will be used to determine the label and will be passed as the version build arg.
                              **NOTE** `--build-arg version='<version>'` overrides `--version <version>`
   --build-arg=<buildarg>     Build arg to pass to container build, can be passed multiple times.
+  --no-latest                Do not include the 'latest' tag when building and shipping images. By default, the 'latest' tag is built and
+                             shipped with the versioned tag.
 ```
 
 ### `puppet-docker build`

data/bin/puppet-docker

@@ -7,11 +7,11 @@ doc = <<DOC
 Utilities for building and releasing Puppet docker images.
 
 Usage:
-  puppet-docker build [DIRECTORY] [--dockerfile=<dockerfile>] [--repository=<repo>] [--namespace=<namespace>] [--no-cache] [--version=<version] [--build-arg=<buildarg> ...]
+  puppet-docker build [DIRECTORY] [--dockerfile=<dockerfile>] [--repository=<repo>] [--namespace=<namespace>] [--no-cache] [--version=<version] [--build-arg=<buildarg> ...] [--no-latest]
   puppet-docker lint [DIRECTORY] [--dockerfile=<dockerfile>]
   puppet-docker local-lint [DIRECTORY] [--dockerfile=<dockerfile>]
   puppet-docker pull [IMAGE] [--repository=<repo>]
-  puppet-docker push [DIRECTORY] [--dockerfile=<dockerfile>] [--repository=<repo>] [--namespace=<namespace>]
+  puppet-docker push [DIRECTORY] [--dockerfile=<dockerfile>] [--repository=<repo>] [--namespace=<namespace>] [--no-latest]
   puppet-docker rev-labels [DIRECTORY] [--dockerfile=<dockerfile>] [--namespace=<namespace>]
   puppet-docker spec [DIRECTORY]
   puppet-docker test [DIRECTORY] [--dockerfile=<dockerfile>]
@@ -32,6 +32,8 @@ Options:
   --version=<version>        Version to build. This field will be used to determine the label and will be passed as the version build arg.
                              **NOTE** `--build-arg version='<version>'` overrides `--version <version>`
   --build-arg=<buildarg>     Build arg to pass to container build, can be passed multiple times.
+  --no-latest                Do not include the 'latest' tag when building and shipping images. By default, the 'latest' tag is built and
+                             shipped with the versioned tag.
 DOC
 
 begin
@@ -49,6 +51,7 @@ defaults = {
   '--dockerfile' => 'Dockerfile',
   '--version'    => nil,
   '--build-arg'  => [],
+  '--no-latest'  => false,
 }
 
 options.merge!(defaults) do |key, option, default|
@@ -73,14 +76,19 @@ begin
   else
     command_runner = PuppetDockerTools::Runner.new(directory: options['DIRECTORY'], repository: options['--repository'], namespace: options['--namespace'], dockerfile: options['--dockerfile'])
 
+    latest = true
+    if options['--no-latest']
+      latest = false
+    end
+
     if options['build']
-      command_runner.build(no_cache: options['--no-cache'], version: options['--version'], build_args: options['--build-arg'])
+      command_runner.build(no_cache: options['--no-cache'], version: options['--version'], build_args: options['--build-arg'], latest: latest)
     elsif options['lint']
       command_runner.lint
     elsif options['local-lint']
       command_runner.local_lint
     elsif options['push']
-      command_runner.push
+      command_runner.push(latest: latest)
     elsif options['rev-labels']
       command_runner.rev_labels
     elsif options['spec']

data/lib/puppet_docker_tools/runner.rb

@@ -29,7 +29,9 @@ class PuppetDockerTools
     # @param build_args Pass arbitrary buildargs to the container build.
     #        Expected to be an array of strings, each string formatted like
     #        'arg=value'.
-    def build(no_cache: false, version: nil, build_args: [])
+    # @param latest Whether or not to build the latest tag along with the
+    #        versioned image build.
+    def build(no_cache: false, version: nil, build_args: [], latest: true)
       image_name = File.basename(directory)
       build_args_hash = {
         'vcs_ref' => PuppetDockerTools::Utilities.current_git_sha(directory),
@@ -47,6 +49,8 @@ class PuppetDockerTools
         build_args_hash.merge!(PuppetDockerTools::Utilities.parse_build_args(Array(build_args)))
       end
 
+      build_args_hash = PuppetDockerTools::Utilities.filter_build_args(build_args: build_args_hash, dockerfile: "#{directory}/#{dockerfile}")
+
       # This variable is meant to be used for building the non-latest tagged build
       # If the version was set via `version` or `build_args`, use that. If not,
       # use `get_value_from_env` to parse that value from the dockerfile.
@@ -58,17 +62,22 @@ class PuppetDockerTools
       version = build_args_hash['version'] || PuppetDockerTools::Utilities.get_value_from_env('version', namespace: namespace, directory: directory, dockerfile: dockerfile)
 
       path = "#{repository}/#{image_name}"
-      puts "Building #{path}:latest"
 
-      # 't' in the build_options sets the tag for the image we're building
-      build_options = { 't' => "#{path}:latest", 'dockerfile' => dockerfile, 'buildargs' => "#{build_args_hash.to_json}"}
+      build_options = {'dockerfile' => dockerfile, 'buildargs' => "#{build_args_hash.to_json}"}
 
       if no_cache
         puts "Ignoring cache for #{path}"
         build_options['nocache'] = true
       end
 
-      Docker::Image.build_from_dir(directory, build_options)
+      if latest
+        puts "Building #{path}:latest"
+
+        # 't' in the build_options sets the tag for the image we're building
+        build_options['t'] = "#{path}:latest"
+
+        Docker::Image.build_from_dir(directory, build_options)
+      end
 
       if version
         puts "Building #{path}:#{version}"
@@ -107,12 +116,14 @@ class PuppetDockerTools
 
     # Push an image to hub.docker.com
     #
-    def push
+    # @param latest Whether or not to push the latest tag along with the
+    #        versioned image build.
+    def push(latest: true)
       image_name = File.basename(directory)
       path = "#{repository}/#{image_name}"
       version = PuppetDockerTools::Utilities.get_value_from_label(path, value: 'version', namespace: namespace)
 
-      # We always want to push a versioned label in addition to the latest label
+      # We always want to push a versioned label
       unless version
         fail "No version specified in #{dockerfile} for #{path}"
       end
@@ -123,10 +134,12 @@ class PuppetDockerTools
         fail "Pushing #{path}:#{version} to dockerhub failed!"
       end
 
-      puts "Pushing #{path}:latest to Docker Hub"
-      exitstatus, _ = PuppetDockerTools::Utilities.push_to_dockerhub("#{path}:latest")
-      unless exitstatus == 0
-        fail "Pushing #{path}:latest to dockerhub failed!"
+      if latest
+        puts "Pushing #{path}:latest to Docker Hub"
+        exitstatus, _ = PuppetDockerTools::Utilities.push_to_dockerhub("#{path}:latest")
+        unless exitstatus == 0
+          fail "Pushing #{path}:latest to dockerhub failed!"
+        end
       end
     end
 

data/lib/puppet_docker_tools/utilities.rb

@@ -45,6 +45,32 @@ class PuppetDockerTools
       args_hash
     end
 
+    # Filter build args to only include ARGs listed in the dockerfile. This is
+    # meant for compatibility with old versions of docker.
+    #
+    # @param build_args hash of build args to filter
+    # @param dockerfile the dockerfile to look for ARGs in
+    def filter_build_args(build_args: , dockerfile: )
+      fail "File #{dockerfile} doesn't exist!" unless File.exist? dockerfile
+      text = File.read(dockerfile)
+
+      # Docker only supports passing a single ARG on each line, so
+      # look for arg, and ignore any default settings since we only care
+      # whether or not the key is available
+      implemented_args = text.scan(/arg +([^\n=]+)/i).flatten
+
+      # reject any entries for args that are not in the dockerfile
+      build_args.reject { |k,v|
+        if implemented_args.include?(k)
+          false
+        else
+          puts "Rejecting `--build-arg #{k}` since that ARG isn't in the Dockerfile"
+          true
+        end
+      }
+    end
+
+
     # Get a value from the labels on a docker image
     #
     # @param image The docker image you want to get a value from, e.g. 'puppet/puppetserver'
@@ -70,11 +96,18 @@ class PuppetDockerTools
       text = File.read(file)
 
       value = text.scan(/#{Regexp.escape(namespace)}\.(.+)=(.+) \\?/).to_h[label]
+      # tracking to make sure we aren't in an infinite variable loop
+      checked_variables = []
+
       # expand out environment variables
       # This supports either label=$variable or label="$variable"
-      if value.start_with?('$') || value.start_with?('"$')
+      while ! value.nil? && (value.start_with?('$') || value.start_with?('"$'))
         # if variable is quoted, get rid of leading and trailing quotes
         value.gsub!(/\A"|"\Z/, '')
+
+        fail "Looks like there's an infinite loop with '#{value}'" if checked_variables.include?(value)
+
+        checked_variables << value
         value = get_value_from_variable(value, directory: directory, dockerfile: dockerfile, dockerfile_contents: text)
       end
       # check in higher-level image if we didn't find it defined in this docker file
@@ -208,9 +241,11 @@ class PuppetDockerTools
         fail "File #{file} doesn't exist!" unless File.exist? file
         dockerfile_contents = File.read("#{file}")
       end
+      variable_clone = String.new(variable)
       # get rid of the leading $ for the variable
-      variable[0] = ''
-      dockerfile_contents[/#{variable}=(["a-zA-Z0-9\.]+)/, 1]
+      variable_clone[0] = ''
+
+      dockerfile_contents[/#{variable_clone}=([^\s]+)/, 1]
     end
     private :get_value_from_variable
   end

data/spec/lib/puppet_docker_tools/runner_spec.rb

@@ -14,6 +14,15 @@ describe PuppetDockerTools::Runner do
   let(:buildargs) {{ 'vcs_ref' => 'b0c5ead01b6cabdb3f01871bce699be165c3288f', 'build_date' => '2018-06-08T17:18:13Z' }.to_json}
   let(:buildargs_with_version) {{ 'vcs_ref' => 'b0c5ead01b6cabdb3f01871bce699be165c3288f', 'build_date' => '2018-06-08T17:18:13Z', 'version' => '1.2.3' }.to_json}
   let(:extra_buildargs) {{ 'vcs_ref' => 'b0c5ead01b6cabdb3f01871bce699be165c3288f', 'build_date' => '2018-06-08T17:18:13Z', 'foo' => 'bar', 'baz' => 'test=with=equals' }.to_json}
+  let(:read_dockerfile) {
+    "FROM ubuntu:16.04\n\nARG vcs_ref\nARG build_date"
+  }
+  let(:read_dockerfile_with_version) {
+    "FROM ubuntu:16.04\n\nARG version\nARG vcs_ref\nARG build_date"
+  }
+  let(:read_dockerfile_with_arbitrary_args) {
+    "FROM ubuntu:16.04\n\nARG foo\nARG baz\nARG vcs_ref\nARG build_date"
+  }
 
   describe '#initialize' do
     it "should fail if the dockerfile doesn't exist" do
@@ -26,6 +35,7 @@ describe PuppetDockerTools::Runner do
     let(:image) { double(Docker::Image) }
 
     it 'builds a latest and version tag if version is found' do
+      expect(File).to receive(:read).with("#{runner.directory}/#{runner.dockerfile}").and_return(read_dockerfile)
       expect(PuppetDockerTools::Utilities).to receive(:get_value_from_env).with('version', namespace: runner.namespace, directory: runner.directory, dockerfile: runner.dockerfile).and_return('1.2.3')
       expect(Docker::Image).to receive(:build_from_dir).with(runner.directory, { 't' => 'test/test-image:1.2.3', 'dockerfile' => runner.dockerfile, 'buildargs' => buildargs }).and_return(image)
       expect(Docker::Image).to receive(:build_from_dir).with(runner.directory, { 't' => 'test/test-image:latest', 'dockerfile' => runner.dockerfile, 'buildargs' => buildargs }).and_return(image)
@@ -33,30 +43,42 @@ describe PuppetDockerTools::Runner do
     end
 
     it 'builds just a latest tag if no version is found' do
+      expect(File).to receive(:read).with("#{runner.directory}/#{runner.dockerfile}").and_return(read_dockerfile)
       expect(PuppetDockerTools::Utilities).to receive(:get_value_from_env).with('version', namespace: runner.namespace, directory: runner.directory, dockerfile: runner.dockerfile).and_return(nil)
       expect(Docker::Image).to receive(:build_from_dir).with(runner.directory, { 't' => 'test/test-image:latest', 'dockerfile' => runner.dockerfile, 'buildargs' => buildargs }).and_return(image)
       runner.build
     end
 
+    it 'does not build a latest tag if latest is set to false' do
+      expect(File).to receive(:read).with("#{runner.directory}/#{runner.dockerfile}").and_return(read_dockerfile)
+      expect(PuppetDockerTools::Utilities).to receive(:get_value_from_env).with('version', namespace: runner.namespace, directory: runner.directory, dockerfile: runner.dockerfile).and_return('1.2.3')
+      expect(Docker::Image).to receive(:build_from_dir).with(runner.directory, { 't' => 'test/test-image:1.2.3', 'dockerfile' => runner.dockerfile, 'buildargs' => buildargs }).and_return(image)
+      runner.build(latest: false)
+    end
+
     it 'ignores the cache when that parameter is set' do
+      expect(File).to receive(:read).with("#{runner.directory}/#{runner.dockerfile}").and_return(read_dockerfile)
       expect(PuppetDockerTools::Utilities).to receive(:get_value_from_env).with('version', namespace: runner.namespace, directory: runner.directory, dockerfile: runner.dockerfile).and_return(nil)
       expect(Docker::Image).to receive(:build_from_dir).with(runner.directory, { 't' => 'test/test-image:latest', 'nocache' => true, 'dockerfile' => runner.dockerfile, 'buildargs' => buildargs }).and_return(image)
       runner.build(no_cache: true)
     end
 
     it 'passes the version when that parameter is set' do
+      expect(File).to receive(:read).with("#{runner.directory}/#{runner.dockerfile}").and_return(read_dockerfile_with_version)
       expect(Docker::Image).to receive(:build_from_dir).with(runner.directory, { 't' => 'test/test-image:latest', 'dockerfile' => runner.dockerfile, 'buildargs' => buildargs_with_version }).and_return(image)
       expect(Docker::Image).to receive(:build_from_dir).with(runner.directory, { 't' => 'test/test-image:1.2.3', 'dockerfile' => runner.dockerfile, 'buildargs' => buildargs_with_version }).and_return(image)
       runner.build(version: '1.2.3')
     end
 
     it 'passes arbitrary build args' do
+      expect(File).to receive(:read).with("#{runner.directory}/#{runner.dockerfile}").and_return(read_dockerfile_with_arbitrary_args)
       expect(PuppetDockerTools::Utilities).to receive(:get_value_from_env).with('version', namespace: runner.namespace, directory: runner.directory, dockerfile: runner.dockerfile).and_return(nil)
       expect(Docker::Image).to receive(:build_from_dir).with(runner.directory, { 't' => 'test/test-image:latest', 'dockerfile' => runner.dockerfile, 'buildargs' => extra_buildargs }).and_return(image)
       runner.build(build_args: ['foo=bar', 'baz=test=with=equals'])
     end
 
     it 'prioritizes version as a build arg over regular version' do
+      expect(File).to receive(:read).with("#{runner.directory}/#{runner.dockerfile}").and_return(read_dockerfile_with_version)
       expect(Docker::Image).to receive(:build_from_dir).with(runner.directory, { 't' => 'test/test-image:latest', 'dockerfile' => runner.dockerfile, 'buildargs' => buildargs_with_version }).and_return(image)
       expect(Docker::Image).to receive(:build_from_dir).with(runner.directory, { 't' => 'test/test-image:1.2.3', 'dockerfile' => runner.dockerfile, 'buildargs' => buildargs_with_version }).and_return(image)
       runner.build(version: '1.2.4', build_args: ['version=1.2.3'])
@@ -64,6 +86,7 @@ describe PuppetDockerTools::Runner do
 
     it 'uses a custom dockerfile if passed' do
       allow(File).to receive(:exist?).with('/tmp/test-image/Dockerfile.test').and_return(true)
+      expect(File).to receive(:read).with('/tmp/test-image/Dockerfile.test').and_return(read_dockerfile)
       expect(PuppetDockerTools::Utilities).to receive(:get_value_from_env).with('version', namespace: 'org.label-schema', directory: '/tmp/test-image', dockerfile: 'Dockerfile.test').and_return(nil)
       expect(Docker::Image).to receive(:build_from_dir).with('/tmp/test-image', { 't' => 'test/test-image:latest', 'dockerfile' => 'Dockerfile.test', 'buildargs' => buildargs }).and_return(image)
       local_runner = create_runner(directory: '/tmp/test-image', repository: 'test', namespace: 'org.label-schema', dockerfile: 'Dockerfile.test')
@@ -141,6 +164,12 @@ describe PuppetDockerTools::Runner do
       expect(PuppetDockerTools::Utilities).to receive(:push_to_dockerhub).with('test/test-image:latest').and_return([0, nil])
       runner.push
     end
+
+    it 'should not push the latest tag if latest is set to false' do
+      expect(PuppetDockerTools::Utilities).to receive(:get_value_from_label).with('test/test-image', value: 'version', namespace: runner.namespace).and_return('1.2.3')
+      expect(PuppetDockerTools::Utilities).to receive(:push_to_dockerhub).with('test/test-image:1.2.3').and_return([0, nil])
+      runner.push(latest: false)
+    end
   end
 
   describe '#rev_labels' do

data/spec/lib/puppet_docker_tools/utilities_spec.rb

@@ -37,6 +37,49 @@ LABEL maintainer="Puppet Release Team <release@puppet.com>" \\
       com.puppet.dockerfile="/Dockerfile"
 HERE
   }
+  let(:infinite_dockerfile_contents) { <<-HERE
+FROM ubuntu:16.04
+
+ENV PUPPET_SERVER_VERSION=$foo foo=$PUPPET_SERVER_VERSION
+
+LABEL maintainer="Puppet Release Team <release@puppet.com>" \\
+      org.label-schema.vendor="Puppet" \\
+      org.label-schema.url="https://github.com/puppetlabs/puppet-in-docker" \\
+      org.label-schema.name="Puppet Server (No PuppetDB)" \\
+      org.label-schema.license="Apache-2.0" \\
+      org.label-schema.version="$PUPPET_SERVER_VERSION" \\
+      org.label-schema.vcs-url="https://github.com/puppetlabs/puppet-in-docker" \\
+      org.label-schema.vcs-ref="b75674e1fbf52f7821f7900ab22a19f1a10cafdb" \\
+      org.label-schema.build-date="2018-05-09T20:11:01Z" \\
+      org.label-schema.schema-version="1.0" \\
+      com.puppet.dockerfile="/Dockerfile"
+HERE
+  }
+  let(:dockerfile_with_args) { <<-HERE
+FROM puppet/puppetserver-standalone:5.3.1
+
+ARG version
+ARG foo
+ARG bar
+HERE
+  }
+
+  let(:build_args) {
+    {
+      'version' => '1.2.3',
+      'foo' => 'test',
+      'bar' => 'baz',
+      'test' => 'test2',
+    }
+  }
+
+  let(:filtered_build_args) {
+    {
+      'version' => '1.2.3',
+      'foo' => 'test',
+      'bar' => 'baz',
+    }
+  }
 
   let(:config_labels) { {
     'Config' => {
@@ -49,6 +92,20 @@ HERE
   }
   }
 
+  describe "#filter_build_args" do
+    it "should fail if the dockerfile doesn't exist" do
+      allow(File).to receive(:exist?).with("/tmp/test-dir/#{dockerfile}").and_return(false)
+      expect { PuppetDockerTools::Utilities.filter_build_args(build_args: build_args, dockerfile: "/tmp/test-dir/#{dockerfile}") }.to raise_error(RuntimeError, /doesn't exist/)
+    end
+
+    it "should filter out any buildargs not in the dockerfile" do
+      allow(File).to receive(:exist?).with("/tmp/test-dir/#{dockerfile}").and_return(true)
+      allow(File).to receive(:read).with("/tmp/test-dir/#{dockerfile}").and_return(dockerfile_with_args)
+      expect(PuppetDockerTools::Utilities.filter_build_args(build_args: build_args, dockerfile: "/tmp/test-dir/#{dockerfile}")).to eq(filtered_build_args)
+    end
+
+  end
+
   describe "#get_value_from_label" do
     let(:image) { double(Docker::Image).as_null_object }
 
@@ -89,6 +146,12 @@ HERE
       allow(File).to receive(:read).with("/tmp/test-dir/../puppetserver-standalone/#{dockerfile}").and_return(base_dockerfile_contents)
       expect(PuppetDockerTools::Utilities.get_value_from_env('version', namespace: 'org.label-schema', directory: '/tmp/test-dir')).to eq('5.3.1')
     end
+
+    it "doesn't get stuck in an infinite loop if there's a bad variable definition" do
+      allow(File).to receive(:exist?).with("/tmp/test-dir/#{dockerfile}").and_return(true)
+      allow(File).to receive(:read).with("/tmp/test-dir/#{dockerfile}").and_return(infinite_dockerfile_contents)
+      expect { PuppetDockerTools::Utilities.get_value_from_env('version', namespace: 'org.label-schema', directory: '/tmp/test-dir')}.to raise_error(RuntimeError, /infinite loop/)
+    end
   end
 
   describe "#format_timestamp" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: puppet_docker_tools
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - Puppet, Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-06-21 00:00:00.000000000 Z
+date: 2018-06-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: docker-api