nutkins 0.6.1 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ef4b59fb3d6c03246d6b310132556bcac28545c6
-  data.tar.gz: c4491a84ab7d296609fb4c7f48a6fa54dba79cf4
+  metadata.gz: 3bfa69ab3c9cd1fa876a151921a9e04f1af3de8d
+  data.tar.gz: 52b50703467b3f9f010733a7681f9739653002c9
 SHA512:
-  metadata.gz: df798fc1aa5262eaa5523566be44d690498705a7e641d6b446e9df53c58f586aba6c5bf88436b1b8b32da5274f6b9e357b7b8398689f7a4bfd3e8c7c58e6e8eb
-  data.tar.gz: 59860a712db9a24d02479794ef8a2f09f0c978c989c8b87843d773b4e2e56da76877c12d9b78f159db2dc1eff7e93a2cf6ac59b523d8b64ab08c7ca548ba7d09
+  metadata.gz: a6ff593c34f989c41d76c1f45395ed35b2a44dd289ae1e1fbe9ced7909467315c15a8a3896b712874d5ab4895748c3ab3d46187a8ec6048c0ef82e341c8dcbca
+  data.tar.gz: 258db6118fb8c67204521773771497b1e6f51a21731feb41ab6a82a1274fb36b43a426f6dce404bd261f29e3dd522d75b86e052d1341fcd36218e1ea751882b5

data/lib/nutkins/docker.rb

@@ -5,14 +5,28 @@ module Nutkins::Docker
     self.run_get_stdout 'inspect', '--format="{{.Id}}"', tag
   end
 
-  def self.container_id_for_tag tag
-    regex = /^[0-9a-f]+ +#{tag} +/
-    `docker ps -a`.each_line do |line|
+  def self.kill_and_remove_container id
+    raise "issue killing existing container" unless Nutkins::Docker.run 'kill', id
+    raise "issue removing existing container" unless Nutkins::Docker.run 'rm', id
+  end
+
+  def self.container_id_for_tag tag, running: false
+    flags = running ? '' : ' -a'
+    regex = /^[0-9a-f]+ +#{Regexp.escape tag} +/
+    `docker ps#{flags}`.each_line do |line|
       return line.split(' ')[0] if line =~ regex
     end
     nil
   end
 
+  def self.get_short_commit commit
+    if /^sha256:/.match(commit)
+      commit[7...19]
+    else
+      commit
+    end
+  end
+
   def self.container_id_for_name name
     self.run_get_stdout 'inspect', '--format="{{.Id}}"', name
   end

data/lib/nutkins/docker_builder.rb

@@ -0,0 +1,120 @@
+require_relative "docker"
+require "json"
+require "digest"
+
+module Nutkins::DockerBuilder
+  def self.build cfg
+    base = cfg["base"]
+    raise "to use build commands you must specify the base image" unless base
+
+    # TODO: build cache from this and use to determine restore point
+    # Nutkins::Docker.run 'inspect', tag, stderr: false
+
+    unless Nutkins::Docker.run 'inspect', base, stderr: false
+      puts "getting base image"
+      Docker.run 'pull', base, stdout: true
+    end
+
+    cont_id = Nutkins::Docker.container_id_for_tag base, running: true
+    if cont_id
+      puts "found existing container #{cont_id}"
+      Nutkins::Docker.kill_and_remove_container cont_id
+      puts "killed and removed existing container"
+    end
+
+    # the base image to start rebuilding from
+    parent_img_id = base
+    cont_id = nil
+    pwd = Dir.pwd
+    begin
+      Dir.chdir cfg["directory"]
+
+      cache_is_dirty = false
+      build_commands = cfg["build"]["commands"]
+      build_commands.each do |build_cmd|
+        cmd = /^\w+/.match(build_cmd).to_s.downcase
+        cmd_args = build_cmd[(cmd.length + 1)..-1].strip
+
+        docker_args = []
+        # the commit_msg is used to look up cache entries, it can be
+        # modified if the command uses dynamic data, e.g. to add checksums
+        commit_msg = nil
+
+        case cmd
+        when "run"
+          cmd_args.gsub! /\n+/, ' '
+          docker_args = ['exec', '%CONT_ID%', cfg['shell'], '-c', cmd_args]
+          commit_msg = cmd + ' ' + cmd_args
+        when "add"
+          *srcs, dest = cmd_args.split ' '
+          srcs = srcs.map { |src| Dir.glob src }.flatten
+
+          docker_args = srcs.map { |src| ['cp', src, '%CONT_ID%' + ':' + dest] }
+          # ensure checksum of each file is embedded into commit_msg
+          # if any file changes the cache is dirtied
+          commit_msg = 'add ' + srcs.map do |src|
+            src + ':' + Digest::MD5.file(src).to_s
+          end.push(dest).join(' ')
+        when "cmd", "entrypoint", "env", "expose", "label", "onbuild", "user", "volume", "workdir"
+          commit_msg = commit_change = build_cmd
+        else
+          raise "unsupported command: #{cmd}"
+          # TODO add metadata flags
+        end
+
+        if (commit_change or docker_args) and commit_msg
+          commit_msg = "#{parent_img_id} -> #{commit_msg}"
+
+          unless cache_is_dirty
+            # searches the commit messages of all images for the one matching the expected
+            # cache entry for the given content
+            all_images = Nutkins::Docker.run_get_stdout('images', '-aq').split("\n")
+            images_meta = JSON.parse(Nutkins::Docker.run_get_stdout('inspect', *all_images))
+            cache_entry = images_meta.find do |image_meta|
+              if image_meta['Comment'] == commit_msg
+                parent_img_id = Nutkins::Docker.get_short_commit(image_meta['Id'])
+                true
+              end
+            end
+
+            if cache_entry
+              puts "cached: #{commit_msg}"
+              next
+            else
+              puts "starting build container from commit #{parent_img_id}"
+              Nutkins::Docker.run 'run', '-d', parent_img_id, 'sleep', '3600'
+              cont_id = Nutkins::Docker.container_id_for_tag parent_img_id, running: true
+              puts "started build container #{cont_id}"
+              cache_is_dirty = true
+            end
+          end
+
+          puts "#{cmd}: #{cmd_args}"
+
+          unless docker_args.empty?
+            # docker can be an array of one set of args, or an array of arrays of args
+            docker_args = [ docker_args ] unless docker_args[0].kind_of? Array
+            docker_args.each do |one_docker_args|
+              run_args = one_docker_args.map { |arg| arg.gsub '%CONT_ID%', cont_id }
+              puts "run #{run_args.join ' '}"
+              unless Nutkins::Docker.run *run_args, stdout: true
+                raise "build failed: #{one_docker_args.join ' '}"
+              end
+            end
+          end
+
+          commit_args = commit_change ? ['-c', commit_change] : []
+          parent_img_id = Nutkins::Docker.run_get_stdout 'commit', '-m', commit_msg, *commit_args, cont_id
+          raise "could not commit docker image" if parent_img_id.nil?
+          parent_img_id = Nutkins::Docker.get_short_commit parent_img_id
+        else
+          puts "TODO: support cmd #{build_cmd}"
+        end
+      end
+    ensure
+      Dir.chdir pwd
+      Nutkins::Docker.kill_and_remove_container cont_id if cont_id
+      puts "killed and removed build container"
+    end
+  end
+end

data/lib/nutkins/version.rb

@@ -1,3 +1,3 @@
 module Nutkins
-  VERSION = "0.6.1"
+  VERSION = "0.7.0"
 end

data/lib/nutkins.rb

@@ -9,6 +9,7 @@ require "net/http"
 module Nutkins ; end
 
 require "nutkins/docker"
+require "nutkins/docker_builder"
 require "nutkins/download"
 require "nutkins/version"
 
@@ -36,6 +37,8 @@ module Nutkins
       raise "directory `#{img_dir}' does not exist" unless Dir.exists? img_dir
       tag = cfg['tag']
 
+      prev_image_id = Docker.image_id_for_tag tag
+
       build_cfg = cfg["build"]
       if build_cfg
         # download each of the files in the resources section if it doesn't exist
@@ -43,16 +46,22 @@ module Nutkins
         Download.download_resources img_dir, resources if resources
       end
 
-      prev_image_id = Docker.image_id_for_tag tag
-
-      if Docker.run 'build', '-t', cfg['latest_tag'], '-t', tag, img_dir, stdout: true
-        image_id = Docker.image_id_for_tag tag
-        if prev_image_id and image_id != prev_image_id
-          puts "deleting previous image #{prev_image_id}"
-          Docker.run "rmi", prev_image_id
+      if cfg.dig "build", "commands"
+        # if build commands are available use nutkins built-in builder
+        DockerBuilder::build cfg
+      else
+        # fallback to `docker build` which is less good
+        if not Docker.run 'build', '-t', cfg['latest_tag'], '-t', tag, img_dir, stdout: true
+          raise "issue building docker image for #{img_name}"
         end
+      end
+
+      image_id = Docker.image_id_for_tag tag
+      if prev_image_id and image_id != prev_image_id
+        puts "deleting previous image #{prev_image_id}"
+        Docker.run "rmi", prev_image_id
       else
-        raise "issue building docker image for #{img_name}"
+        puts "image is identical to cached version"
       end
     end
 
@@ -286,6 +295,7 @@ module Nutkins
       img_cfg_path = File.join directory, IMG_CONFIG_FILE_NAME
       img_cfg = File.exists?(img_cfg_path) ? YAML.load_file(img_cfg_path) : {}
       img_cfg['image'] ||= path if path != '.'
+      img_cfg['shell'] ||= 'sh'
       img_cfg['directory'] = directory
       img_cfg["version"] ||= @config.version if @config.version
       img_cfg['version'] = img_cfg['version'].to_s

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: nutkins
 version: !ruby/object:Gem::Version
-  version: 0.6.1
+  version: 0.7.0
 platform: ruby
 authors:
 - James Pike
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-08-21 00:00:00.000000000 Z
+date: 2016-08-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -86,6 +86,7 @@ files:
 - circle.yml
 - lib/nutkins.rb
 - lib/nutkins/docker.rb
+- lib/nutkins/docker_builder.rb
 - lib/nutkins/download.rb
 - lib/nutkins/version.rb
 - nutkins.gemspec