nutkins 0.9.0 → 0.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1bd8ba58c448c0926badec45b1f9b008e64f3b24
-  data.tar.gz: 224e661371deed2e36281e9371370b7d3df757fb
+  metadata.gz: e73d665ef1da7e2713d6d7ab469d963f50969c39
+  data.tar.gz: 02399c0c0dca4636432ec49625978433b9e3f994
 SHA512:
-  metadata.gz: c3a1d0b249945812bd1e62d16400f99f2d89ec7e1acac4c68f87c3f7096f861c9dad186e9a7e74e805ca1c7d7123149e28eb4e75dc68c09578bbc92bc5f4ed5c
-  data.tar.gz: 667e711943593dd8f1d6b10b883225402dc6b5d8b6ded9a95abf90fdd1c517080dde6811ea58c012f2a1df29a8460268988e1c315b319db982e0b3c71d809d0c
+  metadata.gz: 6d0cc75a3a1d4b93d7ffc50ebed8233e69d3c17577cabffe1f96e94a4aee7ac94432b683e479b5fdd83f11f390bd6c2003ce699d5dac4ed259429b54c4e6b140
+  data.tar.gz: eca9bb3a1db88257cf7813e0eb62fe2cab4c5ec7b41067ac4d691f010b9de3ac9a778f505ec1370c441a3ec4559a9e5faff6e9d0c92a7ac7eaba3630498f1256

data/bin/nutkins

@@ -24,24 +24,24 @@ module Nutkins::Command
 
       op.on '-p', '--project dir', 'override path to project', 'project_dir'
 
-      op.subcommand 'build,b *names', 'build docker image from dockerfile'
+      op.subcommand 'build,b *paths', 'build docker image[s] from nutkin.yamls/dockerfiles'
 
-      op.subcommand 'create,c *names', 'create container from image' do |subop|
+      op.subcommand 'create,c *paths', 'create container from image' do |subop|
         subop.on '-r', '--reuse', 'reuse previously built image', 'reuse'
         subop.on '-p', '--preserve', 'preserve existing container', 'preserve'
       end
 
-      op.subcommand 'delete,d *names', 'delete container corresponding to image'
+      op.subcommand 'delete,d *paths', 'delete container corresponding to image'
       op.subcommand 'delete-all', 'delete containers corresponding to all images'
 
-      op.subcommand 'run,r name', 'run created container' do |subop|
+      op.subcommand 'run,r path', 'run created container' do |subop|
         subop.on '-r', '--reuse', 'reuse previously created container', 'reuse'
         subop.on '-s', '--shell', 'add bash shell to console', 'shell'
       end
 
-      op.subcommand 'exec,e name *cmd', 'execute a command in a running container'
+      op.subcommand 'exec,e path *cmd', 'execute a command in a running container'
       op.subcommand 'build-secret,B path', 'build secret files/volumes'
-      op.subcommand 'extract-secrets,X [*names]', 'extract secret files/volumes'
+      op.subcommand 'extract-secrets,X [*paths]', 'extract secret files/volumes'
 
       op.subcommand 'start-etcd', 'start container running etcd'
       op.subcommand 'stop-etcd', 'stop container running etcd'
@@ -62,23 +62,23 @@ module Nutkins::Command
 
     case command
     when 'build'
-      config.names.each &nutkins.method(:build)
+      config.paths.each &nutkins.method(:build)
     when 'create'
-      config.names.each do |name|
-        nutkins.create name, preserve: config.preserve, reuse: config.reuse
+      config.paths.each do |path|
+        nutkins.create path, preserve: config.preserve, reuse: config.reuse
       end
     when 'delete'
-      config.names.each &nutkins.method(:delete)
+      config.paths.each &nutkins.method(:delete)
     when 'delete-all'
       nutkins.delete_all
     when 'run'
-      nutkins.run config.name, reuse: config.reuse, shell: config.shell
+      nutkins.run config.path, reuse: config.reuse, shell: config.shell
     when 'exec'
-      nutkins.exec config.name, config.cmd
+      nutkins.exec config.path, config.cmd
     when 'build-secret'
       nutkins.build_secret config.path
     when 'extract-secrets'
-      nutkins.extract_secrets config.names
+      nutkins.extract_secrets config.paths
     when 'start-etcd'
       nutkins.start_etcd_container
     when 'stop-etcd'

data/lib/hash_dig.rb

@@ -0,0 +1,14 @@
+unless Hash.new.respond_to? :dig
+  class Hash
+    def dig first_key, *other_keys
+      val = self[first_key]
+      if other_keys.empty?
+        val
+      elsif val and val.kind_of? Hash
+        val.dig *other_keys
+      else
+        nil
+      end
+    end
+  end
+end

data/lib/nutkins/docker_builder.rb

@@ -2,21 +2,23 @@ require_relative "docker"
 require "json"
 require "digest"
 
-module Nutkins::DockerBuilder
+module Nutkins::Docker::Builder
+  Docker = Nutkins::Docker
+
   def self.build cfg
     base = cfg["base"]
     raise "to use build commands you must specify the base image" unless base
 
     # TODO: build cache from this and use to determine restore point
-    # Nutkins::Docker.run 'inspect', tag, stderr: false
+    # Docker.run 'inspect', tag, stderr: false
 
-    unless Nutkins::Docker.run 'inspect', base, stderr: false
+    unless Docker.run 'inspect', base, stderr: false
       puts "getting base image"
       Docker.run 'pull', base, stdout: true
     end
 
     # the base image to start rebuilding from
-    parent_img_id = Nutkins::Docker.get_short_commit Nutkins::Docker.container_id_for_name(base)
+    parent_img_id = Docker.get_short_commit Docker.container_id_for_name(base)
     pwd = Dir.pwd
     begin
       Dir.chdir cfg["directory"]
@@ -80,7 +82,7 @@ module Nutkins::DockerBuilder
 
           if run_args
             puts "run #{run_args}"
-            unless Nutkins::Docker.run 'run', parent_img_id, *run_shell_cmd, stdout: true
+            unless Docker.run 'run', parent_img_id, *run_shell_cmd, stdout: true
               raise "run failed: #{run_args}"
             end
 
@@ -88,18 +90,18 @@ module Nutkins::DockerBuilder
             begin
               if add_files
                 add_files.each do |src|
-                  if not Nutkins::Docker.run 'cp', src, "#{cont_id}:#{add_files_dest}"
+                  if not Docker.run 'cp', src, "#{cont_id}:#{add_files_dest}"
                     raise "could not copy #{src} to #{cont_id}:#{add_files_dest}"
                   end
                 end
               end
 
               commit_args = env_args ? ['-c', env_args] : []
-              parent_img_id = Nutkins::Docker.run_get_stdout 'commit', *commit_args, cont_id
+              parent_img_id = Docker.run_get_stdout 'commit', *commit_args, cont_id
               raise "could not commit docker image" if parent_img_id.nil?
-              parent_img_id = Nutkins::Docker.get_short_commit parent_img_id
+              parent_img_id = Docker.get_short_commit parent_img_id
             ensure
-              if not Nutkins::Docker.run 'rm', cont_id
+              if not Docker.run 'rm', cont_id
                 puts "could not remove build container #{cont_id}"
               end
             end
@@ -112,16 +114,16 @@ module Nutkins::DockerBuilder
       Dir.chdir pwd
     end
 
-    Nutkins::Docker.run 'tag', parent_img_id, cfg['tag']
+    Docker.run 'tag', parent_img_id, cfg['tag']
   end
 
   def self.find_cached_img_id parent_img_id, command
-    all_images = Nutkins::Docker.run_get_stdout('images', '-aq').split("\n")
-    images_meta = JSON.parse(Nutkins::Docker.run_get_stdout('inspect', *all_images))
+    all_images = Docker.run_get_stdout('images', '-aq').split("\n")
+    images_meta = JSON.parse(Docker.run_get_stdout('inspect', *all_images))
     images_meta.each do |image_meta|
       if image_meta.dig('ContainerConfig', 'Cmd') == command and
-         Nutkins::Docker.get_short_commit(image_meta['Parent']) == parent_img_id
-        return Nutkins::Docker.get_short_commit(image_meta['Id'])
+         Docker.get_short_commit(image_meta['Parent']) == parent_img_id
+        return Docker.get_short_commit(image_meta['Id'])
       end
     end
     nil

data/lib/nutkins/version.rb

@@ -1,3 +1,3 @@
 module Nutkins
-  VERSION = "0.9.0"
+  VERSION = "0.10.0"
 end

data/lib/nutkins.rb

@@ -13,6 +13,8 @@ require "nutkins/docker_builder"
 require "nutkins/download"
 require "nutkins/version"
 
+require_relative "hash_dig"
+
 # Must be somedomain.net instead of somedomain.net/, otherwise, it will throw exception.
 module Nutkins
   CONFIG_FILE_NAME = 'nutkins.yaml'
@@ -22,6 +24,9 @@ module Nutkins
 
   class CloudManager
     def initialize(project_dir: nil)
+      @img_configs = {}
+      # when an image is built true is stored against it's name to avoid building it again
+      @built = {}
       @project_root = project_dir || Dir.pwd
       cfg_path = File.join(@project_root, CONFIG_FILE_NAME)
       if File.exists? cfg_path
@@ -31,12 +36,23 @@ module Nutkins
       end
     end
 
-    def build img_name
-      cfg = get_image_config img_name
-      img_dir = get_project_dir img_name
+    def build path
+      cfg = get_image_config path
+      img_dir = cfg['directory']
+      img_name = cfg['image']
+      return if @built[img_name]
+
       raise "directory `#{img_dir}' does not exist" unless Dir.exists? img_dir
       tag = cfg['tag']
 
+      # TODO: flag to suppress building base image?
+      base = cfg['base']
+      unless @built[base]
+        if image_in_project? base
+          puts "building parent of #{img_name}: #{base}"
+          build base
+        end
+      end
       prev_image_id = Docker.image_id_for_tag tag
 
       build_cfg = cfg["build"]
@@ -48,11 +64,11 @@ module Nutkins
 
       if cfg.dig "build", "commands"
         # if build commands are available use nutkins built-in builder
-        DockerBuilder::build cfg
+        Docker::Builder::build cfg
       else
         # fallback to `docker build` which is less good
         if not Docker.run 'build', '-t', cfg['latest_tag'], '-t', tag, img_dir, stdout: true
-          raise "issue building docker image for #{img_name}"
+          raise "issue building docker image for #{path}"
         end
       end
 
@@ -69,18 +85,19 @@ module Nutkins
       else
         puts "no image exists for image... what went wrong?"
       end
+      @built[img_name] = true
     end
 
-    def create img_name, preserve: false, docker_args: [], reuse: false
+    def create path, preserve: false, docker_args: [], reuse: false
       flags = []
-      cfg = get_image_config img_name
+      cfg = get_image_config path
       create_cfg = cfg["create"]
       if create_cfg
         (create_cfg["ports"] or []).each do |port|
           flags.push '-p', "#{port}:#{port}"
         end
 
-        img_dir = get_project_dir img_name
+        img_dir = cfg['directory']
         (create_cfg["volumes"] or []).each do |volume|
           src, dest = volume.split ' -> '
           src_dir = File.absolute_path File.join(img_dir, VOLUMES_PATH, src)
@@ -108,12 +125,12 @@ module Nutkins
           Docker.run "rm", prev_container_id
           prev_container_id = nil
         end
-        build img_name
+        build path
       end
 
       puts "creating new docker image"
       unless Docker.run "create", "-it", *flags, tag, *docker_args
-        raise "failed to create `#{img_name}' container"
+        raise "failed to create `#{path}' container"
       end
 
       unless preserve
@@ -124,16 +141,18 @@ module Nutkins
         end
       end
 
-      puts "created `#{img_name}' container"
+      puts "created `#{path}' container"
     end
 
-    def run img_name, reuse: false, shell: false
-      cfg = get_image_config img_name
+    def run path, reuse: false, shell: false
+      cfg = get_image_config path
       tag = cfg['tag']
       create_args = []
       if shell
         raise '--shell and --reuse arguments are incompatible' if reuse
 
+        # TODO: fix crash when image doesn't exist yet... the tag isn't
+        #       there to be inspected yet
         # TODO: test for smell-baron
         create_args = JSON.parse(`docker inspect #{tag}`)[0]["Config"]["Cmd"]
 
@@ -148,21 +167,20 @@ module Nutkins
 
       id = reuse && Docker.container_id_for_tag(tag)
       unless id
-        create img_name, docker_args: create_args
+        create path, docker_args: create_args
         id = Docker.container_id_for_tag tag
-        raise "couldn't create container to run `#{img_name}'" unless id
+        raise "couldn't create container to run `#{path}'" unless id
       end
 
       Kernel.exec "docker", "start", "-ai", id
     end
 
-    def delete img_name
-      cfg = get_image_config img_name
+    def delete path
+      cfg = get_image_config path
       tag = cfg['tag']
       container_id = Docker.container_id_for_tag tag
       raise "no container to delete" if container_id.nil?
       puts "deleting container #{container_id}"
-      # TODO: also delete :latest
       Docker.run "rm", container_id
     end
 
@@ -186,13 +204,15 @@ module Nutkins
       File.unlink secret if path_is_dir
     end
 
-    def extract_secrets img_names
-      if img_names.empty?
-        img_names = get_all_img_names(img_names).push '.'
+    def extract_secrets img_dirs
+      if img_dirs.empty?
+        img_dirs = get_all_img_dirs
+        # there may be secrets in the root even if there is no image build there
+        img_dirs.push '.' unless img_dirs.include? '.'
       end
 
-      img_names.each do |img_name|
-        get_secrets(img_name).each do |secret|
+      img_dirs.each do |img_dir|
+        get_secrets(img_dir).each do |secret|
           loop do
             puts "enter passphrase for #{secret}"
             break if system 'gpg', secret
@@ -207,10 +227,11 @@ module Nutkins
       end
     end
 
-    def exec img_name, *cmd
-      puts "TODO: exec #{img_name}: #{cmd.join ' '}"
+    def exec path, *cmd
+      puts "TODO: exec #{path}: #{cmd.join ' '}"
     end
 
+    # TODO: move this stuff into another file
     def start_etcd_container
       name = get_etcd_container_name
       return unless name
@@ -232,14 +253,15 @@ module Nutkins
                  '-advertise-client-urls', "http://#{gateway}:#{ETCD_PORT}",
                  '-listen-client-urls', "http://0.0.0.0:#{ETCD_PORT}"
 
-      img_names = get_all_img_names(img_names)
-      configs = img_names.map &method(:get_image_config)
+      img_dirs = get_all_img_dirs
+      configs = img_dirs.map &method(:get_image_config)
       etcd_store = {}
       configs.each do |config|
         etcd_store.merge! config['etcd']['data'] if config.dig('etcd', 'data')
 
-        if config.dig('etcd', 'files')
-          config['etcd']['files'].each do |file|
+        etcd_files = config.dig('etcd', 'files')
+        if etcd_files
+          etcd_files.each do |file|
             etcd_data_path = File.join config['directory'], file
             begin
               etcd_store.merge! YAML.load_file(etcd_data_path)
@@ -296,19 +318,30 @@ module Nutkins
       repository && "nutkins-etcd-#{repository}"
     end
 
+    # path should be "." or a single element path referencing the project root
     def get_image_config path
+      cached = @img_configs[path]
+      return cached if cached
+
       directory =  get_project_dir(path)
       img_cfg_path = File.join directory, IMG_CONFIG_FILE_NAME
-      img_cfg = File.exists?(img_cfg_path) ? YAML.load_file(img_cfg_path) : {}
+      raise "missing #{img_cfg_path}" unless File.exists?(img_cfg_path)
+      img_cfg = YAML.load_file(img_cfg_path)
       img_cfg['image'] ||= path if path != '.'
+      raise "#{img_cfg_path} must contain 'image' entry" unless img_cfg['image']
+
       img_cfg['shell'] ||= '/bin/sh'
+      img_cfg['path'] ||= img_cfg_path
       img_cfg['directory'] = directory
       img_cfg["version"] ||= @config.version if @config.version
       img_cfg['version'] = img_cfg['version'].to_s
-      raise 'missing mandatory version field' unless img_cfg.has_key? 'version'
+      raise "#{img_cfg_path} must contain 'version' entry" unless img_cfg.has_key? 'version'
       img_cfg['latest_tag'] = get_tag img_cfg
       img_cfg['tag'] = img_cfg['latest_tag'] + ':' + img_cfg['version']
-      img_cfg
+
+      base = img_cfg['base']
+      raise "#{img_cfg_path} must include `base` field" unless  base
+      @img_configs[path] = img_cfg
     end
 
     def get_project_dir path
@@ -316,10 +349,6 @@ module Nutkins
     end
 
     def get_tag img_cfg
-      unless img_cfg.has_key? "image"
-        raise "nutkins.yaml should contain `image' entry for this command"
-      end
-
       repository = img_cfg['repository'] || @config.repository
       if repository.nil?
         raise "nutkins.yaml or nutkin.yaml should contain `repository' entry for this command"
@@ -327,15 +356,19 @@ module Nutkins
       repository + '/' + img_cfg['image']
     end
 
-    def get_all_img_names img_names
-      Dir.glob("#{@project_root}/*/Dockerfile").map do |path|
+    def get_all_img_dirs
+      Dir.glob("#{@project_root}{,/*}/nutkin.yaml").map do |path|
         File.basename File.dirname(path)
       end
     end
 
-    # can supply img_name or . for project root
-    def get_secrets img_name
-      img_dir = get_project_dir img_name
+    def image_in_project? image_name
+      get_all_img_dirs.map(&method(:get_image_config)).find do |cfg|
+        cfg['image'] == image_name
+      end
+    end
+
+    def get_secrets img_dir
       Dir.glob("#{img_dir}/{volumes,secrets}/*.gpg")
     end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: nutkins
 version: !ruby/object:Gem::Version
-  version: 0.9.0
+  version: 0.10.0
 platform: ruby
 authors:
 - James Pike
@@ -84,6 +84,7 @@ files:
 - bin/nutkins
 - bin/setup
 - circle.yml
+- lib/hash_dig.rb
 - lib/nutkins.rb
 - lib/nutkins/docker.rb
 - lib/nutkins/docker_builder.rb