leap_cli 1.5.6 → 1.6.2

data/lib/leap_cli/commands/deploy.rb

@@ -5,7 +5,7 @@ module LeapCli
     desc 'Apply recipes to a node or set of nodes.'
     long_desc 'The FILTER can be the name of a node, service, or tag.'
     arg_name 'FILTER'
-    command :deploy do |c|
+    command [:deploy, :d] do |c|
 
       # --fast
       c.switch :fast, :desc => 'Makes the deploy command faster by skipping some slow steps. A "fast" deploy can be used safely if you recently completed a normal deploy.',
@@ -17,9 +17,12 @@ module LeapCli
       # --force
       c.switch :force, :desc => 'Deploy even if there is a lockfile.', :negatable => false
 
+      # --dev
+      c.switch :dev, :desc => "Development mode: don't run 'git submodule update' before deploy.", :negatable => false
+
       # --tags
       c.flag :tags, :desc => 'Specify tags to pass through to puppet (overriding the default).',
-                    :default_value => DEFAULT_TAGS.join(','), :arg_name => 'TAG[,TAG]'
+                    :arg_name => 'TAG[,TAG]'
 
       c.flag :port, :desc => 'Override the default SSH port.',
                     :arg_name => 'PORT'
@@ -28,9 +31,12 @@ module LeapCli
                     :arg_name => 'IPADDRESS'
 
       c.action do |global,options,args|
-        init_submodules
 
-        nodes = filter_deploy_nodes(args)
+        if options[:dev] != true
+          init_submodules
+        end
+
+        nodes = manager.filter!(args)
         if nodes.size > 1
           say "Deploying to these nodes: #{nodes.keys.join(', ')}"
           if !global[:yes] && !agree("Continue? ")
@@ -38,7 +44,16 @@ module LeapCli
           end
         end
 
-        compile_hiera_files
+        environments = nodes.field('environment').uniq
+        if environments.empty?
+          environments = [nil]
+        end
+        environments.each do |env|
+          check_platform_pinning(env)
+        end
+        # compile hiera files for all the nodes in every environment that is
+        # being deployed and only those environments.
+        compile_hiera_files(manager.filter(environments))
 
         ssh_connect(nodes, connect_options(options)) do |ssh|
           ssh.leap.log :checking, 'node' do
@@ -58,39 +73,130 @@ module LeapCli
             end
           end
         end
+        if !Util.exit_status.nil? && Util.exit_status != 0
+          log :warning, "puppet did not finish successfully."
+        end
       end
     end
 
     private
 
+    #
+    # The currently activated provider.json could have loaded some pinning
+    # information for the platform. If this is the case, refuse to deploy
+    # if there is a mismatch.
+    #
+    # For example:
+    #
+    # "platform": {
+    #   "branch": "develop"
+    #   "version": "1.0..99"
+    #   "commit": "e1d6280e0a8c565b7fb1a4ed3969ea6fea31a5e2..HEAD"
+    # }
+    #
+    def check_platform_pinning(environment)
+      provider = manager.env(environment).provider
+      return unless provider['platform']
+
+      if environment.nil? || environment == 'default'
+        provider_json = 'provider.json'
+      else
+        provider_json = 'provider.' + environment + '.json'
+      end
+
+      # can we have json schema verification already?
+      unless provider.platform.is_a? Hash
+        bail!('`platform` attribute in #{provider_json} must be a hash (was %s).' % provider.platform.inspect)
+      end
+
+      # check version
+      if provider.platform['version']
+        if !Leap::Platform.version_in_range?(provider.platform.version)
+          say("The platform is pinned to a version range of '#{provider.platform.version}' "+
+            "by the `platform.version` property in #{provider_json}, but the platform "+
+            "(#{Path.platform}) has version #{Leap::Platform.version}.")
+          quit!("OK. Bye.") unless agree("Do you really want to deploy from the wrong version? ")
+        end
+      end
+
+      # check branch
+      if provider.platform['branch']
+        if !is_git_directory?(Path.platform)
+          say("The platform is pinned to a particular branch by the `platform.branch` property "+
+            "in #{provider_json}, but the platform directory (#{Path.platform}) is not a git repository.")
+          quit!("OK. Bye.") unless agree("Do you really want to deploy anyway? ")
+        end
+        unless provider.platform.branch == current_git_branch(Path.platform)
+          say("The platform is pinned to branch '#{provider.platform.branch}' by the `platform.branch` property "+
+            "in #{provider_json}, but the current branch is '#{current_git_branch(Path.platform)}' " +
+            "(for directory '#{Path.platform}')")
+          quit!("OK. Bye.") unless agree("Do you really want to deploy from the wrong branch? ")
+        end
+      end
+
+      # check commit
+      if provider.platform['commit']
+        if !is_git_directory?(Path.platform)
+          say("The platform is pinned to a particular commit range by the `platform.commit` property "+
+            "in #{provider_json}, but the platform directory (#{Path.platform}) is not a git repository.")
+          quit!("OK. Bye.") unless agree("Do you really want to deploy anyway? ")
+        end
+        current_commit = current_git_commit(Path.platform)
+        Dir.chdir(Path.platform) do
+          commit_range = assert_run!("git log --pretty='format:%H' '#{provider.platform.commit}'",
+            "The platform is pinned to a particular commit range by the `platform.commit` property "+
+            "in #{provider_json}, but git was not able to find commits in the range specified "+
+            "(#{provider.platform.commit}).")
+          commit_range = commit_range.split("\n")
+          if !commit_range.include?(current_commit) &&
+              provider.platform.commit.split('..').first != current_commit
+            say("The platform is pinned via the `platform.commit` property in #{provider_json} " +
+              "to a commit in the range #{provider.platform.commit}, but the current HEAD " +
+              "(#{current_commit}) is not in that range.")
+            quit!("OK. Bye.") unless agree("Do you really want to deploy from the wrong commit? ")
+          end
+        end
+      end
+    end
+
     def sync_hiera_config(ssh)
-      dest_dir = provider.hiera_sync_destination
       ssh.rsync.update do |server|
         node = manager.node(server.host)
         hiera_file = Path.relative_path([:hiera, node.name])
-        ssh.leap.log hiera_file + ' -> ' + node.name + ':' + dest_dir + '/hiera.yaml'
+        ssh.leap.log hiera_file + ' -> ' + node.name + ':' + Leap::Platform.hiera_path
         {
           :source => hiera_file,
-          :dest => dest_dir + '/hiera.yaml',
+          :dest => Leap::Platform.hiera_path,
           :flags => "-rltp --chmod=u+rX,go-rwx"
         }
       end
     end
 
+    #
+    # sync various support files.
+    #
     def sync_support_files(ssh)
-      dest_dir = provider.hiera_sync_destination
+      dest_dir = Leap::Platform.files_dir
+      source_files = []
+      if Path.defined?(:custom_puppet_dir) && file_exists?(:custom_puppet_dir)
+        source_files += [:custom_puppet_dir, :custom_puppet_modules_dir, :custom_puppet_manifests_dir].collect{|path|
+          Path.relative_path(path, Path.provider) + '/' # rsync needs trailing slash
+        }
+        ensure_dir :custom_puppet_modules_dir
+      end
       ssh.rsync.update do |server|
         node = manager.node(server.host)
         files_to_sync = node.file_paths.collect {|path| Path.relative_path(path, Path.provider) }
+        files_to_sync += source_files
         if files_to_sync.any?
           ssh.leap.log(files_to_sync.join(', ') + ' -> ' + node.name + ':' + dest_dir)
           {
-            :chdir => Path.provider,
+            :chdir => Path.named_path(:files_dir),
             :source => ".",
             :dest => dest_dir,
             :excludes => "*",
-            :includes => calculate_includes_from_files(files_to_sync),
-            :flags => "-rltp --chmod=u+rX,go-rwx --relative --delete --delete-excluded --filter='protect hiera.yaml' --copy-links"
+            :includes => calculate_includes_from_files(files_to_sync, '/files'),
+            :flags => "-rltp --chmod=u+rX,go-rwx --relative --delete --delete-excluded --copy-links"
           }
         else
           nil
@@ -100,9 +206,9 @@ module LeapCli
 
     def sync_puppet_files(ssh)
       ssh.rsync.update do |server|
-        ssh.leap.log(Path.platform + '/[bin,tests,puppet] -> ' + server.host + ':' + LeapCli::PUPPET_DESTINATION)
+        ssh.leap.log(Path.platform + '/[bin,tests,puppet] -> ' + server.host + ':' + Leap::Platform.leap_dir)
         {
-          :dest => LeapCli::PUPPET_DESTINATION,
+          :dest => Leap::Platform.leap_dir,
           :source => '.',
           :chdir => Path.platform,
           :excludes => '*',
@@ -112,7 +218,12 @@ module LeapCli
       end
     end
 
+    #
+    # ensure submodules are up to date, if the platform is a git
+    # repository.
+    #
     def init_submodules
+      return unless is_git_directory?(Path.platform)
       Dir.chdir Path.platform do
         assert_run! "git submodule sync"
         statuses = assert_run! "git submodule status"
@@ -126,11 +237,17 @@ module LeapCli
       end
     end
 
-    def calculate_includes_from_files(files)
+    #
+    # converts an array of file paths into an array
+    # suitable for --include of rsync
+    #
+    # if set, `prefix` is stripped off.
+    #
+    def calculate_includes_from_files(files, prefix=nil)
       return nil unless files and files.any?
 
       # prepend '/' (kind of like ^ for rsync)
-      includes = files.collect {|file| '/' + file}
+      includes = files.collect {|file| file =~ /^\// ? file : '/' + file }
 
       # include all sub files of specified directories
       includes.size.times do |i|
@@ -148,6 +265,10 @@ module LeapCli
         end
       end
 
+      if prefix
+        includes.map! {|path| path.sub(/^#{Regexp.escape(prefix)}\//, '/')}
+      end
+
       return includes
     end
 
@@ -155,23 +276,11 @@ module LeapCli
       if options[:tags]
         tags = options[:tags].split(',')
       else
-        tags = LeapCli::DEFAULT_TAGS.dup
+        tags = Leap::Platform.default_puppet_tags.dup
       end
       tags << 'leap_slow' unless options[:fast]
       tags.join(',')
     end
 
-    #
-    # for safety, we allow production deploys to be turned off in the Leapfile.
-    #
-    def filter_deploy_nodes(filter)
-      nodes = manager.filter!(filter)
-      if !leapfile.allow_production_deploy
-        nodes = nodes[:environment => "!production"]
-        assert! nodes.any?, "Skipping deploy because @allow_production_deploy is disabled."
-      end
-      nodes
-    end
-
   end
 end

data/lib/leap_cli/commands/env.rb

@@ -0,0 +1,76 @@
+module LeapCli
+  module Commands
+
+    desc "Manipulate and query environment information."
+    long_desc "The 'environment' node property can be used to isolate sets of nodes into entirely separate environments. "+
+      "A node in one environment will never interact with a node from another environment. "+
+      "Environment pinning works by modifying your ~/.leaprc file and is dependent on the "+
+      "absolute file path of your provider directory (pins don't apply if you move the directory)"
+    command [:env, :e] do |c|
+      c.desc "List the available environments. The pinned environment, if any, will be marked with '*'. Will also set the pin if run with an environment argument."
+      c.arg_name 'ENVIRONMENT', :optional => true
+      c.command :ls do |ls|
+        ls.action do |global_options, options, args|
+          environment = get_env_from_args(args)
+          if environment
+            pin(environment)
+            LeapCli.leapfile.load
+          end
+          print_envs
+        end
+      end
+
+      c.desc 'Pin the environment to ENVIRONMENT. All subsequent commands will only apply to nodes in this environment.'
+      c.arg_name 'ENVIRONMENT'
+      c.command :pin do |pin|
+        pin.action do |global_options,options,args|
+          environment = get_env_from_args(args)
+          if environment
+            pin(environment)
+          else
+            bail! "There is no environment `#{environment}`"
+          end
+        end
+      end
+
+      c.desc "Unpin the environment. All subsequent commands will apply to all nodes."
+      c.command :unpin do |unpin|
+        unpin.action do |global_options, options, args|
+          LeapCli.leapfile.unset('environment')
+          log 0, :saved, "~/.leaprc, removing environment property."
+        end
+      end
+
+      c.default_command :ls
+    end
+
+    protected
+
+    def get_env_from_args(args)
+      environment = args.first
+      if environment == 'default' || (environment && manager.environment_names.include?(environment))
+        return environment
+      else
+        return nil
+      end
+    end
+
+    def pin(environment)
+      LeapCli.leapfile.set('environment', environment)
+      log 0, :saved, "~/.leaprc with environment set to #{environment}."
+    end
+
+    def print_envs
+      envs = ["default"] + manager.environment_names.compact.sort
+      envs.each do |env|
+        if env
+          if LeapCli.leapfile.environment == env
+            puts "* #{env}"
+          else
+            puts "  #{env}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/leap_cli/commands/facts.rb

@@ -79,14 +79,21 @@ module LeapCli; module Commands
   private
 
   def update_facts(global_options, options, args)
-    nodes = manager.filter(args)
+    nodes = manager.filter(args, :local => false)
     new_facts = {}
     ssh_connect(nodes) do |ssh|
       ssh.leap.run_with_progress(facter_cmd) do |response|
-        new_facts[response[:host]] = response[:data].strip
+        node = manager.node(response[:host])
+        if node
+          new_facts[node.name] = response[:data].strip
+        else
+          log :warning, 'Could not find node for hostname %s' % response[:host]
+        end
       end
     end
-    overwrite_existing = args.empty?
+    # only overwrite the entire facts file if and only if we are gathering facts
+    # for all nodes in all environments.
+    overwrite_existing = args.empty? && LeapCli.leapfile.environment.nil?
     update_facts_file(new_facts, overwrite_existing)
   end
 

data/lib/leap_cli/commands/inspect.rb

@@ -2,7 +2,7 @@ module LeapCli; module Commands
 
   desc 'Prints details about a file. Alternately, the argument FILE can be the name of a node, service or tag.'
   arg_name 'FILE'
-  command :inspect do |c|
+  command [:inspect, :i] do |c|
     c.switch 'base', :desc => 'Inspect the FILE from the provider_base (i.e. without local inheritance).', :negatable => false
     c.action do |global_options,options,args|
       object = args.first
@@ -109,7 +109,7 @@ module LeapCli; module Commands
     if options[:base]
       inspect_json manager.base_provider
     elsif arg =~ /provider\.(.*)\.json/
-      inspect_json manager.providers[$1]
+      inspect_json manager.env($1).provider
     else
       inspect_json manager.provider
     end

data/lib/leap_cli/commands/list.rb

@@ -11,28 +11,29 @@ module LeapCli; module Commands
             "`leap list openvpn +local` matches all nodes with service \"openvpn\" AND tag \"local\""
 
   arg_name 'FILTER', :optional => true
-  command :list do |c|
+  command [:list,:ls] do |c|
     c.flag 'print', :desc => 'What attributes to print (optional)'
     c.switch 'disabled', :desc => 'Include disabled nodes in the list.', :negatable => false
     c.action do |global_options,options,args|
+      # don't rely on default manager(), because we want to pass custom options to load()
+      manager = LeapCli::Config::Manager.new
       if global_options[:color]
         colors = ['cyan', 'white']
       else
         colors = [nil, nil]
       end
       puts
-      if options['disabled']
-        manager.load(:include_disabled => true) # reload, with disabled nodes
-      end
+      manager.load(:include_disabled => options['disabled'], :continue_on_error => true)
       if options['print']
         print_node_properties(manager.filter(args), options['print'])
       else
         if args.any?
           NodeTable.new(manager.filter(args), colors).run
         else
-          TagTable.new('SERVICES', manager.services, colors).run
-          TagTable.new('TAGS', manager.tags, colors).run
-          NodeTable.new(manager.nodes, colors).run
+          environment = LeapCli.leapfile.environment || '_all_'
+          TagTable.new('SERVICES', manager.env(environment).services, colors).run
+          TagTable.new('TAGS', manager.env(environment).tags, colors).run
+          NodeTable.new(manager.filter(), colors).run
         end
       end
     end
@@ -41,11 +42,9 @@ module LeapCli; module Commands
   private
 
   def self.print_node_properties(nodes, properties)
-    node_list = manager.nodes
     properties = properties.split(',')
     max_width = nodes.keys.inject(0) {|max,i| [i.size,max].max}
     nodes.each_node do |node|
-      node.evaluate
       value = properties.collect{|prop|
         if node[prop].nil?
           "null"
@@ -68,7 +67,7 @@ module LeapCli; module Commands
       @colors = colors
     end
     def run
-      tags = @tag_list.keys.sort
+      tags = @tag_list.keys.select{|tag| tag !~ /^_/}.sort # sorted list of tags, excluding _partials
       max_width = [20, (tags+[@heading]).inject(0) {|max,i| [i.size,max].max}].max
       table :border => false do
         row :color => @colors[0]  do
@@ -76,6 +75,7 @@ module LeapCli; module Commands
           column "NODES", :width => HighLine::SystemExtensions.terminal_size.first - max_width - 2, :padding => 2
         end
         tags.each do |tag|
+          next if @tag_list[tag].node_list.empty?
           row :color => @colors[1] do
             column tag
             column @tag_list[tag].node_list.keys.sort.join(', ')

data/lib/leap_cli/commands/node.rb

@@ -1,6 +1,9 @@
-require 'net/ssh/known_hosts'
-require 'tempfile'
-require 'ipaddr'
+#
+# fyi: the `node init` command lives in node_init.rb,
+#      but all other `node x` commands live here.
+#
+
+autoload :IPAddr, 'ipaddr'
 
 module LeapCli; module Commands
 
@@ -9,7 +12,7 @@ module LeapCli; module Commands
   ##
 
   desc 'Node management'
-  command :node do |node|
+  command [:node, :n] do |node|
     node.desc 'Create a new configuration file for a node named NAME.'
     node.long_desc ["If specified, the optional argument SEED can be used to seed values in the node configuration file.",
                     "The format is property_name:value.",
@@ -44,45 +47,6 @@ module LeapCli; module Commands
       end
     end
 
-    node.desc 'Bootstraps a node or nodes, setting up SSH keys and installing prerequisite packages'
-    node.long_desc "This command prepares a server to be used with the LEAP Platform by saving the server's SSH host key, " +
-                   "copying the authorized_keys file, installing packages that are required for deploying, and registering important facts. " +
-                   "Node init must be run before deploying to a server, and the server must be running and available via the network. " +
-                   "This command only needs to be run once, but there is no harm in running it multiple times."
-    node.arg_name 'FILTER' #, :optional => false, :multiple => false
-    node.command :init do |init|
-      init.switch 'echo', :desc => 'If set, passwords are visible as you type them (default is hidden)', :negatable => false
-      init.flag :port, :desc => 'Override the default SSH port.', :arg_name => 'PORT'
-      init.flag :ip,   :desc => 'Override the default SSH IP address.', :arg_name => 'IPADDRESS'
-
-      init.action do |global,options,args|
-        assert! args.any?, 'You must specify a FILTER'
-        finished = []
-        manager.filter!(args).each_node do |node|
-          is_node_alive(node, options)
-          save_public_host_key(node, global, options) unless node.vagrant?
-          update_compiled_ssh_configs
-          ssh_connect_options = connect_options(options).merge({:bootstrap => true, :echo => options[:echo]})
-          ssh_connect(node, ssh_connect_options) do |ssh|
-            if node.vagrant?
-              ssh.install_insecure_vagrant_key
-            end
-            ssh.install_authorized_keys
-            ssh.install_prerequisites
-            ssh.leap.capture(facter_cmd) do |response|
-              if response[:exitcode] == 0
-                update_node_facts(node.name, response[:data])
-              else
-                log :failed, "to run facter on #{node.name}"
-              end
-            end
-          end
-          finished << node.name
-        end
-        log :completed, "initialization of nodes #{finished.join(', ')}"
-      end
-    end
-
     node.desc 'Renames a node file, and all its related files.'
     node.arg_name 'OLD_NAME NEW_NAME'
     node.command :mv do |mv|
@@ -117,30 +81,6 @@ module LeapCli; module Commands
   ## PUBLIC HELPERS
   ##
 
-  #
-  # generates the known_hosts file.
-  #
-  # we do a 'late' binding on the hostnames and ip part of the ssh pub key record in order to allow
-  # for the possibility that the hostnames or ip has changed in the node configuration.
-  #
-  def update_known_hosts
-    buffer = StringIO.new
-    buffer << "#\n"
-    buffer << "# This file is automatically generated by the command `leap`. You should NOT modify this file.\n"
-    buffer << "# Instead, rerun `leap node init` on whatever node is causing SSH problems.\n"
-    buffer << "#\n"
-    manager.nodes.keys.sort.each do |node_name|
-      node = manager.nodes[node_name]
-      hostnames = [node.name, node.domain.internal, node.domain.full, node.ip_address].join(',')
-      pub_key = read_file([:node_ssh_pub_key,node.name])
-      if pub_key
-        buffer << [hostnames, pub_key].join(' ')
-        buffer << "\n"
-      end
-    end
-    write_file!(:known_hosts, buffer.string)
-  end
-
   def get_node_from_args(args, options={})
     node_name = args.first
     node = manager.node(node_name)
@@ -151,71 +91,6 @@ module LeapCli; module Commands
     node
   end
 
-  private
-
-  ##
-  ## PRIVATE HELPERS
-  ##
-
-  #
-  # saves the public ssh host key for node into the provider directory.
-  #
-  # see `man sshd` for the format of known_hosts
-  #
-  def save_public_host_key(node, global, options)
-    log :fetching, "public SSH host key for #{node.name}"
-    address = options[:ip] || node.ip_address
-    port = options[:port] || node.ssh.port
-    public_key = get_public_key_for_ip(address, port)
-    pub_key_path = Path.named_path([:node_ssh_pub_key, node.name])
-    if Path.exists?(pub_key_path)
-      if public_key == SshKey.load(pub_key_path)
-        log :trusted, "- Public SSH host key for #{node.name} matches previously saved key", :indent => 1
-      else
-        bail! do
-          log :error, "The public SSH host key we just fetched for #{node.name} doesn't match what we have saved previously.", :indent => 1
-          log "Remove the file #{pub_key_path} if you really want to change it.", :indent => 2
-        end
-      end
-    elsif public_key.in_known_hosts?(node.name, node.ip_address, node.domain.name)
-      log :trusted, "- Public SSH host key for #{node.name} is trusted (key found in your ~/.ssh/known_hosts)"
-    else
-      puts
-      say("This is the SSH host key you got back from node \"#{node.name}\"")
-      say("Type        -- #{public_key.bits} bit #{public_key.type.upcase}")
-      say("Fingerprint -- " + public_key.fingerprint)
-      say("Public Key  -- " + public_key.key)
-      if !global[:yes] && !agree("Is this correct? ")
-        bail!
-      else
-        puts
-        write_file! [:node_ssh_pub_key, node.name], public_key.to_s
-      end
-    end
-  end
-
-  def get_public_key_for_ip(address, port=22)
-    assert_bin!('ssh-keyscan')
-    output = assert_run! "ssh-keyscan -p #{port} -t ecdsa #{address}", "Could not get the public host key from #{address}:#{port}. Maybe sshd is not running?"
-    line = output.split("\n").grep(/^[^#]/).first
-    if line =~ /No route to host/
-      bail! :failed, 'ssh-keyscan: no route to %s' % address
-    elsif line =~ /no hostkey alg/
-      bail! :failed, 'ssh-keyscan: no hostkey alg (must be missing an ecdsa public host key)'
-    end
-    assert! line, "Got zero host keys back!"
-    ip, key_type, public_key = line.split(' ')
-    return SshKey.load(public_key, key_type)
-  end
-
-  def is_node_alive(node, options)
-    address = options[:ip] || node.ip_address
-    port = options[:port] || node.ssh.port
-    log :connecting, "to node #{node.name}"
-    assert_run! "nc -zw3 #{address} #{port}",
-      "Failed to reach #{node.name} (address #{address}, port #{port}). You can override the configured IP address and port with --ip or --port."
-  end
-
   def seed_node_data(node, args)
     args.each do |seed|
       key, value = seed.split(':')