leap_cli 1.2.5

data/lib/leap_cli/commands/new.rb

@@ -0,0 +1,126 @@
+require 'fileutils'
+
+module LeapCli; module Commands
+
+  desc 'Creates a new provider instance in the specified directory, creating it if necessary.'
+  arg_name 'DIRECTORY'
+  skips_pre
+  command :new do |c|
+    c.flag 'name', :desc => "The name of the provider." #, :default_value => 'Example'
+    c.flag 'domain', :desc => "The primary domain of the provider." #, :default_value => 'example.org'
+    c.flag 'platform', :desc => "File path of the leap_platform directory." #, :default_value => '../leap_platform'
+    c.flag 'contacts', :desc => "Default email address contacts." #, :default_value => 'root'
+
+    c.action do |global, options, args|
+      directory = File.expand_path(args.first)
+      create_provider_directory(global, directory)
+      options[:domain]   ||= ask("The primary domain of the provider: ") {|q| q.default = 'example.org'}
+      options[:name]     ||= ask("The name of the provider: ") {|q| q.default = 'Example'}
+      options[:platform] ||= ask("File path of the leap_platform directory: ") {|q| q.default = File.expand_path('../leap_platform', directory)}
+      options[:contacts] ||= ask("Default email address contacts: ") {|q| q.default = 'root@' + options[:domain]}
+      options[:platform] = relative_path(options[:platform])
+      create_initial_provider_files(directory, global, options)
+    end
+  end
+
+  private
+
+  DEFAULT_REPO = 'https://leap.se/git/leap_platform.git'
+
+  #
+  # creates a new provider directory
+  #
+  def create_provider_directory(global, directory)
+    unless directory && directory.any?
+      help! "Directory name is required."
+    end
+    unless File.exists?(directory)
+      if global[:yes] || agree("Create directory #{directory}? ")
+        ensure_dir directory
+      else
+        bail! { log :missing, "directory #{directory}" }
+      end
+    end
+    Path.set_provider_path(directory)
+  end
+
+  #
+  # see provider with initial files
+  #
+  def create_initial_provider_files(directory, global, options)
+    Dir.chdir(directory) do
+      assert_files_missing! 'provider.json', 'common.json', 'Leapfile', :base => directory
+
+      platform_dir = File.expand_path(options[:platform], "./")
+      unless File.symlink?(platform_dir) || File.directory?(platform_dir)
+        if global[:yes] || agree("The platform directory \"#{platform_dir}\" does not exist.\nDo you want me to create it by cloning from the\ngit repository #{DEFAULT_REPO}? ")
+          assert_bin! 'git'
+          ensure_dir platform_dir
+          Dir.chdir(platform_dir) do
+            log :cloning, "leap_platform into #{platform_dir}"
+            pty_run "git clone --branch master #{DEFAULT_REPO} ."
+            pty_run 'git submodule update --init'
+          end
+        else
+          bail!
+        end
+      end
+      write_file! '.gitignore', GITIGNORE_CONTENT
+      write_file! 'provider.json', provider_content(options)
+      write_file! 'common.json', COMMON_CONTENT
+      write_file! 'Leapfile', leapfile_content(options)
+      ["nodes", "services", "tags"].each do |dir|
+        ensure_dir dir
+      end
+      log :completed, 'initialization'
+    end
+  end
+
+  def relative_path(path)
+    Pathname.new(path).relative_path_from(Pathname.new(Path.provider)).to_s
+  end
+
+  def leapfile_content(options)
+    %[@platform_directory_path = "#{options[:platform]}"\n# see https://leap.se/en/docs/platform/config for more options]
+  end
+
+  GITIGNORE_CONTENT = <<EOS
+test/Vagrantfile
+test/.vagrant
+test/openvpn
+test/cert
+EOS
+
+  def provider_content(options)
+  %[//
+// General service provider configuration.
+//
+{
+  "domain": "#{options[:domain]}",
+  "name": {
+    "en": "#{options[:name]}"
+  },
+  "description": {
+    "en": "You really should change this text"
+  },
+  "contacts": {
+    "default": "#{options[:contacts]}"
+  },
+  "languages": ["en"],
+  "default_language": "en",
+  "enrollment_policy": "open"
+}
+]
+  end
+
+  COMMON_CONTENT = <<EOS
+//
+// Options put here are inherited by all nodes.
+//
+{
+}
+EOS
+
+end; end
+
+

data/lib/leap_cli/commands/node.rb

@@ -0,0 +1,272 @@
+require 'net/ssh/known_hosts'
+require 'tempfile'
+require 'ipaddr'
+
+module LeapCli; module Commands
+
+  ##
+  ## COMMANDS
+  ##
+
+  desc 'Node management'
+  command :node do |node|
+    node.desc 'Create a new configuration file for a node named NAME.'
+    node.long_desc ["If specified, the optional argument SEED can be used to seed values in the node configuration file.",
+                    "The format is property_name:value.",
+                    "For example: `leap node add web1 ip_address:1.2.3.4 services:webapp`.",
+                    "To set nested properties, property name can contain '.', like so: `leap node add web1 ssh.port:44`",
+                    "Separeate multiple values for a single property with a comma, like so: `leap node add mynode services:webapp,dns`"].join("\n\n")
+    node.arg_name 'NAME [SEED]' # , :optional => false, :multiple => false
+    node.command :add do |add|
+      add.switch :local, :desc => 'Make a local testing node (by automatically assigning the next available local IP address). Local nodes are run as virtual machines on your computer.', :negatable => false
+      add.action do |global_options,options,args|
+        # argument sanity checks
+        name = args.first
+        assert! name, 'No <node-name> specified.'
+        assert! name =~ /^[0-9a-z-]+$/, "illegal characters used in node name '#{name}'"
+        assert_files_missing! [:node_config, name]
+
+        # create and seed new node
+        node = Config::Node.new(manager)
+        if options[:local]
+          node['ip_address'] = pick_next_vagrant_ip_address
+        end
+        seed_node_data(node, args[1..-1])
+        validate_ip_address(node)
+
+        # write the file
+        write_file! [:node_config, name], node.dump_json + "\n"
+        node['name'] = name
+        if file_exists? :ca_cert, :ca_key
+          generate_cert_for_node(manager.reload_node(node))
+        end
+      end
+    end
+
+    node.desc 'Bootstraps a node or nodes, setting up SSH keys and installing prerequisite packages'
+    node.long_desc "This command prepares a server to be used with the LEAP Platform by saving the server's SSH host key, " +
+                   "copying the authorized_keys file, installing packages that are required for deploying, and registering important facts. " +
+                   "Node init must be run before deploying to a server, and the server must be running and available via the network. " +
+                   "This command only needs to be run once, but there is no harm in running it multiple times."
+    node.arg_name 'FILTER' #, :optional => false, :multiple => false
+    node.command :init do |init|
+      init.switch 'echo', :desc => 'If set, passwords are visible as you type them (default is hidden)', :negatable => false
+      init.flag :port, :desc => 'Override the default SSH port.', :arg_name => 'PORT'
+      init.flag :ip,   :desc => 'Override the default SSH IP address.', :arg_name => 'IPADDRESS'
+
+      init.action do |global,options,args|
+        assert! args.any?, 'You must specify a FILTER'
+        finished = []
+        manager.filter!(args).each_node do |node|
+          is_node_alive(node, options)
+          save_public_host_key(node, global, options) unless node.vagrant?
+          update_compiled_ssh_configs
+          ssh_connect_options = connect_options(options).merge({:bootstrap => true, :echo => options[:echo]})
+          ssh_connect(node, ssh_connect_options) do |ssh|
+            ssh.install_authorized_keys
+            ssh.install_prerequisites
+            ssh.leap.capture(facter_cmd) do |response|
+              if response[:exitcode] == 0
+                update_node_facts(node.name, response[:data])
+              else
+                log :failed, "to run facter on #{node.name}"
+              end
+            end
+          end
+          finished << node.name
+        end
+        log :completed, "initialization of nodes #{finished.join(', ')}"
+      end
+    end
+
+    node.desc 'Renames a node file, and all its related files.'
+    node.arg_name 'OLD_NAME NEW_NAME'
+    node.command :mv do |mv|
+      mv.action do |global_options,options,args|
+        node = get_node_from_args(args)
+        new_name = args.last
+        ensure_dir [:node_files_dir, new_name]
+        Leap::Platform.node_files.each do |path|
+          rename_file! [path, node.name], [path, new_name]
+        end
+        remove_directory! [:node_files_dir, node.name]
+        rename_node_facts(node.name, new_name)
+      end
+    end
+
+    node.desc 'Removes all the files related to the node named NAME.'
+    node.arg_name 'NAME' #:optional => false #, :multiple => false
+    node.command :rm do |rm|
+      rm.action do |global_options,options,args|
+        node = get_node_from_args(args)
+        (Leap::Platform.node_files + [:node_files_dir]).each do |path|
+          remove_file! [path, node.name]
+        end
+        if node.vagrant?
+          vagrant_command("destroy --force", [node.name])
+        end
+        remove_node_facts(node.name)
+      end
+    end
+  end
+
+  ##
+  ## PUBLIC HELPERS
+  ##
+
+  #
+  # generates the known_hosts file.
+  #
+  # we do a 'late' binding on the hostnames and ip part of the ssh pub key record in order to allow
+  # for the possibility that the hostnames or ip has changed in the node configuration.
+  #
+  def update_known_hosts
+    buffer = StringIO.new
+    buffer << "#\n"
+    buffer << "# This file is automatically generated by the command `leap`. You should NOT modify this file.\n"
+    buffer << "# Instead, rerun `leap node init` on whatever node is causing SSH problems.\n"
+    buffer << "#\n"
+    manager.nodes.keys.sort.each do |node_name|
+      node = manager.nodes[node_name]
+      hostnames = [node.name, node.domain.internal, node.domain.full, node.ip_address].join(',')
+      pub_key = read_file([:node_ssh_pub_key,node.name])
+      if pub_key
+        buffer << [hostnames, pub_key].join(' ')
+        buffer << "\n"
+      end
+    end
+    write_file!(:known_hosts, buffer.string)
+  end
+
+  def get_node_from_args(args, options={})
+    node_name = args.first
+    node = manager.node(node_name)
+    if node.nil? && options[:include_disabled]
+      node = manager.disabled_node(node_name)
+    end
+    assert!(node, "Node '#{node_name}' not found.")
+    node
+  end
+
+  private
+
+  ##
+  ## PRIVATE HELPERS
+  ##
+
+  #
+  # saves the public ssh host key for node into the provider directory.
+  #
+  # see `man sshd` for the format of known_hosts
+  #
+  def save_public_host_key(node, global, options)
+    log :fetching, "public SSH host key for #{node.name}"
+    address = options[:ip] || node.ip_address
+    port = options[:port] || node.ssh.port
+    public_key = get_public_key_for_ip(address, port)
+    pub_key_path = Path.named_path([:node_ssh_pub_key, node.name])
+    if Path.exists?(pub_key_path)
+      if public_key == SshKey.load(pub_key_path)
+        log :trusted, "- Public SSH host key for #{node.name} matches previously saved key", :indent => 1
+      else
+        bail! do
+          log :error, "The public SSH host key we just fetched for #{node.name} doesn't match what we have saved previously.", :indent => 1
+          log "Remove the file #{pub_key_path} if you really want to change it.", :indent => 2
+        end
+      end
+    elsif public_key.in_known_hosts?(node.name, node.ip_address, node.domain.name)
+      log :trusted, "- Public SSH host key for #{node.name} is trusted (key found in your ~/.ssh/known_hosts)"
+    else
+      puts
+      say("This is the SSH host key you got back from node \"#{node.name}\"")
+      say("Type        -- #{public_key.bits} bit #{public_key.type.upcase}")
+      say("Fingerprint -- " + public_key.fingerprint)
+      say("Public Key  -- " + public_key.key)
+      if !global[:yes] && !agree("Is this correct? ")
+        bail!
+      else
+        puts
+        write_file! [:node_ssh_pub_key, node.name], public_key.to_s
+      end
+    end
+  end
+
+  def get_public_key_for_ip(address, port=22)
+    assert_bin!('ssh-keyscan')
+    output = assert_run! "ssh-keyscan -p #{port} -t ecdsa #{address}", "Could not get the public host key from #{address}:#{port}. Maybe sshd is not running?"
+    line = output.split("\n").grep(/^[^#]/).first
+    if line =~ /No route to host/
+      bail! :failed, 'ssh-keyscan: no route to %s' % address
+    elsif line =~ /no hostkey alg/
+      bail! :failed, 'ssh-keyscan: no hostkey alg (must be missing an ecdsa public host key)'
+    end
+    assert! line, "Got zero host keys back!"
+    ip, key_type, public_key = line.split(' ')
+    return SshKey.load(public_key, key_type)
+  end
+
+  def is_node_alive(node, options)
+    address = options[:ip] || node.ip_address
+    port = options[:port] || node.ssh.port
+    log :connecting, "to node #{node.name}"
+    assert_run! "nc -zw3 #{address} #{port}",
+      "Failed to reach #{node.name} (address #{address}, port #{port}). You can override the configured IP address and port with --ip or --port."
+  end
+
+  def seed_node_data(node, args)
+    args.each do |seed|
+      key, value = seed.split(':')
+      value = format_seed_value(value)
+      assert! key =~ /^[0-9a-z\._]+$/, "illegal characters used in property '#{key}'"
+      if key =~ /\./
+        key_parts = key.split('.')
+        final_key = key_parts.pop
+        current_object = node
+        key_parts.each do |key_part|
+          current_object[key_part] ||= Config::Object.new
+          current_object = current_object[key_part]
+        end
+        current_object[final_key] = value
+      else
+        node[key] = value
+      end
+    end
+  end
+
+  #
+  # conversations:
+  #
+  #   "x,y,z" => ["x","y","z"]
+  #
+  #   "22" => 22
+  #
+  #   "5.1" => 5.1
+  #
+  def format_seed_value(v)
+    if v =~ /,/
+      v = v.split(',')
+      v.map! do |i|
+        i = i.to_i if i.to_i.to_s == i
+        i = i.to_f if i.to_f.to_s == i
+        i
+      end
+    else
+      v = v.to_i if v.to_i.to_s == v
+      v = v.to_f if v.to_f.to_s == v
+    end
+    return v
+  end
+
+  def validate_ip_address(node)
+    IPAddr.new(node['ip_address'])
+  rescue ArgumentError
+    bail! do
+      if node['ip_address']
+        log :invalid, "ip_address #{node['ip_address'].inspect}"
+      else
+        log :missing, "ip_address"
+      end
+    end
+  end
+
+end; end

data/lib/leap_cli/commands/pre.rb

@@ -0,0 +1,99 @@
+
+#
+# check to make sure we can find the root directory of the platform
+#
+module LeapCli; module Commands
+
+  desc 'Verbosity level 0..5'
+  arg_name 'LEVEL'
+  default_value '1'
+  flag [:v, :verbose]
+
+  desc 'Override default log file'
+  arg_name 'FILE'
+  default_value nil
+  flag :log
+
+  desc 'Display version number and exit'
+  switch :version, :negatable => false
+
+  desc 'Skip prompts and assume "yes"'
+  switch :yes, :negatable => false
+
+  pre do |global,command,options,args|
+    #
+    # set verbosity
+    #
+    LeapCli.log_level = global[:verbose].to_i
+    if LeapCli.log_level > 1
+      ENV['GLI_DEBUG'] = "true"
+    else
+      ENV['GLI_DEBUG'] = "false"
+    end
+
+    #
+    # load Leapfile
+    #
+    unless LeapCli.leapfile.load
+      bail! { log :missing, 'Leapfile in directory tree' }
+    end
+    Path.set_platform_path(LeapCli.leapfile.platform_directory_path)
+    Path.set_provider_path(LeapCli.leapfile.provider_directory_path)
+    if !Path.provider || !File.directory?(Path.provider)
+      bail! { log :missing, "provider directory '#{Path.provider}'" }
+    end
+    if !Path.platform || !File.directory?(Path.platform)
+      bail! { log :missing, "platform directory '#{Path.platform}'" }
+    end
+
+    if LeapCli.leapfile.platform_branch && LeapCli::Util.is_git_directory?(Path.platform)
+      branch = LeapCli::Util.current_git_branch(Path.platform)
+      if branch != LeapCli.leapfile.platform_branch
+        bail! "Wrong branch for #{Path.platform}. Was '#{branch}', should be '#{LeapCli.leapfile.platform_branch}'. Edit Leapfile to disable this check."
+      end
+    end
+
+    #
+    # set log file
+    #
+    LeapCli.log_file = global[:log] || LeapCli.leapfile.log
+    LeapCli::Util.log_raw(:log) { $0 + ' ' + ORIGINAL_ARGV.join(' ')}
+    log_version
+
+    #
+    # load all the nodes everything
+    #
+    manager
+
+    #
+    # check requirements
+    #
+    REQUIREMENTS.each do |key|
+      assert_config! key
+    end
+
+  end
+
+  private
+
+  #
+  # add a log entry for the leap command and leap platform versions
+  #
+  def log_version
+    if LeapCli.log_level >= 2
+      str = "leap command v#{LeapCli::VERSION}"
+      cli_dir = File.dirname(__FILE__)
+      if Util.is_git_directory?(cli_dir)
+        str << " (%s %s)" % [Util.current_git_branch(cli_dir), Util.current_git_commit(cli_dir)]
+      end
+      log 2, str
+      str = "leap platform v#{Leap::Platform.version}"
+      if Util.is_git_directory?(Path.platform)
+        str << " (%s %s)" % [Util.current_git_branch(Path.platform), Util.current_git_commit(Path.platform)]
+      end
+      log 2, str
+    end
+  end
+
+
+end; end

data/lib/leap_cli/commands/shell.rb

@@ -0,0 +1,67 @@
+module LeapCli; module Commands
+
+  desc 'Log in to the specified node with an interactive shell.'
+  arg_name 'NAME' #, :optional => false, :multiple => false
+  command :ssh do |c|
+    c.action do |global_options,options,args|
+      exec_ssh(:ssh, args)
+    end
+  end
+
+  desc 'Log in to the specified node with an interactive shell using mosh (requires node to have mosh.enabled set to true).'
+  arg_name 'NAME'
+  command :mosh do |c|
+    c.action do |global_options,options,args|
+      exec_ssh(:mosh, args)
+    end
+  end
+
+  protected
+
+  #
+  # allow for ssh overrides of all commands that use ssh_connect
+  #
+  def connect_options(options)
+    connect_options = {:ssh_options=>{}}
+    if options[:port]
+      connect_options[:ssh_options][:port] = options[:port]
+    end
+    if options[:ip]
+      connect_options[:ssh_options][:host_name] = options[:ip]
+    end
+    return connect_options
+  end
+
+  private
+
+  def exec_ssh(cmd, args)
+    node = get_node_from_args(args, :include_disabled => true)
+    options = [
+      "-o 'HostName=#{node.ip_address}'",
+      # "-o 'HostKeyAlias=#{node.name}'", << oddly incompatible with ports in known_hosts file, so we must not use this or non-standard ports break.
+      "-o 'GlobalKnownHostsFile=#{path(:known_hosts)}'",
+      "-o 'UserKnownHostsFile=/dev/null'"
+    ]
+    if node.vagrant?
+      options << "-i #{vagrant_ssh_key_file}"
+      options << "-o 'StrictHostKeyChecking=no'" # blindly accept host key and don't save it (since userknownhostsfile is /dev/null)
+    else
+      options << "-o 'StrictHostKeyChecking=yes'"
+    end
+    username = 'root'
+    if LeapCli.log_level >= 3
+      options << "-vv"
+    elsif LeapCli.log_level >= 2
+      options << "-v"
+    end
+    ssh = "ssh -l #{username} -p #{node.ssh.port} #{options.join(' ')}"
+    if cmd == :ssh
+      command = "#{ssh} #{node.name}"
+    elsif cmd == :mosh
+      command = "MOSH_TITLE_NOPREFIX=1 mosh --ssh \"#{ssh}\" #{node.name}"
+    end
+    log 2, command
+    exec "#{command}"
+  end
+
+end; end

data/lib/leap_cli/commands/test.rb

@@ -0,0 +1,55 @@
+module LeapCli; module Commands
+
+  desc 'Run tests.'
+  command :test do |test|
+    test.desc 'Creates files needed to run tests.'
+    test.command :init do |init|
+      init.action do |global_options,options,args|
+        generate_test_client_openvpn_configs
+      end
+    end
+
+    test.desc 'Run tests.'
+    test.command :run do |run|
+      run.action do |global_options,options,args|
+        manager.filter!(args).each_node do |node|
+          ssh_connect(node) do |ssh|
+            ssh.run(test_cmd)
+          end
+        end
+      end
+    end
+
+    test.default_command :run
+  end
+
+  private
+
+  def test_cmd
+    "#{PUPPET_DESTINATION}/bin/run_tests"
+  end
+
+  #
+  # generates a whole bunch of openvpn configs that can be used to connect to different openvpn gateways
+  #
+  def generate_test_client_openvpn_configs
+    assert_config! 'provider.ca.client_certificates.unlimited_prefix'
+    assert_config! 'provider.ca.client_certificates.limited_prefix'
+    template = read_file! Path.find_file(:test_client_openvpn_template)
+    manager.environments.each do |env|
+      vpn_nodes = manager.nodes[:environment => env][:services => 'openvpn']['openvpn.allow_limited' => true]
+      if vpn_nodes.any?
+        generate_test_client_cert(provider.ca.client_certificates.limited_prefix) do |key, cert|
+          write_file! [:test_openvpn_config, [env, 'limited'].compact.join('_')], Util.erb_eval(template, binding)
+        end
+      end
+      vpn_nodes = manager.nodes[:environment => env][:services => 'openvpn']['openvpn.allow_unlimited' => true]
+      if vpn_nodes.any?
+        generate_test_client_cert(provider.ca.client_certificates.unlimited_prefix) do |key, cert|
+          write_file! [:test_openvpn_config, [env, 'unlimited'].compact.join('_')], Util.erb_eval(template, binding)
+        end
+      end
+    end
+  end
+
+end; end