leap_cli 1.5.1 → 1.5.6

data/lib/leap_cli.rb

@@ -7,6 +7,7 @@ require 'leap/platform.rb'
 require 'leap_cli/version.rb'
 require 'leap_cli/constants.rb'
 require 'leap_cli/requirements.rb'
+require 'leap_cli/exceptions.rb'
 
 require 'leap_cli/leapfile.rb'
 require 'core_ext/hash'

data/lib/leap_cli/commands/compile.rb

@@ -125,7 +125,7 @@ module LeapCli
       end
 
       # all other records
-      manager.environments.each do |env|
+      manager.environment_names.each do |env|
         next if env == 'local'
         nodes = manager.nodes[:environment => env]
         next unless nodes.any?

data/lib/leap_cli/commands/db.rb

@@ -0,0 +1,20 @@
+module LeapCli; module Commands
+
+  desc 'Database commands.'
+  command :db do |db|
+    db.desc 'Destroy all the databases.'
+    db.command :destroy do |destroy|
+      destroy.action do |global_options,options,args|
+        say 'You are about to permanently destroy all database data.'
+        return unless agree("Continue? ")
+        nodes = manager.nodes[:services => 'couchdb']
+        ssh_connect(nodes, connect_options(options)) do |ssh|
+          ssh.run('/etc/init.d/bigcouch stop && test ! -z "$(ls /opt/bigcouch/var/lib/ 2> /dev/null)" && rm -r /opt/bigcouch/var/lib/* && echo "db destroyed" || echo "db already destroyed"')
+        end
+      end
+    end
+  end
+
+  private
+
+end; end

data/lib/leap_cli/commands/deploy.rb

@@ -38,7 +38,7 @@ module LeapCli
           end
         end
 
-        compile_hiera_files(nodes)
+        compile_hiera_files
 
         ssh_connect(nodes, connect_options(options)) do |ssh|
           ssh.leap.log :checking, 'node' do
@@ -54,7 +54,7 @@ module LeapCli
           end
           unless options[:sync]
             ssh.leap.log :applying, "puppet" do
-              ssh.puppet.apply(:verbosity => LeapCli.log_level, :tags => tags(options), :force => options[:force])
+              ssh.puppet.apply(:verbosity => [LeapCli.log_level,5].min, :tags => tags(options), :force => options[:force])
             end
           end
         end

data/lib/leap_cli/commands/list.rb

@@ -15,6 +15,11 @@ module LeapCli; module Commands
     c.flag 'print', :desc => 'What attributes to print (optional)'
     c.switch 'disabled', :desc => 'Include disabled nodes in the list.', :negatable => false
     c.action do |global_options,options,args|
+      if global_options[:color]
+        colors = ['cyan', 'white']
+      else
+        colors = [nil, nil]
+      end
       puts
       if options['disabled']
         manager.load(:include_disabled => true) # reload, with disabled nodes
@@ -23,11 +28,11 @@ module LeapCli; module Commands
         print_node_properties(manager.filter(args), options['print'])
       else
         if args.any?
-          NodeTable.new(manager.filter(args)).run
+          NodeTable.new(manager.filter(args), colors).run
         else
-          TagTable.new('SERVICES', manager.services).run
-          TagTable.new('TAGS', manager.tags).run
-          NodeTable.new(manager.nodes).run
+          TagTable.new('SERVICES', manager.services, colors).run
+          TagTable.new('TAGS', manager.tags, colors).run
+          NodeTable.new(manager.nodes, colors).run
         end
       end
     end
@@ -57,20 +62,21 @@ module LeapCli; module Commands
 
   class TagTable
     include CommandLineReporter
-    def initialize(heading, tag_list)
+    def initialize(heading, tag_list, colors)
       @heading = heading
       @tag_list = tag_list
+      @colors = colors
     end
     def run
       tags = @tag_list.keys.sort
       max_width = [20, (tags+[@heading]).inject(0) {|max,i| [i.size,max].max}].max
       table :border => false do
-        row :header => true, :color => 'cyan'  do
+        row :color => @colors[0]  do
           column @heading, :align => 'right', :width => max_width
           column "NODES", :width => HighLine::SystemExtensions.terminal_size.first - max_width - 2, :padding => 2
         end
         tags.each do |tag|
-          row do
+          row :color => @colors[1] do
             column tag
             column @tag_list[tag].node_list.keys.sort.join(', ')
           end
@@ -85,8 +91,9 @@ module LeapCli; module Commands
   #
   class NodeTable
     include CommandLineReporter
-    def initialize(node_list)
+    def initialize(node_list, colors)
       @node_list = node_list
+      @colors = colors
     end
     def run
       rows = @node_list.keys.sort.collect do |node_name|
@@ -102,13 +109,13 @@ module LeapCli; module Commands
       max_service_width = (rows.map{|i|i[1]} + ["SERVICES"]).inject(0) {|max,i| [i.size+padding+padding,max].max}
       max_tag_width     = (rows.map{|i|i[2]} + ["TAGS"]    ).inject(0) {|max,i| [i.size,max].max}
       table :border => false do
-        row :header => true, :color => 'cyan'  do
+        row :color => @colors[0]  do
           column "NODES", :align => 'right', :width => max_node_width
           column "SERVICES", :width => max_service_width, :padding => 2
           column "TAGS", :width => max_tag_width
         end
         rows.each do |r|
-          row do
+          row :color => @colors[1] do
             column r[0]
             column r[1]
             column r[2]

data/lib/leap_cli/commands/new.rb

@@ -14,10 +14,11 @@ module LeapCli; module Commands
     c.action do |global, options, args|
       directory = File.expand_path(args.first)
       create_provider_directory(global, directory)
-      options[:domain]   ||= ask("The primary domain of the provider: ") {|q| q.default = 'example.org'}
-      options[:name]     ||= ask("The name of the provider: ") {|q| q.default = 'Example'}
-      options[:platform] ||= ask("File path of the leap_platform directory: ") {|q| q.default = File.expand_path('../leap_platform', directory)}
-      options[:contacts] ||= ask("Default email address contacts: ") {|q| q.default = 'root@' + options[:domain]}
+      options[:domain]   ||= ask_string("The primary domain of the provider: ") {|q| q.default = 'example.org'}
+      options[:name]     ||= ask_string("The name of the provider: ") {|q| q.default = 'Example'}
+      options[:platform] ||= ask_string("File path of the leap_platform directory: ") {|q| q.default = File.expand_path('../leap_platform', directory)}
+      options[:platform] = "./" + options[:platform] unless options[:platform] =~ /^\//
+      options[:contacts] ||= ask_string("Default email address contacts: ") {|q| q.default = 'root@' + options[:domain]}
       options[:platform] = relative_path(options[:platform])
       create_initial_provider_files(directory, global, options)
     end
@@ -27,6 +28,21 @@ module LeapCli; module Commands
 
   DEFAULT_REPO = 'https://leap.se/git/leap_platform.git'
 
+  #
+  # don't let the user specify any of the following: y, yes, n, no
+  # they must actually input a real string
+  #
+  def ask_string(str, &block)
+    while true
+      value = ask(str, &block)
+      if value =~ /^(y|yes|n|no)$/i
+        say "`#{value}` is not a valid value. Try again"
+      else
+        return value
+      end
+    end
+  end
+
   #
   # creates a new provider directory
   #
@@ -77,7 +93,7 @@ module LeapCli; module Commands
   end
 
   def relative_path(path)
-    Pathname.new(path).relative_path_from(Pathname.new(Path.provider)).to_s
+    Pathname.new(File.expand_path(path)).relative_path_from(Pathname.new(Path.provider)).to_s
   end
 
   def leapfile_content(options)

data/lib/leap_cli/commands/node.rb

@@ -22,8 +22,7 @@ module LeapCli; module Commands
       add.action do |global_options,options,args|
         # argument sanity checks
         name = args.first
-        assert! name, 'No <node-name> specified.'
-        assert! name =~ /^[0-9a-z-]+$/, "illegal characters used in node name '#{name}'"
+        assert_valid_node_name!(name, options[:local])
         assert_files_missing! [:node_config, name]
 
         # create and seed new node
@@ -33,12 +32,14 @@ module LeapCli; module Commands
         end
         seed_node_data(node, args[1..-1])
         validate_ip_address(node)
-
-        # write the file
-        write_file! [:node_config, name], node.dump_json + "\n"
-        node['name'] = name
-        if file_exists? :ca_cert, :ca_key
-          generate_cert_for_node(manager.reload_node(node))
+        begin
+          write_file! [:node_config, name], node.dump_json + "\n"
+          node['name'] = name
+          if file_exists? :ca_cert, :ca_key
+            generate_cert_for_node(manager.reload_node!(node))
+          end
+        rescue LeapCli::ConfigError => exc
+          remove_node_files(name)
         end
       end
     end
@@ -64,10 +65,9 @@ module LeapCli; module Commands
           ssh_connect_options = connect_options(options).merge({:bootstrap => true, :echo => options[:echo]})
           ssh_connect(node, ssh_connect_options) do |ssh|
             if node.vagrant?
-              ssh.install_authorized_keys2
-            else
-              ssh.install_authorized_keys
+              ssh.install_insecure_vagrant_key
             end
+            ssh.install_authorized_keys
             ssh.install_prerequisites
             ssh.leap.capture(facter_cmd) do |response|
               if response[:exitcode] == 0
@@ -89,6 +89,7 @@ module LeapCli; module Commands
       mv.action do |global_options,options,args|
         node = get_node_from_args(args)
         new_name = args.last
+        assert_valid_node_name!(new_name, node.vagrant?)
         ensure_dir [:node_files_dir, new_name]
         Leap::Platform.node_files.each do |path|
           rename_file! [path, node.name], [path, new_name]
@@ -103,9 +104,7 @@ module LeapCli; module Commands
     node.command :rm do |rm|
       rm.action do |global_options,options,args|
         node = get_node_from_args(args)
-        (Leap::Platform.node_files + [:node_files_dir]).each do |path|
-          remove_file! [path, node.name]
-        end
+        remove_node_files(node.name)
         if node.vagrant?
           vagrant_command("destroy --force", [node.name])
         end
@@ -237,8 +236,14 @@ module LeapCli; module Commands
     end
   end
 
+  def remove_node_files(node_name)
+    (Leap::Platform.node_files + [:node_files_dir]).each do |path|
+      remove_file! [path, node_name]
+    end
+  end
+
   #
-  # conversations:
+  # conversions:
   #
   #   "x,y,z" => ["x","y","z"]
   #
@@ -273,4 +278,13 @@ module LeapCli; module Commands
     end
   end
 
+  def assert_valid_node_name!(name, local=false)
+    assert! name, 'No <node-name> specified.'
+    if local
+      assert! name =~ /^[0-9a-z]+$/, "illegal characters used in node name '#{name}' (note: Vagrant does not allow hyphens or underscores)"
+    else
+      assert! name =~ /^[0-9a-z-]+$/, "illegal characters used in node name '#{name}' (note: Linux does not allow underscores)"
+    end
+  end
+
 end; end

data/lib/leap_cli/commands/test.rb

@@ -46,7 +46,7 @@ module LeapCli; module Commands
     assert_config! 'provider.ca.client_certificates.unlimited_prefix'
     assert_config! 'provider.ca.client_certificates.limited_prefix'
     template = read_file! Path.find_file(:test_client_openvpn_template)
-    manager.environments.each do |env|
+    manager.environment_names.each do |env|
       vpn_nodes = manager.nodes[:environment => env][:services => 'openvpn']['openvpn.allow_limited' => true]
       if vpn_nodes.any?
         generate_test_client_cert(provider.ca.client_certificates.limited_prefix) do |key, cert|

data/lib/leap_cli/commands/vagrant.rb

@@ -156,7 +156,7 @@ module LeapCli; module Commands
           if node.vagrant?
             lines << %[  config.vm.define :#{node.name} do |config|]
             lines << %[    config.vm.box = "leap-wheezy"]
-            lines << %[    config.vm.box_url = "https://downloads.leap.se/leap-debian.box"]
+            lines << %[    config.vm.box_url = "https://downloads.leap.se/platform/leap-debian.box"]
             lines << %[    config.vm.network :hostonly, "#{node.ip_address}", :netmask => "#{netmask}"]
             lines << %[    config.vm.customize ["modifyvm", :id, "--natdnshostresolver1", "on"]]
             lines << %[    config.vm.customize ["modifyvm", :id, "--name", "#{node.name}"]]
@@ -170,7 +170,7 @@ module LeapCli; module Commands
           if node.vagrant?
             lines << %[  config.vm.define :#{node.name} do |config|]
             lines << %[    config.vm.box = "leap-wheezy"]
-            lines << %[    config.vm.box_url = "https://downloads.leap.se/leap-debian.box"]
+            lines << %[    config.vm.box_url = "https://downloads.leap.se/platform/leap-debian.box"]
             lines << %[    config.vm.network :private_network, ip: "#{node.ip_address}"]
             lines << %[    config.vm.provider "virtualbox" do |v|]
             lines << %[      v.customize ["modifyvm", :id, "--natdnshostresolver1", "on"]]

data/lib/leap_cli/config/macros.rb

@@ -1,3 +1,4 @@
+# encoding: utf-8
 #
 # MACROS
 # these are methods available when eval'ing a value in the .json configuration
@@ -5,6 +6,8 @@
 # This module is included in Config::Object
 #
 
+require 'base32'
+
 module LeapCli; module Config
   module Macros
     ##
@@ -22,7 +25,7 @@ module LeapCli; module Config
     # grab an environment appropriate provider
     #
     def provider
-      global.providers[@node.environment] || global.provider
+      global.env(@node.environment).provider
     end
 
     #
@@ -60,9 +63,9 @@ module LeapCli; module Config
       filepath = Path.find_file(filename)
       if filepath
         if filepath =~ /\.erb$/
-          ERB.new(File.read(filepath), nil, '%<>').result(binding)
+          ERB.new(File.read(filepath, :encoding => 'UTF-8'), nil, '%<>').result(binding)
         else
-          File.read(filepath)
+          File.read(filepath, :encoding => 'UTF-8')
         end
       else
         raise FileMissing.new(Path.named_path(filename), options)
@@ -129,6 +132,16 @@ module LeapCli; module Config
       @manager.secrets.set(name, Util::Secret.generate(length), @node[:environment])
     end
 
+    # inserts a base32 encoded secret
+    def base32_secret(name, length=20)
+      @manager.secrets.set(name, Base32.encode(Util::Secret.generate(length)), @node[:environment])
+    end
+
+    # Picks a random obfsproxy port from given range
+    def rand_range(name, range)
+      @manager.secrets.set(name, rand(range), @node[:environment])
+    end
+
     #
     # inserts an hexidecimal secret string, generating it if needed.
     #
@@ -343,14 +356,14 @@ module LeapCli; module Config
       hash = {}
       keys = Dir.glob(Path.named_path([:user_ssh, '*']))
       keys.sort.each do |keyfile|
-        ssh_type, ssh_key = File.read(keyfile).strip.split(" ")
+        ssh_type, ssh_key = File.read(keyfile, :encoding => 'UTF-8').strip.split(" ")
         name = File.basename(File.dirname(keyfile))
         hash[name] = {
           "type" => ssh_type,
           "key" => ssh_key
         }
       end
-      ssh_type, ssh_key = File.read(Path.named_path(:monitor_pub_key)).strip.split(" ")
+      ssh_type, ssh_key = File.read(Path.named_path(:monitor_pub_key), :encoding => 'UTF-8').strip.split(" ")
       hash[Leap::Platform.monitor_username] = {
         "type" => ssh_type,
         "key" => ssh_key
@@ -402,5 +415,16 @@ module LeapCli; module Config
       end
     end
 
+    #
+    # wrap something that might fail in `try`. e.g.
+    #
+    # "= try{ nodes[:services => 'tor'].first.ip_address } "
+    #
+    def try(&block)
+      yield
+    rescue NoMethodError
+      nil
+    end
+
   end
 end; end

data/lib/leap_cli/config/manager.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 require 'json/pure'
 
 if $ruby_version < [1,9]
@@ -7,16 +9,24 @@ end
 module LeapCli
   module Config
 
+    class Environment
+      attr_accessor :services, :tags, :provider
+    end
+
     #
     # A class to manage all the objects in all the configuration files.
     #
     class Manager
 
+      def initialize
+        @environments = {} # hash of `Environment` objects, keyed by name.
+      end
+
       ##
       ## ATTRIBUTES
       ##
 
-      attr_reader :services, :tags, :nodes, :provider, :providers, :common, :secrets
+      attr_reader :nodes, :common, :secrets
       attr_reader :base_services, :base_tags, :base_provider, :base_common
 
       #
@@ -30,10 +40,24 @@ module LeapCli
       # returns an Array of all the environments defined for this provider.
       # the returned array includes nil (for the default environment)
       #
-      def environments
-        @environments ||= [nil] + self.tags.collect {|name, tag| tag['environment']}.compact
+      def environment_names
+        @environment_names ||= [nil] + env.tags.collect {|name, tag| tag['environment']}.compact
       end
 
+      #
+      # Returns the appropriate environment variable
+      #
+      def env(env=nil)
+        env ||= 'default'
+        e = @environments[env] ||= Environment.new
+        yield e if block_given?
+        e
+      end
+
+      def services; env('default').services; end
+      def tags; env('default').tags; end
+      def provider; env('default').provider; end
+
       ##
       ## IMPORT EXPORT
       ##
@@ -46,34 +70,43 @@ module LeapCli
 
         # load base
         @base_services = load_all_json(Path.named_path([:service_config, '*'], Path.provider_base), Config::Tag)
-        @base_tags     = load_all_json(Path.named_path([:tag_config, '*'], Path.provider_base), Config::Tag)
-        @base_common   = load_json(Path.named_path(:common_config, Path.provider_base), Config::Object)
-        @base_provider = load_json(Path.named_path(:provider_config, Path.provider_base), Config::Provider)
+        @base_tags     = load_all_json(Path.named_path([:tag_config, '*'],     Path.provider_base), Config::Tag)
+        @base_common   = load_json(    Path.named_path(:common_config,         Path.provider_base), Config::Object)
+        @base_provider = load_json(    Path.named_path(:provider_config,       Path.provider_base), Config::Provider)
 
         # load provider
-        provider_path = Path.named_path(:provider_config, @provider_dir)
-        common_path = Path.named_path(:common_config, @provider_dir)
-        Util::assert_files_exist!(provider_path, common_path)
-        @services = load_all_json(Path.named_path([:service_config, '*'], @provider_dir), Config::Tag)
-        @tags     = load_all_json(Path.named_path([:tag_config, '*'],     @provider_dir), Config::Tag)
-        @nodes    = load_all_json(Path.named_path([:node_config, '*'],    @provider_dir), Config::Node)
-        @common   = load_json(common_path, Config::Object)
-        @provider = load_json(provider_path, Config::Provider)
-        @secrets  = load_json(Path.named_path(:secrets_config,  @provider_dir), Config::Secrets)
-
-        ### BEGIN HACK
-        ### remove this after it is likely that no one has any old-style secrets.json
-        if @secrets['webapp_secret_token']
-          @secrets = Config::Secrets.new
-          Util::log :warning, "Creating all new secrets.json (new version is scoped by environment). Make sure to do a full deploy so that new secrets take effect."
+        @nodes    = load_all_json(Path.named_path([:node_config, '*'],  @provider_dir), Config::Node)
+        @common   = load_json(    Path.named_path(:common_config,       @provider_dir), Config::Object)
+        @secrets  = load_json(    Path.named_path(:secrets_config,      @provider_dir), Config::Secrets)
+        @common.inherit_from! @base_common
+
+        # load provider services, tags, and provider.json, DEFAULT environment
+        log 3, :loading, 'default environment.........'
+        env('default') do |e|
+          e.services = load_all_json(Path.named_path([:service_config, '*'], @provider_dir), Config::Tag, :no_dots => true)
+          e.tags     = load_all_json(Path.named_path([:tag_config, '*'],     @provider_dir), Config::Tag, :no_dots => true)
+          e.provider = load_json(    Path.named_path(:provider_config,       @provider_dir), Config::Provider, :assert => true)
+          e.services.inherit_from! @base_services
+          e.tags.inherit_from!     @base_tags
+          e.provider.inherit_from! @base_provider
+          validate_provider(e.provider)
+        end
+
+        # load provider services, tags, and provider.json, OTHER environments
+        environment_names.each do |ename|
+          next unless ename
+          log 3, :loading, '%s environment.........' % ename
+          env(ename) do |e|
+            e.services = load_all_json(Path.named_path([:service_env_config, '*', ename], @provider_dir), Config::Tag)
+            e.tags     = load_all_json(Path.named_path([:tag_env_config, '*', ename],     @provider_dir), Config::Tag)
+            e.provider = load_json(    Path.named_path([:provider_env_config, ename],     @provider_dir), Config::Provider)
+            e.services.inherit_from! env.services
+            e.tags.inherit_from!     env.tags
+            e.provider.inherit_from! env.provider
+            validate_provider(e.provider)
+          end
         end
-        ### END HACK
 
-        # inherit
-        @services.inherit_from! base_services
-        @tags.inherit_from!     base_tags
-        @common.inherit_from!   base_common
-        @provider.inherit_from! base_provider
         @nodes.each do |name, node|
           Util::assert! name =~ /^[0-9a-z-]+$/, "Illegal character(s) used in node name '#{name}'"
           @nodes[name] = apply_inheritance(node)
@@ -82,19 +115,6 @@ module LeapCli
         unless options[:include_disabled]
           remove_disabled_nodes
         end
-
-        # load optional environment specific providers
-        validate_provider(@provider)
-        @providers = {}
-        environments.each do |env|
-          if Path.defined?(:provider_env_config)
-            provider_path = Path.named_path([:provider_env_config, env], @provider_dir)
-            providers[env] = load_json(provider_path, Config::Provider)
-            providers[env].inherit_from! @provider
-            validate_provider(providers[env])
-          end
-        end
-
       end
 
       #
@@ -202,6 +222,11 @@ module LeapCli
       # returns a single Config::Object that corresponds to a Node.
       #
       def node(name)
+        if name =~ /\./
+          # probably got a fqdn, since periods are not allowed in node names.
+          # so, take the part before the first period as the node name
+          name = name.split('.').first
+        end
         @nodes[name]
       end
 
@@ -219,18 +244,19 @@ module LeapCli
         nodes.each_node &block
       end
 
-      def reload_node(node)
-        @nodes[node.name] = apply_inheritance(node)
+      def reload_node!(node)
+        @nodes[node.name] = apply_inheritance!(node)
       end
 
       private
 
-      def load_all_json(pattern, object_class)
+      def load_all_json(pattern, object_class, options={})
         results = Config::ObjectList.new
         Dir.glob(pattern).each do |filename|
+          next if options[:no_dots] && File.basename(filename) !~ /^[^\.]*\.json$/
           obj = load_json(filename, object_class)
           if obj
-            name = File.basename(filename).sub(/\.json$/,'')
+            name = File.basename(filename).force_encoding('utf-8').sub(/^([^\.]+).*\.json$/,'\1')
             obj['name'] ||= name
             results[name] = obj
           end
@@ -238,7 +264,10 @@ module LeapCli
         results
       end
 
-      def load_json(filename, object_class)
+      def load_json(filename, object_class, options={})
+        if options[:assert]
+          Util::assert_files_exist!(filename)
+        end
         if !File.exists?(filename)
           return object_class.new(self)
         end
@@ -252,7 +281,7 @@ module LeapCli
         # https://www.ietf.org/rfc/rfc4627.txt
         #
         buffer = StringIO.new
-        File.open(filename, "rb") do |f|
+        File.open(filename, "rb", :encoding => 'UTF-8') do |f|
           while (line = f.gets)
             next if line =~ /^\s*\/\//
             buffer << line
@@ -300,22 +329,36 @@ module LeapCli
       #
       # makes a node inherit options from appropriate the common, service, and tag json files.
       #
-      def apply_inheritance(node)
+      def apply_inheritance(node, throw_exceptions=false)
         new_node = Config::Node.new(self)
         name = node.name
 
+        # Guess the environment of the node from the tag names.
+        # (Technically, this is wrong: a tag that sets the environment might not be
+        #  named the same as the environment. This code assumes that it is).
+        node_env = self.env
+        if node['tags']
+          node['tags'].to_a.each do |tag|
+            if self.environment_names.include?(tag)
+              node_env = self.env(tag)
+            end
+          end
+        end
+
         # inherit from common
         new_node.deep_merge!(@common)
 
         # inherit from services
         if node['services']
           node['services'].to_a.each do |node_service|
-            service = @services[node_service]
+            service = node_env.services[node_service]
             if service.nil?
-              log 0, :error, 'in node "%s": the service "%s" does not exist.' % [node['name'], node_service]
+              msg = 'in node "%s": the service "%s" does not exist.' % [node['name'], node_service]
+              log 0, :error, msg
+              raise LeapCli::ConfigError.new(node, "error " + msg) if throw_exceptions
             else
               new_node.deep_merge!(service)
-              service.node_list.add(name, new_node)
+              self.services[node_service].node_list.add(name, new_node)
             end
           end
         end
@@ -326,12 +369,14 @@ module LeapCli
         end
         if node['tags']
           node['tags'].to_a.each do |node_tag|
-            tag = @tags[node_tag]
+            tag = node_env.tags[node_tag]
             if tag.nil?
-              log 0, :error, 'in node "%s": the tag "%s" does not exist.' % [node['name'], node_tag]
+              msg = 'in node "%s": the tag "%s" does not exist.' % [node['name'], node_tag]
+              log 0, :error, msg
+              raise LeapCli::ConfigError.new(node, "error " + msg) if throw_exceptions
             else
               new_node.deep_merge!(tag)
-              tag.node_list.add(name, new_node)
+              self.tags[node_tag].node_list.add(name, new_node)
             end
           end
         end
@@ -341,6 +386,10 @@ module LeapCli
         return new_node
       end
 
+      def apply_inheritance!(node)
+        apply_inheritance(node, true)
+      end
+
       def remove_disabled_nodes
         @disabled_nodes = Config::ObjectList.new
         @nodes.each do |name, node|
@@ -350,12 +399,12 @@ module LeapCli
             @disabled_nodes[name] = node
             if node['services']
               node['services'].to_a.each do |node_service|
-                @services[node_service].node_list.delete(node.name)
+                self.services[node_service].node_list.delete(node.name)
               end
             end
             if node['tags']
               node['tags'].to_a.each do |node_tag|
-                @tags[node_tag].node_list.delete(node.name)
+                self.tags[node_tag].node_list.delete(node.name)
               end
             end
           end

data/lib/leap_cli/config/object.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 require 'erb'
 require 'json/pure'  # pure ruby implementation is required for our sorted trick to work.
 
@@ -96,7 +98,9 @@ module LeapCli
       #
       def get!(key)
         key = key.to_s
-        if key =~ /\./
+        if self.has_key?(key)
+          fetch_value(key)
+        elsif key =~ /\./
           # for keys with with '.' in them, we start from the root object (@node).
           keys = key.split('.')
           value = @node.get!(keys.first)
@@ -105,8 +109,6 @@ module LeapCli
           else
             value
           end
-        elsif self.has_key?(key)
-          fetch_value(key)
         else
           raise NoMethodError.new(key, "No method '#{key}' for #{self.class}")
         end

data/lib/leap_cli/config/object_list.rb

@@ -182,7 +182,14 @@ module LeapCli
       end
 
       def tsort_each_child(node_name, &block)
-        self[node_name].test_dependencies.each(&block)
+        if self[node_name]
+          self[node_name].test_dependencies.each do |test_me_first|
+            if self[test_me_first] # TODO: in the future, allow for ability to optionally pull in all dependencies.
+                                   # not just the ones that pass the node filter.
+              yield(test_me_first)
+            end
+          end
+        end
       end
 
       def names_in_test_dependency_order

data/lib/leap_cli/config/secrets.rb

@@ -1,3 +1,4 @@
+# encoding: utf-8
 #
 # A class for the secrets.json file
 #

data/lib/leap_cli/exceptions.rb

@@ -0,0 +1,11 @@
+module LeapCli
+
+  class ConfigError < StandardError
+    attr_accessor :node
+    def initialize(node, msg)
+      @node = node
+      super(msg)
+    end
+  end
+
+end

data/lib/leap_cli/log.rb

@@ -80,6 +80,7 @@ module LeapCli
       if title
         prefix_options = case title
           when :error     then ['error', :red, :bold]
+          when :fatal_error then ['fatal error', :red, :bold]
           when :warning   then ['warning:', :yellow, :bold]
           when :info      then ['info', :cyan, :bold]
           when :updated   then ['updated', :cyan, :bold]

data/lib/leap_cli/remote/tasks.rb

@@ -13,38 +13,53 @@ task :install_authorized_keys, :max_hosts => MAX_HOSTS do
 end
 
 #
-# for vagrant nodes, we don't overwrite authorized_keys, because we want to keep the insecure vagrant key.
-# instead we install to authorized_keys2, which is also used by sshd.
+# for vagrant nodes, we install insecure vagrant key to authorized_keys2, since deploy
+# will overwrite authorized_keys.
 #
-# why?
-#   without it, it might be impossible to re-initialize a node.
-#
-# ok, why is that?
-#   when we init a vagrant node, we force it to use the insecure vagrant key, and not the user's keys
-#   (so re-initialization would be impossible if authorized_keys doesn't include insecure key).
-#
-# ok, why force the insecure vagrant key in the first place?
+# why force the insecure vagrant key?
 #   if we don't do this, then first time initialization might fail if the user has many keys
 #   (ssh will bomb out before it gets to the vagrant key).
 #   and it really doesn't make sense to ask users to pin the insecure vagrant key in their
 #   .ssh/config files.
 #
-task :install_authorized_keys2, :max_hosts => MAX_HOSTS do
-  leap.log :updating, "authorized_keys2" do
+task :install_insecure_vagrant_key, :max_hosts => MAX_HOSTS do
+  leap.log :installing, "insecure vagrant key" do
     leap.mkdirs '/root/.ssh'
-    upload LeapCli::Path.named_path(:authorized_keys), '/root/.ssh/authorized_keys2', :mode => '600'
+    key_file = File.expand_path('../../../vendor/vagrant_ssh_keys/vagrant.pub', File.dirname(__FILE__))
+    upload key_file, '/root/.ssh/authorized_keys2', :mode => '600'
   end
 end
 
+BAD_APT_GET_UPDATE = /(BADSIG|NO_PUBKEY|KEYEXPIRED|REVKEYSIG|NODATA)/
+
 task :install_prerequisites, :max_hosts => MAX_HOSTS do
+  apt_get = "DEBIAN_FRONTEND=noninteractive apt-get -q -y -o DPkg::Options::=--force-confold"
   leap.mkdirs LeapCli::PUPPET_DESTINATION
+  run "echo 'en_US.UTF-8 UTF-8' > /etc/locale.gen"
   leap.log :updating, "package list" do
-    run "apt-get update"
+    run "apt-get update" do |channel, stream, data|
+      # sadly exitcode is unreliable measure if apt-get update hit a failure.
+      if data =~ BAD_APT_GET_UPDATE
+        LeapCli::Util.bail! do
+          LeapCli::Util.log :fatal_error, "in `apt-get update`: #{data}", :host => channel[:host]
+        end
+      else
+        logger.log(1, data, channel[:host])
+      end
+    end
+  end
+  leap.log :updating, "server time" do
+    run "( test -f /etc/init.d/ntp && /etc/init.d/ntp stop ) || true"
+    run "test -f /usr/sbin/ntpdate || #{apt_get} install ntpdate"
+    leap.log :running, "ntpdate..." do
+      run "test -f /usr/sbin/ntpdate && ntpdate 0.debian.pool.ntp.org 1.debian.pool.ntp.org 2.debian.pool.ntp.org 3.debian.pool.ntp.org"
+    end
+    run "( test -f /etc/init.d/ntp && /etc/init.d/ntp start ) || true"
   end
   leap.log :installing, "required packages" do
-    run "DEBIAN_FRONTEND=noninteractive apt-get -q -y -o DPkg::Options::=--force-confold install #{leap.required_packages}"
+    run "#{apt_get} install #{leap.required_packages}"
   end
-  run "echo 'en_US.UTF-8 UTF-8' > /etc/locale.gen; locale-gen"
+  #run "locale-gen"
   leap.mkdirs("/etc/leap", "/srv/leap")
   leap.mark_initialized
 end

data/lib/leap_cli/util.rb

@@ -193,13 +193,13 @@ module LeapCli
     def read_file!(filepath)
       filepath = Path.named_path(filepath)
       assert_files_exist!(filepath)
-      File.read(filepath)
+      File.read(filepath, :encoding => 'UTF-8')
     end
 
     def read_file(filepath)
       filepath = Path.named_path(filepath)
       if file_exists?(filepath)
-        File.read(filepath)
+        File.read(filepath, :encoding => 'UTF-8')
       end
     end
 
@@ -219,7 +219,7 @@ module LeapCli
           write_file!(filepath, content)
         end
       else
-        File.open(filepath, File::RDWR|File::CREAT, 0600) do |f|
+        File.open(filepath, File::RDWR|File::CREAT, 0600, :encoding => 'UTF-8') do |f|
           f.flock(File::LOCK_EX)
           old_content = f.read
           new_content = yield(old_content)
@@ -286,7 +286,7 @@ module LeapCli
         end
       end
 
-      File.open(filepath, 'w', 0600) do |f|
+      File.open(filepath, 'w', 0600, :encoding => 'UTF-8') do |f|
         f.write contents
       end
 

data/lib/leap_cli/util/remote_command.rb

@@ -13,7 +13,7 @@ module LeapCli; module Util; module RemoteCommand
     node_list = parse_node_list(nodes)
 
     cap = new_capistrano
-    cap.logger = LeapCli::Logger.new(:level => LeapCli.log_level)
+    cap.logger = LeapCli::Logger.new(:level => [LeapCli.log_level,3].min)
     user = options[:user] || 'root'
     cap.set :user, user
     cap.set :ssh_options, ssh_options # ssh options common to all nodes
@@ -31,7 +31,7 @@ module LeapCli; module Util; module RemoteCommand
     end
 
     node_list.each do |name, node|
-      cap.server node.name, :dummy_arg, node_options(node, options[:ssh_options])
+      cap.server node.domain.full, :dummy_arg, node_options(node, options[:ssh_options])
     end
 
     yield cap
@@ -48,9 +48,34 @@ module LeapCli; module Util; module RemoteCommand
   #
   # For available options, see http://net-ssh.github.com/net-ssh/classes/Net/SSH.html#method-c-start
   #
+  # Capistrano has some very evil behavior in it's ssh.rb:
+  #
+  #   ssh_options = Net::SSH.configuration_for(
+  #     server.host, ssh_options.fetch(:config, true)
+  #   ).merge(ssh_options)
+  #   # Once we've loaded the config, we don't need Net::SSH to do it again.
+  #   ssh_options[:config] = false
+  #
+  # Net:SSH is supposed to call Net::SSH.configuration_for, but Capistrano is doing it
+  # in advance and then disabling loading of configs.
+  #
+  # The result of this is the following: if you have IdentityFile in your ~/.ssh/config
+  # file, then the above code will transform the ssh_options by reading ~/.ssh/config
+  # and adding the keys specified via IdentityFile to ssh_options...
+  # AND IT WILL SET :keys_only TO TRUE.
+  #
+  # The problem is that :keys_only will disable Net:SSH's ability to use ssh-agent.
+  # With :keys_only set to true, it will not consult the ssh-agent at all.
+  #
+  # So nice of capistrano to parse ~/.ssh/config for us, but then add flags to the
+  # ssh_options that prevent's these options from being useful.
+  #
+  # The current hackaround is to force :keys_only to be false. This allows the config
+  # to be read and also allows ssh-agent to still be used.
+  #
   def ssh_options
     {
-      :config => false,
+      :keys_only => false, # Don't you dare change this.
       :global_known_hosts_file => path(:known_hosts),
       :user_known_hosts_file => '/dev/null',
       :paranoid => true
@@ -67,13 +92,12 @@ module LeapCli; module Util; module RemoteCommand
   #  return {:password => password_proc}
   #
   def node_options(node, ssh_options_override=nil)
-    ssh_options_override ||= {}
     {
       :ssh_options => {
         # :host_key_alias => node.name, << incompatible with ports in known_hosts
         :host_name => node.ip_address,
         :port => node.ssh.port
-      }.merge(contingent_ssh_options_for_node(node)).merge(ssh_options_override)
+      }.merge(contingent_ssh_options_for_node(node)).merge(ssh_options_override||{})
     }
   end
 

data/lib/leap_cli/util/secret.rb

@@ -1,3 +1,4 @@
+# encoding: utf-8
 #
 # A simple secret generator
 #

data/lib/leap_cli/version.rb

@@ -1,7 +1,7 @@
 module LeapCli
   unless defined?(LeapCli::VERSION)
-    VERSION = '1.5.1'
-    COMPATIBLE_PLATFORM_VERSION = '0.3.0'..'1.99'
+    VERSION = '1.5.6'
+    COMPATIBLE_PLATFORM_VERSION = '0.5.2'..'1.99'
     SUMMARY = 'Command line interface to the LEAP platform'
     DESCRIPTION = 'The command "leap" can be used to manage a bevy of servers running the LEAP platform from the comfort of your own home.'
     LOAD_PATHS = ['lib', 'vendor/certificate_authority/lib', 'vendor/rsync_command/lib']

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: leap_cli
 version: !ruby/object:Gem::Version
-  version: 1.5.1
+  version: 1.5.6
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-03-17 00:00:00.000000000 Z
+date: 2014-06-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: minitest
@@ -114,7 +114,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 2.13.5
+        version: 2.15.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -122,7 +122,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 2.13.5
+        version: 2.15.5
 - !ruby/object:Gem::Dependency
   name: net-ssh
   requirement: !ruby/object:Gem::Requirement
@@ -203,6 +203,22 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: base32
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: activemodel
   requirement: !ruby/object:Gem::Requirement
@@ -244,6 +260,7 @@ files:
 - lib/leap_cli/commands/ca.rb
 - lib/leap_cli/commands/pre.rb
 - lib/leap_cli/commands/compile.rb
+- lib/leap_cli/commands/db.rb
 - lib/leap_cli/commands/user.rb
 - lib/leap_cli/commands/node.rb
 - lib/leap_cli/commands/clean.rb
@@ -265,6 +282,7 @@ files:
 - lib/leap_cli/requirements.rb
 - lib/leap_cli/path.rb
 - lib/leap_cli/leapfile.rb
+- lib/leap_cli/exceptions.rb
 - lib/leap_cli/log.rb
 - lib/leap_cli/util.rb
 - lib/leap_cli/config/manager.rb