cult 0.1.3.pre → 0.1.4.pre

data/lib/cult/cli/node_cmd.rb

@@ -1,5 +1,6 @@
-require 'SecureRandom'
+require 'securerandom'
 require 'fileutils'
+require 'json'
 
 module Cult
   module CLI
@@ -16,65 +17,105 @@ module Cult
           conceptually description of a server.
         EOD
 
-        run(arguments: 0) do |opts, args, cmd|
+        run(arguments: none) do |opts, args, cmd|
           puts cmd.help
+          exit
         end
       end
 
       node_ssh = Cri::Command.define do
         name        'ssh'
-        usage       'ssh NODE'
-        summary     'Starts an interactive SSH shell to NODE'
+        usage       'ssh /NODE+/ [command...]'
+        summary     'Starts an SSH shell to NODE'
+
+        flag :i, :interactive,       "Force interactive mode"
+        flag :I, :'non-interactive', "Force non-interactive mode"
+
         description <<~EOD.format_description
+          With no additional arguments, initiates an interactive SSH connection
+          to a node, authenticated with the node's public key.
+
+          Additional arguments are passed to the 'ssh' command to allow for
+          scripting or running one-off commands on the node.
+
+          By default, cult assumes an interactive SSH session when no extra
+          SSH arguments are passed, and a non-interactive session otherwise.
+          You can force this behavior one way or the other with --interactive
+          or --non-interactive.
+
+          Cult will run SSH commands over all matching nodes in parallel if it
+          considers your command non-interactive.
         EOD
 
-        run(arguments: 1) do |opts, args, cmd|
-          node = CLI.fetch_item(args[0], from: Node)
-          exec "ssh", '-i', node.ssh_private_key_file,
-               "#{node.user}@#{node.host}"
+        esc = ->(s) { Shellwords.escape(s) }
+
+        run(arguments: 1 .. unlimited) do |opts, args, cmd|
+          if opts[:interactive] && opts[:'non-interactive']
+            fail CLIError, "can't specify --interactive and --non-interactive"
+          end
+
+          interactive = opts[:interactive] ||
+                        (opts[:'non-interactive'] && false)
+
+          nodes = CLI.fetch_items(args[0], from: Node)
+
+          # With args, we'll assume it's a non-interactive session and run them
+          # in parallel, otherwise, we'll assume it's interactive and force
+          # them to run one at a time.
+          ssh_extra = args[1 .. -1]
+          interactive ||= ssh_extra.empty?
+          concurrent = interactive || nodes.size == 1 ? 1 : nil
+
+          Cult.paramap(nodes, concurrent: concurrent) do |node|
+            ssh_args = 'ssh', '-i', esc.(node.ssh_private_key_file),
+                       '-p', esc.(node.ssh_port.to_s),
+                       '-o', "UserKnownHostsFile=#{esc.(node.ssh_known_hosts_file)}",
+                       esc.("#{node.user}@#{node.host}")
+            ssh_args += ssh_extra
+            # We used to use exec here, but with paramap, the forked process
+            # has to live to long enough to report it's return value.
+            system(*ssh_args)
+            exit if interactive
+          end
         end
       end
       node.add_command(node_ssh)
 
-      node_create = Cri::Command.define do
-        name        'create'
-        aliases     'new'
-        usage       'create [options] NAME...'
+      node_new = Cri::Command.define do
+        name        'new'
+        usage       'new -r /ROLE+/ [options] NAME0 NAME1 ...'
         summary     'Create a new node'
         description <<~EOD.format_description
           This command creates a new node specification and then creates it with
           your provider.
 
           The newly created node will have all the roles listed in --role.  If
-          none are specified, it'll have the role "all".  If no name is
-          provided, it will be named for its role(s).
+          none are specified, it'll have the role "base".  If no name is
+          provided, it will be named after its role(s).
 
           If multiple names are provided, a new node is created for each name
-          given.
+          given.  The --count option is incompatible with multiple names given
+          on the command line.
 
           The --count option lets you create an arbitrary amount of new nodes.
-          The nodes will be identical, except they'll be named with increasing
-          sequential numbers, like:
+          The nodes will be identical, except they'll be named with arbitrary
+          random suffixes, like:
 
-          > web-01, web-02
+          > web-fjfowhs7, web-48pqee6v
 
-          And so forth.  The --count option is incompatible with multiple names
-          given on the command line.  If --count is specified with one name, the
-          name will become the prefix for all nodes created.  If --count is
-          specified with no names, the prefix will be generated from the role
-          names used, as discussed above.
+          And so forth.
         EOD
 
-        required :r, :role,      'Specify possibly multiple roles',
+        required :r, :role,      'Specify possibly multiple /ROLE+/',
                                   multiple: true
         required :n, :count,     'Generates <value> number of nodes'
 
-        required :p, :provider,  'Provider'
+        required :p, :provider,  'Use /PROVIDER/ to create the node'
         required :Z, :zone,      'Provider zone'
         required :I, :image,     'Provider image'
         required :S, :size,      'Provider instance size'
 
-        run(arguments: 0..-1) do |opts, args, cmd|
+        run(arguments: unlimited) do |opts, args, cmd|
           random_suffix = ->(basename) do
             begin
               suffix = CLI.unique_id
@@ -85,42 +126,32 @@ module Cult
           end
 
           generate_sequenced_names = ->(name, n) do
-            result = []
-            result.push(random_suffix.(name)) until result.size == n
-            result
-          end
-
-          unless opts[:count].nil? || opts[:count].match(/^\d+$/)
-            fail CLIError, "--count must be an integer"
+            (0...n).map do
+              random_suffix.(name)
+            end
           end
 
           names = args.dup
 
-          roles = opts[:role] ? CLI.fetch_items(opts[:role], from: Role) : []
-
-          if roles.empty?
-            roles = CLI.fetch_items('all', from: Role)
-            if names.empty?
-              begin
-                names.push CLI.fetch_item('node', from: Node, exist: false)
-              rescue
-                names.push random_suffix.('node')
-              end
-            end
+          unless opts[:count].nil? || opts[:count].match(/^\d+$/)
+            fail CLIError, "--count must be an integer"
           end
 
           if names.size > 1 && opts[:count]
             fail CLIError, "cannot specify both --count and more than one name"
           end
 
-          if names.empty? && !roles.empty?
-            names.push roles.map(&:name).sort.join('-')
-          end
+          roles = CLI.fetch_items(opts[:role] || 'base', from: Role)
 
-          if opts[:count]
-            names = generate_sequenced_names.(names[0], opts[:count].to_i)
+          if names.empty?
+            names.push roles.map(&:name).join("-")
+            opts[:count] ||= 1
           end
 
+          names = opts[:count] ? generate_sequenced_names.(names[0],
+                                                           opts[:count].to_i)
+                               : names
+
           # Makes sure they're all new.
           names = names.map do |name|
             CLI.fetch_item(name, from: Node, exist: false)
@@ -140,36 +171,40 @@ module Cult
             [m, value]
           end.to_h
 
-          nodes = names.map do |name|
+          Cult.paramap(names) do |name|
             data = {
               name: name,
               roles: roles.map(&:name)
             }
 
             Node.from_data!(Cult.project, data).tap do |node|
+              puts "Provisioning #{node.name}..."
               prov_data = provider.provision!(name: node.name,
                                               image: node_spec[:image],
                                               size: node_spec[:size],
                                               zone: node_spec[:zone],
                                               ssh_public_key: node.ssh_public_key_file)
               prov_data['provider'] = provider.name
-              File.write(Cult.project.dump_name(node.state_path),
-                         Cult.project.dump_object(prov_data))
+              File.write(node.state_path, JSON.pretty_generate(prov_data))
 
               c = Commander.new(project: Cult.project, node: node)
+              puts "Bootstrapping #{node.name}..."
               c.bootstrap!
+
+              puts "Installing roles for #{node.name}..."
               c.install!(node)
+
+              puts "Node installed: #{node.name}"
             end
           end
 
         end
       end
-      node.add_command node_create
+      node.add_command(node_new)
 
-      node_destroy = Cri::Command.define do
-        name        'destroy'
-        aliases     'rm'
-        usage       'destroy NODE'
+      node_rm = Cri::Command.define do
+        name        'rm'
+        usage       'rm /NODE+/ ...'
         summary     'Destroy nodes'
         description <<~EOD.format_description
           Destroys all nodes named NODE, or match the pattern described by
@@ -181,9 +216,10 @@ module Cult
           be prompted before each destroy.
         EOD
 
-        run(arguments: 1..-1) do |opts, args, cmd|
+        run(arguments: 1 .. unlimited) do |opts, args, cmd|
           nodes = CLI.fetch_items(args, from: Node)
-          nodes.each do |node|
+          concurrent = CLI.yes? ? :max : 1
+          Cult.paramap(nodes, concurrent: concurrent) do |node|
             if CLI.yes_no?("Destroy node `#{node}`?")
               puts "destroying #{node}"
               begin
@@ -197,15 +233,16 @@ module Cult
               fail unless node.path.match(/#{Regexp.escape(node.name)}/)
               FileUtils.rm_rf(node.path)
             end
+            nil
           end
         end
       end
-      node.add_command(node_destroy)
+      node.add_command(node_rm)
 
-      node_list = Cri::Command.define do
-        name        'list'
-        aliases     'ls'
+      node_ls = Cri::Command.define do
+        name        'ls'
         summary     'List nodes'
+        usage       'ls /NODE+/ ...'
         description <<~EOD.format_description
           This command lists the nodes in the project.
         EOD
@@ -213,9 +250,9 @@ module Cult
         required :r, :role, 'List only nodes which include <value>',
                      multiple: true
 
-        run(arguments: 0..1) do |opts, args, cmd|
+        run(arguments: unlimited) do |opts, args, cmd|
           nodes = args.empty? ? Cult.project.nodes
-                              : CLI.fetch_items(*args, from: Node)
+                              : CLI.fetch_items(args, from: Node)
 
           if opts[:role]
             roles = CLI.fetch_items(opts[:role], from: Role)
@@ -224,12 +261,141 @@ module Cult
             end
           end
 
+          Cult.paramap(nodes) do |node|
+            role_string = node.build_order.map do |role|
+              if node.zone_leader?(role)
+                Rainbow('*' + role.name).cyan
+              else
+                '=' + role.name
+              end
+            end.join(', ')
+
+            puts "#{node.name}\t#{node.provider&.name}\t" +
+                "#{node.zone}\t#{node.addr(:public)}\t#{node.addr(:private)}\t" +
+                "#{role_string}"
+          end
+        end
+      end
+      node.add_command(node_ls)
+
+
+      node_sync = Cri::Command.define do
+        name        'sync'
+        usage       'sync /NODE+/ ...'
+        summary     'Synchronize host information across fleet'
+        description <<~EOD.format_description
+          Computes, pre-processes, and executes "sync" tasks on every NODE,
+          or all nodes if none are specified.
+
+          Sync tasks are tasks that begin with 'sync-'.  They are meant to
+          process dynamic information about the fleet well after a node has
+          been created.  Typically, you'll run `cult node sync` to let each
+          instance know about its new neighborhood after you add or remove
+          new nodes.
+
+          Sync tasks can optionally specify a "pass", with "sync-P0-..." or
+          "sync-P1-...".  When `cult node sync` executes, it ensures that:
+
+            1. On a given node, all tasks in the current pass are executed
+               sequentially, in role and asciibetical order.
+
+            2. Across the fleet, nodes which have tasks to run in a given pass
+               are run concurently with each other.
+
+            3. The entire fleet (or NODE selection) synchonizes between passes.
+               "Pass 0" has run on EVERY node (across any role boundaries)
+               before "Pass 1" is started on ANY node.
+
+          Sync tasks without a specified pass are implicitly in "Pass 0".
+
+          The sync can be restricted to a specified set of passes with the
+          --pass option.  Note that this skips dependent passes.
+
+          The sync can be restricted to a specified set of CONCRETE role tasks
+          with the --role option.  No dependencies are considered: Cult
+          calculates the tasks it would've ran, then removes all tasks not
+          belonging to a role given to --roles
+        EOD
+
+        required :R, :role, "Skip sync tasks not in /ROLE/.  Can be specified " +
+                            "more than once.",
+                            multiple: true
+
+        required :P, :pass, "Only execute PASS.  Can be specified more than " +
+                            "once.",
+                            multiple: true
+
+        run(arguments: unlimited) do |opts, args, cmd|
+          nodes = args.empty? ? Cult.project.nodes
+                              : CLI.fetch_items(args, from: Node)
+          roles = opts[:role].nil? ? Cult.project.roles
+                                   : CLI.fetch_items(opts[:role], from: Role)
+          c = CommanderSync.new(project: Cult.project, nodes: nodes)
+          passes = opts[:pass] ? opts[:pass].map(&:to_i) : nil
+          c.sync!(roles: roles, passes: passes)
+        end
+      end
+      node.add_command(node_sync)
+
+      node_ping = Cri::Command.define do
+        name        'ping'
+        summary     'Check the responsiveness of each node'
+        usage       'ping /NODE+/'
+
+        flag :d, :destroy, 'Destroy nodes that are not responding.'
+
+        description <<~EOD.format_description
+          Connects to each node and reports health information.
+        EOD
+
+        run(arguments: unlimited) do |opts, args, cmd|
+          nodes = args.empty? ? Cult.project.nodes
+                              : CLI.fetch_items(args, from: Node)
+          Cult.paramap(nodes) do |node|
+            c = Commander.new(project: Cult.project, node: node)
+            if (r = c.ping)
+              puts Rainbow(node.name).green + ": #{r}"
+              nil
+            else
+              puts Rainbow(node.name).red + ": Unreachable"
+              node
+            end
+          end
+        end
+      end
+      node.add_command(node_ping)
+
+      node_addr = Cri::Command.define do
+        name        'addr'
+        aliases     'ip'
+        summary     'print IP address of node'
+        usage       'addr [/NODE+/ ...]'
+        flag        :p, :private, 'Print private address'
+        flag        :'6', :ipv6, 'Print ipv6 address'
+        flag        :'4', :ipv4, 'Print ipv4 address'
+        description <<~EOD.format_description
+        EOD
+
+        run(arguments: unlimited) do |opts, args, cmd|
+          prot = Cult.project.default_ip_protocol
+          if opts[:ipv4] && opts[:ipv6]
+            fail CLIError, "can't specify --ipv4 and --ipv6"
+          end
+
+          prot = :ipv6 if opts[:ipv6]
+          prot = :ipv4 if opts[:ipv4]
+
+          priv = opts[:private] ? :private: :public
+
+          nodes = args.empty? ? Cult.project.nodes
+                              : CLI.fetch_items(args, from: Node)
           nodes.each do |node|
-            puts "#{node.name}\t#{node.provider&.name}\t#{node.roles.map(&:name)}"
+            puts node.addr(priv, prot)
           end
         end
       end
-      node.add_command node_list
+      node.add_command(node_addr)
+
 
       return node
     end

data/lib/cult/cli/provider_cmd.rb

@@ -1,4 +1,5 @@
 require 'cult/drivers/load'
+require 'json'
 
 module Cult
   module CLI
@@ -9,7 +10,7 @@ module Cult
         optional_project
         name        'provider'
         aliases     'providers'
-        summary     'Provider Commands'
+        summary     'Provider commands'
         description <<~EOD.format_description
           A provider is a VPS service.  Cult ships with drivers for quite a few
           services, (which can be listed with `cult provider drivers`).
@@ -30,15 +31,16 @@ module Cult
           a driver of the same name.
         EOD
 
-        run(arguments: 0) do |opts, args, cmd|
+        run(arguments: none) do |opts, args, cmd|
           puts cmd.help
+          exit
         end
       end
 
 
-      provider_list = Cri::Command.define do
-        name        'list'
-        aliases     'ls'
+      provider_ls = Cri::Command.define do
+        name        'ls'
+        usage       'ls [/PROVIDER+/ ...]'
         summary     'List Providers'
         description <<~EOD.format_description
           Lists Providers for this project.  If --driver is specified, it only
@@ -46,7 +48,7 @@ module Cult
         EOD
         required :d, :driver, "Restrict list to providers using DRIVER"
 
-        run(arguments: 0..1) do |opts, args, cmd|
+        run(arguments: 0 .. 1) do |opts, args, cmd|
           providers = Cult.project.providers
 
           # Filtering
@@ -65,7 +67,7 @@ module Cult
 
         end
       end
-      provider.add_command(provider_list)
+      provider.add_command(provider_ls)
 
 
       provider_avail = Cri::Command.define do
@@ -77,7 +79,7 @@ module Cult
           gem dependencies.
         EOD
 
-        run(arguments: 0) do |opts, args, cmd|
+        run(arguments: none) do |opts, args, cmd|
           Cult::Drivers.all.each do |p|
             printf "%-20s %-s\n", p.driver_name, p.required_gems
           end
@@ -86,10 +88,9 @@ module Cult
       provider.add_command(provider_avail)
 
 
-      provider_create = Cri::Command.define do
-        name        'create'
-        aliases     'new'
-        usage       'create NAME'
+      provider_new = Cri::Command.define do
+        name        'new'
+        usage       'new NAME'
         summary     'creates a new provider for your project'
         required    :d, :driver, 'Specify driver, if different than NAME'
         description <<~EOD.format_description
@@ -111,10 +112,12 @@ module Cult
           driver = CLI.fetch_item(opts[:driver] || name, from: Driver)
           name = CLI.fetch_item(name, from: Provider, exist: false)
 
+          puts JSON.pretty_generate(driver.setup!)
+          fail "FIXME"
           puts [driver, name].inspect
         end
       end
-      provider.add_command(provider_create)
+      provider.add_command(provider_new)
 
 
       provider

data/lib/cult/cli/role_cmd.rb

@@ -1,4 +1,5 @@
 require 'fileutils'
+require 'json'
 
 module Cult
   module CLI
@@ -36,28 +37,28 @@ module Cult
           `rat-site` and `tempo-site` roles and be happily serving both sites.
 
           By default, `cult init` generates two root roles that don't depend on
-          anything else: `all` and `bootstrap`.  The `bootstrap` role exists
+          anything else: `base` and `bootstrap`.  The `bootstrap` role exists
           to get a node from a clean OS install to a configuration to be
-          managed by the settings in `all'.  Theoretically, if you're happy
+          managed by the settings in `base'.  Theoretically, if you're happy
           doing all deploys as the root user, you don't need a `bootstrap` role
-          at all: Delete it and set the `user` key in `all/role.json` to
+          at all: Delete it and set the `user` key in `base/role.json` to
           "root".
 
-          The tasks in the `all` role are considered shared amongst all roles.
-          However, the only thing special about the `all` role is that Cult
+          The tasks in the `base` role are considered shared amongst all roles.
+          However, the only thing special about the `base` role is that Cult
           assumes roles and nodes without an explicit `includes` setting belong
-          to all.
+          to `base`.
         EOD
 
-        run(arguments: 0) do |opts, args, cmd|
+        run(arguments: none) do |opts, args, cmd|
           puts cmd.help
+          exit
         end
       end
 
 
-      role_create = Cri::Command.define do
-        name        'create'
-        aliases     'new'
+      role_new = Cri::Command.define do
+        name        'new'
         summary     'creates a new role'
         usage       'create [options] NAME'
         description <<~EOD.format_description
@@ -65,7 +66,7 @@ module Cult
           $CULT_PROJECT/roles/$NAME
         EOD
 
-        required :r, :roles, 'this role depends on another role',
+        required :r, :roles, 'this role depends on another /ROLE+/ (multiple)',
                  multiple: true
 
         run(arguments: 1) do |opts, args, cmd|
@@ -79,8 +80,8 @@ module Cult
                                               from: Role).map(&:name)
           end
           FileUtils.mkdir_p(role.path)
-          File.write(Cult.project.dump_name(role.definition_file),
-                     Cult.project.dump_object(data))
+          File.write(role.definition_file,
+                     JSON.pretty_generate(data))
 
           FileUtils.mkdir_p(File.join(role.path, "files"))
           File.write(File.join(role.path, "files", ".keep"), '')
@@ -89,19 +90,18 @@ module Cult
           File.write(File.join(role.path, "tasks", ".keep"), '')
         end
       end
-      role.add_command role_create
+      role.add_command(role_new)
 
 
-      role_destroy = Cri::Command.define do
-        name        'destroy'
-        aliases     'delete', 'rm'
-        usage       'destroy ROLES...'
+      role_rm = Cri::Command.define do
+        name        'rm'
+        usage       'rm /ROLE+/ ...'
         summary     'Destroy role ROLE'
         description <<~EOD.format_description
           Destroys all roles specified.
         EOD
 
-        run(arguments: 1..-1) do |opts, args, cmd|
+        run(arguments: 1 .. unlimited) do |opts, args, cmd|
           roles = args.map do |role_name|
             CLI.fetch_items(role_name, from: Role)
           end.flatten
@@ -114,20 +114,19 @@ module Cult
           end
         end
       end
-      role.add_command(role_destroy)
+      role.add_command(role_rm)
 
 
-      role_list = Cri::Command.define do
-        name        'list'
-        aliases     'ls'
-        usage       'list [ROLES...]'
+      role_ls = Cri::Command.define do
+        name        'ls'
+        usage       'ls [/ROLE+/ ...]'
         summary     'List existing roles'
         description <<~EOD.format_description
           Lists roles in this project.  By default, lists all roles.  If one or
           more ROLES are specified, only lists those
         EOD
 
-        run(arguments: 0..-1) do |opts, args, cmd|
+        run(arguments: unlimited) do |opts, args, cmd|
           roles = Cult.project.roles
           unless args.empty?
             roles = CLI.fetch_items(*args, from: Role)
@@ -140,7 +139,7 @@ module Cult
 
         end
       end
-      role.add_command(role_list)
+      role.add_command(role_ls)
 
 
       role