noms-client 1.9.0

data/bin/noms

@@ -0,0 +1,774 @@
+#!/usr/bin/env ruby
+# /* Copyright 2014 Evernote Corporation. All rights reserved.
+#    Copyright 2013 Proofpoint, Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# */
+
+# = NAME
+#
+# noms - CLI for New Operations Management Stack
+#
+# = SYNOPSIS
+#
+#    noms command [options] [arguments]
+#       cmdb subcommand
+#          query [condition [...]]   - Query for systems
+#          set fqdn field=value [field=value [...]] -
+#                                      Update system attributes
+#          show fqdn                 - Show fields for one system
+#       waitfor expr cmdb query      - Waitfor CMDB query to have
+#                                      specified number of results
+#       instance subcommand          - Operate on NOMS instances
+#          add [cloud] [attribute=value [...]] -
+#                                      Add a new NOMS instance
+#          list [cloud]              - List NOMS instances (in cloud)
+#          show [cloud/]instanceid
+#          show [cloud/]name         - Show details of cloud instance
+#                                      (can include --fields console)
+#          remove [cloud/]instanceid
+#          remove [cloud/]name       - Terminate a cloud instance
+#       environment subcommand - Operate on NOMS environment
+#          add name [environment_name=value] [note=value] -
+#                                      Add a new environment
+#          list                      - List environments
+#          tree                      - Show environments in a tree
+#          show name                 - Show details on <environment>
+#          remove name               - Remove environment
+#       svcinst subcommand           - Operate on NOMS service instances
+#          add [environment/]service - Add service instance
+#          set [environment/]service property=value -
+#                                      Set property on service instance
+#          show [environment/]service- Show details on service instance
+#          list [environment]        - List service instances for environment
+#          remove [environment/]service -
+#                                      Remove a service instance from an
+#                                      environment
+#       describe item                - Describe meta-data for NOMS
+#          clouds                    - List clouds
+#          cloud name                - Describe named cloud
+#          ncc                       - Describe NCC API endpoint
+#          cmdb                      - Describe system fields
+#    Options:
+#       --names                      - Just print entity names, equivalent to
+#                                      --fields name --noheader --nofeedback --nolabel
+#       --fields field[,field[,...]]
+#       --format {text | json | csv} - Print output in specified format
+#       --timeout                    - Timeout for network operations (especially
+#                                      relevant for waitfor)
+#       --noheader
+#       --nofeedback
+#       --nolabel
+#       options as described in Optconfig
+#
+#
+# = DESCRIPTION
+#
+# The +noms+ command is used to set up and manage virtual and physical environments.
+#
+# == Commands
+#
+# * describe
+#
+# The +describe+ command (+desc+ is a synonym) is used to describe different
+# things about the NOMS environment.
+#
+# ** commands
+#
+# List the different commands available
+#
+# * instance
+#
+# The +instance+ command is used for operating on individual instances in the
+# NOMS environment. Use it by invoking subcommands to add, remove and list
+# instances.
+#
+# ** list
+#
+# List instances. Use the *--verbose* option to list more fields
+#
+# Example: +noms instance list us-east-1+
+#
+# ** show
+#
+# Show information about the named instance.
+#
+# Example: +noms instance show us-east-1/webserver1.ops-lab.net+
+#
+# * cmdb
+#
+# Perform operations on the CMDB.
+#
+# ** show
+#
+# Show field values for the specified system (identified by fqdn).
+#
+# Example: +noms cmdb show webserver1.ops-lab.net+
+#
+# ** query
+#
+# Perform a query and display tabular results (see the *--fields*
+# and *--format* option).
+#
+# Example: +noms cmdb query status=allocated data_center_code=XDC1 'tags!~backup'+
+#
+# ** set
+#
+# Set field values for the system in the CMDB.
+#
+# Example: +noms cmdb set webserver1.ops-lab.net status=decommissioned+
+#
+# * waitfor
+#
+# Wait for a CMDB query to have a specified number of result rows. Possible value of
+# the result row count expression is a number signifying a number of rows to exactly match,
+# or '>n' indicating that more than _n_ rows must match. The following syntax is 'cmdb query
+# field-expression', just as with the 'cmdb query' command.
+#
+# Example: +noms waitfor >1 cmdb query status=idle roles=xenserver+
+#
+# == Options
+#
+# * --names
+#
+# Just print the names of the entries. (Synonymous with --noheader --nofeedback --fields=fqdn
+# or --fields=name)
+#
+# Example: +noms cmdb query status=idle --names+
+#
+# Example: +noms instance list --names+
+#
+# * --fields
+#
+# Use the specified comma-separated list of fields for printing results.
+#
+# Example: +noms cmdb query 'status!=decommissioned' --fields=fqdn,operatingsystem,status+
+#
+# * --format
+#
+# Specify the output format. One of: text, csv or json (default is 'text').
+#
+# Example: +noms cmdb show webserver1.ops-lab.net --format=json+
+#
+# * --noheader
+#
+# Don't print column headings (in text or CSV output).
+#
+# * --nofeedback
+#
+# Don't print object counts.
+#
+# = AUTHOR
+#
+# Jeremy Brinkley, <jbrinkley@evernote.com>
+#
+
+require 'rubygems'
+require 'optconfig'
+require 'etc'
+require 'csv'
+require 'noms/client/version'
+require 'ncc/client'
+require 'noms/cmdb'
+
+$VERSION = NOMS::Client::VERSION
+$me = 'noms'
+
+$opt = Optconfig.new('noms', {
+                         'ncc=s%' => {
+                             'url' => 'http://noms/ncc_api/v2' },
+                         'cmdb=s%' => {
+                             'url' => 'http://cmdb/cmdb_api/v1' },
+                         'fields=s' => '',
+                         'mock=s' => nil,
+                         'format=s' => 'text',
+                         'names!' => false,
+                         'label!' => true,
+                         'header!' => true,
+                         'feedback!' => true,
+                         'waitfor-interval=i' => 5,
+                         'timeout=i' => 600,
+                         'default-environment' => 'production',
+                         'default-cloud' => 'ec2',
+                         'format-field=s%' => {
+                             'svcinst' => {
+                                 'fields' => [
+                                     'name', 'type', 'note'
+                                 ],
+                                 'length' => {
+                                     'name' => 20,
+                                     'type' => 20,
+                                     'note' => 30
+                                 }
+                             },
+                             'environment' => {
+                                 'fields' => [
+                                     'name', 'environment_name', 'note',
+                                 ],
+                                 'length' => {
+                                     'name' => 20,
+                                     'environment_name' => 20,
+                                     'note' => 30
+                                 }
+                             },
+                             'instance' => {
+                                 'fields' => [
+                                     'name', 'status', 'size', 'image',
+                                     'id'],
+                                 'length' => {
+                                     'name' => 36,
+                                     'status' => 10,
+                                     'size' => 10,
+                                     'id' => 36,
+                                     'image' => 15
+                                 }
+                             },
+                             'system' => {
+                                 'fields' => [
+                                     'fqdn', 'environment_name', 'status', 'roles', 'ipaddress',
+                                     'data_center_code'],
+                                 'length' => {
+                                     'fqdn' => 36,
+                                     'environment_name' => 16,
+                                     'status' => 15,
+                                     'ipaddress' => 15,
+                                     'data_center_code' => 11,
+                                     'svc_id' => 10,
+                                     'cloud' => 10,
+                                     'serial_number' => 36,
+                                     'tags' => 20
+                                 }
+                             }
+                         }
+                     })
+
+if $opt['mock']
+    NCC::Client.mock! $opt['mock']
+    NOMS::CMDB.mock! $opt['mock']
+end
+
+case $opt['format']
+when 'csv'
+    $opt['feedback'] = false
+when 'json'
+    $opt['feedback'] = false
+    $opt['header'] = false
+when 'text'
+else
+    fatal "Don't know how to produce #{$opt['format']} output"
+end
+
+if $opt['names']
+    $opt['header'] = false
+    $opt['feedback'] = false
+end
+
+$opt['cmdb']['timeout'] ||= $opt['timeout']
+$opt['ncc']['timeout'] ||= $opt['timeout']
+
+$cmdb = NOMS::CMDB.new($opt)
+$ncc = NCC::Client.new($opt)
+
+def parse_format(fields, type)
+    dbg "Specified fields: #{fields.inspect}"
+    if fields.nil? or fields.empty?
+        dbg "   no specification"
+    elsif fields.respond_to? :to_ary
+        dbg "   fields are already array"
+    else
+        dbg "   splitting on ','"
+        fields = fields.split(',')
+    end
+    if ! fields.empty?
+        fields.map do |fieldname|
+            if m = /([^=]+)=(\d+)/.match(fieldname)
+                fieldname = m[1]
+                $opt['format-field'][type]['length'][fieldname] = m[2].to_i
+            end
+            fieldname
+        end
+    else
+        nil
+    end
+end
+
+def fmt_string(type='instance', fields=nil)
+    fields = fmt_fields(type, fields)
+    fields.map do |field|
+        len = $opt['format-field'][type]['length'][field]
+        len = field.length unless (len and len > 0)
+        "%-#{len}s"
+    end.join(" ")
+end
+
+def fmt_fields(type='instance', fields=nil)
+    if fields.nil?
+        fields = parse_format($opt['fields'], type) || $opt['format-field'][type]['fields']
+    end
+    fields
+end
+
+def fmt_line(obj, type='instance', fields=nil)
+    fields = fmt_fields(type, fields)
+    case $opt['format']
+    when 'csv'
+        CSV.generate_line(fields.map { |f| obj[f] }).chomp
+    else
+        fmt_string(type, fields) % fields.map { |f| obj[f] }
+    end
+end
+
+def fmt_header(type='instance', fields=nil)
+    fields = fmt_fields(type, fields)
+    case $opt['format']
+    when 'csv'
+        CSV.generate_line(fields).chomp
+    else
+        fmt_string(type, fields) % fields
+    end
+end
+
+def formatted_output(objects, type='instance', fields=nil)
+    puts fmt_header(type, fields) if $opt['header']
+    results =
+        case $opt['format']
+        when 'json'
+            fields = fmt_fields(type, fields)
+            fieldlist = parse_format($opt['fields'], type) || fields
+            JSON.pretty_generate(objects.map do |o|
+                                     Hash[fieldlist.map { |f| [f, o[f]] }]
+                                 end)
+        else
+            objects.map do |obj|
+            fmt_line(obj, type, fields)
+        end.join("\n")
+    end
+    results += "\n#{objects.length} objects" if $opt['feedback']
+    results
+end
+
+def record_formatted_output(object, type='instance', fields=nil)
+    fields = fmt_fields(type, fields)
+    all_fields = object.keys
+    fields = fields & all_fields
+    fieldlist = parse_format($opt['fields'], type) || fields + (all_fields - fields)
+    case $opt['format']
+    when 'json'
+        JSON.pretty_generate(Hash[fieldlist.map { |f| [f, object[f]] }])
+    when 'csv'
+        ($opt['label'] ? CSV.generate_line(fieldlist) : '') +
+            CSV.generate_line(fieldlist.map { |f| object[f] })
+    else
+        fieldlist.map { |f|
+            $opt['label'] ? "#{f}: #{object[f]}" : object[f]
+        }.join("\n")
+    end
+end
+
+def extractkvs(args)
+    args.inject([{}, []]) do |a, arg|
+        if /^(\w+)=/.match arg
+            key, value = arg.split('=')
+            [a[0].merge({ key.to_sym => value }), a[1]]
+        else
+            [a[0], a[1] << arg]
+        end
+    end
+end
+
+def parsekvs(args)
+   args.inject({}) do |h, arg|
+      key, value = arg.split('=')
+      h.merge({ key.to_sym => value })
+   end
+end
+
+def get_username
+    Etc.getpwuid(Process.uid).name
+end
+
+def fatal(msg)
+   $stderr.puts "noms error: #{msg}"
+   Process.exit(1)
+end
+
+def dbg(msg)
+   if $opt.has_key? 'debug' and $opt['debug'] > 0
+       puts "DBG(noms): #{msg}"
+   end
+end
+
+def hash_from_array(a, field='id')
+    Hash[a.map { |e| [e[field], e] }]
+end
+
+def generic_describe(h)
+    h.map do |k, v|
+        "#{k}: #{v}"
+    end.join("\n")
+end
+
+def desc(args)
+   what = args.shift
+   case what
+   when 'commands', nil
+       generic_describe({
+                            'describe' => 'describe things',
+                            'commands' => 'list commands',
+                            'cmdb' => 'interact with NOMS CMDB (Inventory)',
+                            'instance' => 'interact with clouds (NCC-API)',
+                            'waitfor' => 'wait for CMDB query to be satisfied',
+                            'environment' => 'interact with NOMS CMDB environments',
+                            'svcinst' => 'manage CMDB service instances'
+                        })
+   when 'clouds'
+       generic_describe Hash[$ncc.clouds.map { |e| [e['name'],
+                                     e['provider']] }]
+   when 'cloud'
+       generic_describe $ncc.clouds args.first
+   when 'ncc'
+       generic_describe $ncc.info
+   when 'systems', 'cmdb'
+       help = $cmdb.help 'system'
+       puts help.inspect
+       generic_describe(help.select { |f|
+                            f.respond_to? :has_key? and f.has_key? 'label' }.map { |f| f['label'] })
+   when 'describe', 'desc'
+       generic_describe ({
+                             'describe' => 'describe resources',
+                             'clouds' => 'describe available clouds',
+                             'cloud' => 'describe cloud',
+                             'ncc' => 'describe NCC API',
+                             'systems|cmdb' => 'describe CMDB entries (system)'
+                         })
+   else
+       $stderr.puts "Can't describe '#{what}'"
+   end
+end
+
+def cmdb_query(args)
+    $cmdb.query('system', args)
+end
+
+def cmdb_show(args)
+    fqdn = args.shift
+    system = $cmdb.query('system', 'fqdn=' + fqdn).first
+    record = if ! args.empty?
+                 system.keys.inject({}) do |h, k|
+            if args.include? k
+                h.merge({k => system[k]})
+            else
+                h
+            end
+        end
+             else
+                 system
+             end
+    record
+end
+
+def cmdb(args)
+    $opt['fields'] = ['fqdn'] if $opt['names']
+    cmd = args.shift
+    opt, argv = extractkvs args
+    case cmd
+    when 'query'
+        # Note, args, not argv
+        results = cmdb_query args
+        formatted_output(results, 'system')
+    when 'show','info'
+        record = cmdb_show args
+        record_formatted_output(record, 'system')
+    when 'set'
+        fqdn = args.shift
+        updated = $cmdb.update('system', opt, fqdn)
+        ''
+    else
+        $stderr.puts "Unknown cmdb command '#{cmd}'"
+        Process.exit(2)
+    end
+end
+
+def eval_conditions(count, results)
+    expr =
+        case count
+        when "0", 0
+            lambda { |r| r.nil? or r.length == 0 }
+        when /^>/
+            m = /^>(\d+)/.match count
+            ct = m[1].to_i
+            lambda { |r| !r.nil? and r.length > ct }
+        else
+            ct = count.to_i
+            lambda { |r| !r.nil? and r.length == ct }
+        end
+
+    expr.call results
+end
+
+# TODO: Needs to be factored into general waiting lib
+def waitfor(args)
+    count = args.shift
+    commands = ['cmdb']
+    conditions = []
+
+    command = args.shift
+    subcommand = args.shift
+    continuing = true
+
+    startts = Time.now
+    while continuing
+        results = case command
+                  when 'cmdb'
+                      case subcommand
+                      when 'query'
+                          cmdb_query args
+                      else
+                          fatal "Can only wait for cmdb query or show"
+                      end
+                  else
+                      fatal "Can only wait for cmdb"
+                  end
+        if eval_conditions(count, results)
+            dbg "Conditions satisfied (#{count} records)"
+            Process.exit(0)
+            continuing = false
+        else
+            dbg "Conditions not satisfied"
+            if (Time.now - startts) > $opt['timeout']
+                $stderr.puts "Timed out waiting for #{count} records of " +
+                    "#{command} #{subcommand} #{args}"
+                Process.exit(4)
+            end
+            dbg "Waiting #{$opt['waitfor-interval']}"
+            sleep $opt['waitfor-interval']
+        end
+    end
+end
+
+def find_key(h, k, p=nil)
+    puts "         - looking for #{k} at path #{p.inspect}"
+    if h.has_key? k
+        return (p || []) + [k]
+    else
+        h.keys.each do |nk|
+            find_key(h[nk], k, (p || []) + [nk])
+        end
+    end
+    return nil
+end
+
+def make_tree(envs, tree={})
+    puts "make_tree(#{envs.inspect}, #{tree.inspect})"
+    return tree if envs.length == 0
+    # Kind of ugly using select for side effects
+    pruned_envs = envs.reject do |env|
+        puts "   checking #{env['name']} < #{env['environment_name']}"
+        if env['environment_name'].nil? or env['name'] == env['environment_name']
+            puts "      root environment, setting"
+            tree[env['name']] ||= { }
+            true
+        elsif path = find_key(tree, env['environment_name'])
+            puts "      found parent #{env['environment_name']} at #{path.inspect}"
+            path_set(tree, path, { env['name'] => { } })
+            true
+        else
+            puts "      couldn't find #{env['environment_name']}, save for later"
+            false
+        end
+    end
+    make_tree(pruned_envs, tree)
+end
+
+def path_set(hash, keypath, value)
+    if keypath.length == 1
+        hash[keypath[0]] = value
+    else
+        next_key = keypath.shift
+        path_set(hash[next_key], keypath, value)
+    end
+end
+
+def env(args)
+    $opt['fields'] = ['name'] if $opt['names']
+    command = args.shift
+    case command
+    when 'list'
+        formatted_output $cmdb.environments, 'environment'
+    when 'show', 'info'
+        env_name = args.shift
+        record_formatted_output($cmdb.environment(env_name), 'environment')
+    when 'tree'
+        make_tree($cmdb.environments).to_yaml
+    when 'add'
+        name = args.shift
+        attrs = { :environment_name => 'production' }
+        attrs.update(args.empty? ? { } : parsekvs(args))
+        record_formatted_output($cmdb.create_environment(name, attrs))
+    when 'remove'
+        name = args.shift
+        result = $cmdb.delete_environment name
+        # cmdb-api returns '1'. ugh.
+        nil unless result != 1
+    else
+        $stderr.puts "Unknown env command '#{command}'"
+        Process.exit(2)
+    end
+end
+
+def parse_qualified_name(s, default_qualifier)
+    qualifier, name = s.split('/', 2)
+    if name.nil?
+        name = qualifier
+        qualifier = default_qualifier
+    end
+    return qualifier, name
+end
+
+def parse_svcinst(s)
+    parse_qualified_name(s, $opt['default-environment'])
+end
+
+def svcinst(args)
+    $opt['fields'] = ['name'] if $opt['names']
+    command = args.shift
+    case command
+    when 'list'
+        formatted_output($cmdb.services(args[0] || $opt['default-environment']), 'svcinst')
+    when 'show', 'info'
+        env, svc = parse_svcinst(args[0])
+        record_formatted_output($cmdb.service(env, svc), 'svcinst')
+    when 'add'
+        env, svc = parse_svcinst(args.shift)
+        attrs = parsekvs(args)
+        $cmdb.create_service(env, svc, attrs)
+        nil
+    when 'remove'
+        env, svc = parse_svcinst(args.shift)
+        $cmdb.delete_service(env, svc)
+    when 'set'
+        env, svc = parse_svcinst(args.shift)
+        attrs = parsekvs(args)
+        $cmdb.update_service(env, svc, attrs)
+        nil
+    else
+        $stderr.puts "Unknown svcinst '#{command}'"
+    end
+end
+
+def outs(output)
+    unless [nil, true, false].include? output
+        puts output.to_s if (output.to_s and output.to_s.length != 0)
+    end
+end
+
+def dispatch_command(argv)
+    command = argv.shift
+    case command
+    when 'help'
+        $stderr.puts "Use noms --help"
+        Process.exit(1)
+    when 'describe', 'desc'
+        outs desc argv
+    when 'cmdb', 'inv'
+        outs cmdb argv
+    when 'svc', 'svcinst'
+        outs svcinst argv
+    when 'environment', 'env'
+        outs env argv
+    when 'instance'
+        outs instance argv
+    when 'waitfor'
+        waitfor argv
+    when nil
+        $stderr.puts "Usage:\n   noms --help\n   noms {desc|cmdb|instance|waitfor|...} [arg]"
+        Process.exit(1)
+    else
+        $stderr.puts "Unknown command '#{command}'"
+        Process.exit(1)
+    end
+end
+
+def parse_instance(s)
+    parse_qualified_name(s, $opt['default-cloud'])
+end
+
+def is_name(n)
+    true if /\./.match n
+end
+
+def instance(args)
+    $opt['fields'] = ['name'] if $opt['names']
+    cmd = args.shift
+    case cmd
+    when 'list', 'ls'
+        if args.length < 1
+            args.unshift $opt['default-cloud']
+        end
+        results = $ncc.list *args
+        formatted_output(results, 'instance')
+    when 'info', 'show'
+        if args.length == 2
+            cloud, id = *args
+        elsif args.length == 1
+            cloud, id = parse_instance(args.first)
+        else
+            fatal "Usage: noms instance show [cloud/]instance"
+        end
+        if is_name(id)
+            instance = $ncc.find_by_name cloud, id
+        else
+            instance =  $ncc.instance cloud, id
+        end
+        if $opt['fields'].include? 'console'
+            console = $ncc.console cloud, instance['id']
+            instance['console'] = console['url'] if (console and console.has_key? 'url')
+        end
+        record_formatted_output instance
+    when 'add'
+        instanceopt, argv = extractkvs args
+        if argv.length == 0
+            cloud = $opt['default-cloud']
+        elsif argv.length == 1
+            cloud = argv.first
+        else
+            fatal "Usage: noms instance add [cloud] [param=value [...]]"
+        end
+        attrs = { :username => get_username, :size => 'm1.small' }
+        attrs.update(instanceopt.empty? ? { } : instanceopt)
+        dbg attrs.inspect
+        inst = $ncc.create(cloud, attrs)
+        record_formatted_output(inst) unless inst.nil?
+    when 'remove', 'rm'
+        if args.length == 2
+            cloud, id = *args
+        elsif args.length == 1
+            cloud, id = parse_instance(args.first)
+        else
+            fatal "Usage: noms instance remove [cloud.]instance"
+        end
+        fatal("No id to remove") if id.nil?
+        if is_name(id)
+            attrs = { :name => id }
+        else
+            attrs = { :id => id }
+        end
+        $ncc.delete(cloud, attrs)
+   else
+        $stderr.puts "Unknown instance command #{cmd}"
+        Process.exit(3)
+   end
+end
+
+dispatch_command ARGV
+