cloud-mu 1.9.0.pre.beta

data/test/exec_inspec.py

@@ -0,0 +1,246 @@
+#!/bin/python/
+
+import os, argparse, json, subprocess, sys, re, glob, time, yaml
+
+deploy_dirs = '/opt/mu/var/deployments'
+current_deploys = os.listdir(deploy_dirs)
+
+parser = argparse.ArgumentParser(description='To run Inspec Tests')
+
+# Local Mode Switch
+parser.add_argument('-l', action='store_true',help='Local Mode')
+# Required inspec profile name
+parser.add_argument('-p', type=str,help='Inspec Profile name')
+# Required deployment name
+parser.add_argument('-d', type=str,help='Deployment ID')
+
+args = parser.parse_args()
+
+print "Local Mode: %s" % args.l
+print "Inspec Profile: %s" % args.p
+print "Deployment ID: %s" % args.d
+
+if args.l:
+  workspace = '/opt/mu/lib'
+else:
+  ## From CI server
+  workspace = os.environ['WORKSPACE']
+
+test = workspace+'/test'
+
+def check_deploy_id(deploy_dir):
+  abs_path=deploy_dirs + '/' + deploy_dir
+  if os.path.isdir(abs_path):
+    print "Checking the deployment located in %s" % abs_path
+    return deploy_dir
+  else:
+    print "There is no deployment by the name of %s" % abs_path
+    exit()
+
+def base_controls():
+  return ['base_repositories', 'set_mu_hostname', 'disable-requiretty', 'set_local_fw', 'rsyslog', 'nrpe']
+
+
+def get_profile():
+  which_profile = args.p
+  if os.path.isdir(test+'/'+which_profile):
+    return which_profile
+  else:
+    return 'NOT_PROVIDED'
+
+
+def rebuild_inspec_lock(which_profile):
+  if os.path.exists(workspace+'/test/'+which_profile):
+    os.chdir(workspace+'/test/'+which_profile)
+    os.system('inspec vendor --overwrite')
+
+
+### Inorder to properly trigger inspec exec
+### Need to wait for each server_name_done.txt file to exist in deploy_id dir.
+### server_pools is the only stanza when this will be used
+### The only to retrieve the server_names is from the bok(the json version -- full blown one)
+### the server names are not added in the deployment.json right away therefore using the bok.json
+def get_server_names(deploy_id):
+  server_names = []
+  bok_json = json.load(open(deploy_dirs+'/'+deploy_id+'/basket_of_kittens.json'))
+  server_or_pools = server_or_server_pools(deploy_id)
+  if server_or_pools == 'server_pools':
+    num_servers = len(bok_json['server_pools'])
+    for x in range(0,num_servers):
+      server_names.insert(0,bok_json['server_pools'][x]['name'])
+  
+  return server_names
+
+
+def server_or_server_pools(deploy_id):
+  bok_json = json.load(open(deploy_dirs+'/'+deploy_id+'/basket_of_kittens.json'))
+  if bok_json.has_key('server_pools'):
+    return 'server_pools'
+  elif bok_json.has_key('servers'):
+    return 'servers'
+  else:
+    return None
+
+
+def wait_till_groomed(deploy_id,  seconds_to_poll):
+  node_names = get_server_names(deploy_id)
+  print "Nodes to check: %s "% node_names
+  print "Deploy ID: %s"% deploy_id
+  for each_node in node_names:
+    done = False
+    while done == False:
+      if os.path.isfile(deploy_dirs+'/'+deploy_id+'/'+each_node+'_done.txt'):
+        print "INFO: Server Converged   =====>    %s" % each_node
+        done = True
+      else:
+        done = False
+        print "INFO: Server: %s not fully converged. Checking back in %s seconds." % (each_node, seconds_to_poll)
+        time.sleep(seconds_to_poll)
+
+
+def get_load_balancers(deploy_id):
+  all_loads = []
+  each_load = {}
+  if os.path.isdir(deploy_dirs+'/'+deploy_id):
+      node = json.load(open(deploy_dirs+'/'+deploy_id+'/deployment.json'))
+      
+      ## Check if any?
+      if node.get('loadbalancers') != None:
+        for key,val in node['loadbalancers'].iteritems():
+          for k,v in val.iteritems():
+            each_load = {key:val['dns']}
+            all_loads.append(each_load)
+            break
+  return all_loads 
+
+
+def get_host_info(deploy_id):
+  host_infos = []
+  ssh_info = {}
+  print "INFO: Deploy ID    =====>    %s" %deploy_id
+  if os.path.isdir(deploy_dirs+'/'+deploy_id):
+    node = json.load(open(deploy_dirs+'/'+deploy_id+'/deployment.json'))
+    bok = json.load(open(deploy_dirs+'/'+deploy_id+'/basket_of_kittens.json'))
+    platform=None
+    ssh_user=None
+    fqdn = None
+      
+    try:
+      platform = bok['servers'][0]['platform']
+      ssh_user = bok['servers'][0]['ssh_user']
+    except KeyError as e:
+      platform = bok['server_pools'][0]['platform']
+      ssh_user = bok['server_pools'][0]['ssh_user']
+
+    load_balancers = get_load_balancers(deploy_id)
+    
+    ### Get SSH Key Name from deploy_id dir
+    key_file = open(deploy_dirs+'/'+deploy_id+'/ssh_key_name')
+    ssh_key = key_file.readline().strip()
+    
+    for k,v in node['servers'].iteritems():
+      control = []
+      run_list = []
+      dep = None
+  
+      ### need each servers ssh info
+      for key,val in v.iteritems():
+        if key.startswith(deploy_id+'-'+k.upper()):
+          dep = key
+          print 'INFO: Deploy ID Detected: %s' % dep
+          run_list = val['run_list']
+
+          ### Get DNS
+          if len(v[dep]['public_dns_name']) != 0:
+            fqdn = v[dep]['public_dns_name']
+          else:
+            fqdn = v[dep]['private_ip_address']
+          
+          ### Filter recipe names to map to inspec controls
+          for recipe in run_list:
+            if '::' in recipe \
+            and 'store_attr' not in recipe \
+            and 'default' not in recipe and 'nat' not in recipe:
+              recipe_name = re.search(r"\w+]", recipe).group(0).replace(']','')
+              control.insert(0,recipe_name)
+      
+      ssh_info = {'host_name':dep,'server_name': k, 'fqdn': fqdn, 'ssh_user':ssh_user, 'ssh_file': '~/.ssh/'+ssh_key, 'controls': control, 'platform': platform, 'load_balancers':load_balancers } 
+      host_infos.append(ssh_info)
+    
+  
+  else:
+    raise Exception('ERROR: '+deploy_id+' ==> deploy id does not exist in '+deploy_dirs)
+  return host_infos
+
+
+
+def store_ssh_info(array_of_ssh_info, where='/tmp'):
+  for ssh_info in array_of_ssh_info:
+    print ssh_info
+    ya = open(where+'/'+ssh_info['server_name']+'_attr.yaml','w')
+    yaml.safe_dump(ssh_info, ya,default_flow_style=False)
+
+
+def get_win_pass(deploy_id):
+  command = subprocess.check_output(['knife', 'vault', 'show', deploy_id+'-WINDOWS','windows_credentials'])
+  for line in command.splitlines():
+    if line.startswith('password:'):
+      return line.split(':')[1].strip()
+      break
+
+
+
+def run_windows_tests(deploy_id,profile,controls,ssh_user):
+  pas =  "\'%s\'" % get_win_pass(deploy_id)
+  winrm = "winrm://%s@%s" % (ssh_user,deploy_id.upper()+'-WINDOWS')
+  status = subprocess.call(['inspec','exec',profile,'--controls='+controls,'-t',winrm,'--password',pas])
+  return status
+
+def run_linux_tests(profile, ssh, ssh_file, all_controls):
+  cmd = "inspec exec %s --controls=%s -t %s -i %s" %(profile,all_controls,ssh,ssh_file)
+  stat = os.system(cmd)
+  return stat
+  
+
+inspec_retry_dir = '/tmp/inspec_retries'
+profile = get_profile()
+rebuild_inspec_lock(profile)
+os.chdir(workspace)
+deploy_id = check_deploy_id(args.d)
+server_or_pools = server_or_server_pools(deploy_id)
+if server_or_pools == 'server_pools':
+  wait_till_groomed(deploy_id, 5)
+ssh_infos = get_host_info(deploy_id)
+store_ssh_info(ssh_infos)
+
+try:
+  os.makedirs(inspec_retry_dir)
+except OSError:
+  print "SKIP ===> %s already exists" % inspec_retry_dir
+
+
+#########################################################################
+##### Run Tests ###########################################
+for ssh_info in ssh_infos:
+  
+ 
+  ### when no recipes -- switch to mu-tools-test profile
+  if len(ssh_info['controls']) == 0:
+    profile = 'mu-tools-test'
+  
+  controls = base_controls() + ssh_info['controls']
+  all_controls_spaced_out = ' '.join(controls)
+  print 'Inspec Profile: =====> %s ' %profile
+  print 'Controls To Test =====> '+all_controls_spaced_out+' <====='
+  os.chdir(test)
+  if ssh_info['platform'] == 'windows':
+    exit_status = run_windows_tests(deploy_id, profile, all_controls_spaced_out, ssh_info['ssh_user'])
+  else:
+    ssh = 'ssh://%s@%s' % (ssh_info['ssh_user'], ssh_info['fqdn'])
+    exit_status = run_linux_tests(profile, ssh, ssh_info['ssh_file'],all_controls_spaced_out)
+    if int(exit_status != 0):
+      ssh_info['profile'] = profile
+      ssh_info['bok'] = bok_name
+      retry_dump_host_info = open(inspec_retry_dir+'/'+ssh_info['server_name']+'_retry.yaml','w')
+      yaml.safe_dump(ssh_info, retry_dump_host_info,default_flow_style=False)
+

data/test/exec_mu_install.py

@@ -0,0 +1,241 @@
+#!/bin/python
+
+import subprocess, boto3, os, json, time, datetime
+
+
+### TODO:
+##### Grab the installer from a specific branch
+##### Update installer url with the correct branch name
+##### Have the CI server inject an env var of branch name if there's not one already
+##### But what CI to use -- dunno 
+
+
+#workspace=os.environ['WORKSPACE']
+ssh_data_file = '/tmp/MU-MASTER-INSTALL-TEST.json'
+branch = os.environ['GIT_BRANCH']
+#branch = 'issue_100'
+user_data= """#!/bin/bash 
+sed -i 's/#PermitRootLogin yes/PermitRootLogin yes/g' /etc/ssh/sshd_config
+sed -i 's/#PubkeyAuthentication yes/PubkeyAuthentication yes/g' /etc/ssh/sshd_config
+echo "RSAAuthentication yes" >> /etc/ssh/sshd_config
+sed -i 's/.*ssh-rsa/ssh-rsa/g' /root/.ssh/authorized_keys
+service sshd reload
+yum install wget -y
+yum install git -y
+cd /tmp 
+wget https://raw.githubusercontent.com/cloudamatic/mu/%s/install/installer 
+chmod +x installer
+ip=$(curl http://169.254.169.254/latest/meta-data/public-ipv4)
+admin_email="amrit.gill@eglobaltech.com"
+sed -i "s/\/opt\/mu\/bin\/mu-configure \$\@/\/opt\/mu\/bin\/mu-configure \$\@ -np $ip -m $admin_email/g" installer""" % branch
+
+
+def base_controls():
+  return ['default','init','basepackages','firewall-holes', 'ssl-certs', 'vault']
+
+
+
+def recipe_existence_over_ssh(user,host,key_file):
+  recipe = '/opt/mu/lib/site_cookbooks/demo/recipes/store_attr.rb'
+  output = subprocess.check_output(["ssh","-oStrictHostKeyChecking=no", "-i", key_file, user+"@"+host,"ls", recipe])
+  if 'store_attr' in output:
+    return True
+  else:
+    return False
+
+
+def chef_solo_over_ssh(user, host,key_file):
+  file_path = '/opt/mu/lib/site_cookbooks/demo/recipes/store_attr.rb'
+  exists = recipe_existence_over_ssh(user,host,key_file)
+  if exists:
+    chef_solo = 'chef-solo %s' % file_path
+    run_cmd = 'ssh -oStrictHostKeyChecking=no -i %s %s@%s %s' % (key_file, user, host, chef_solo)
+    os.system(run_cmd)
+  else:
+    raise Exception("store_attr demo recipe ==> DOES NOT EXIST ON NODE!")
+
+
+def run_installer_over_ssh(user,host, key_file, command):
+  if user != None and host != None and command != None:
+    time.sleep(90)
+    counter = 0
+    max_retries = 20
+    while (counter != max_retries):
+      out = subprocess.check_output(["ssh","-oStrictHostKeyChecking=no","-i", key_file, user+"@"+host,"ls","/tmp/"])
+      if 'installer' in out:
+        break
+      else:
+        counter += 1
+        print "Installer does not exist yet... checking back in 10 secs... (Retry %s/%s)" % (counter,max_retries)
+        time.sleep(10)
+    
+    ssh_syntax = "ssh -oStrictHostKeyChecking=no -i %s %s@%s %s" % (key_file,user,host,command)
+    os.system(ssh_syntax)
+    chef_solo_over_ssh(user,host,key_file)
+
+
+def allocat_associate_eip(ins_id):
+  ec2 = boto3.client('ec2')
+  try:
+    allocation = ec2.allocate_address(Domain='vpc')
+    res = ec2.associate_address(AllocationId=allocation['AllocationId'],InstanceId=ins_id)
+    print(res)
+  except ClientError as e:
+    print(e)
+  print allocation['AllocationId']
+  
+  ## to use release_eip, we need the alloc_id
+  return allocation['AllocationId']
+
+
+## de-register eip and then release it
+def release_eip(alloc_id):
+  ec2 = boto3.client('ec2')
+  if alloc_id != None and len(alloc_id) != 0:
+    res = ec2.release_address(AllocationId= alloc_id)
+    return res
+  else:
+    raise Exception("Please pass a valid EIP Allocation ID..")
+
+
+### Create instance to install mu-master
+def create_instance(name="MU-INSTALL-TEST", type='t2.medium',maxx=1,region='us-east-1'):
+  ec2_name = name+"{:%Y%m%d%H%M%S}".format(datetime.datetime.now())
+  ec2 = boto3.resource('ec2', region_name=region)
+  instance = ec2.create_instances(
+    ImageId = 'ami-02e98f78', ## CS7 
+    MinCount = 1,
+    MaxCount = maxx,
+    InstanceType = type,
+    KeyName='MU-MASTER-INSTALL-TEST', ## exists prior
+    UserData = user_data,
+    SecurityGroupIds = ['sg-66fdfe12'],
+    TagSpecifications=[
+      {
+        'ResourceType':'instance',
+        'Tags':[
+          {
+            'Key': 'Name',
+            'Value':ec2_name
+          }
+        ]
+      }
+    ],
+
+    SubnetId = 'subnet-7dd64636',
+    IamInstanceProfile={
+      'Name': 'mu-master-role'
+    }
+  )
+  ins = instance[0]
+  ins.wait_until_running()
+  ins.load() ## reload
+  instances = ins.id
+  print "Instance Initliazed %s " % instances
+  return instances
+
+
+
+### Pull out servers ssh info for inspec
+def desc_instances(ins_ids,region='us-east-1'):
+  ec2 = boto3.client('ec2')
+  bootstrap = {}
+  all_bootstraps = []
+  ins = ec2.describe_instances(InstanceIds=ins_ids)
+
+  for each_running in ins['Reservations']:
+    key = None
+    fqdn = None
+    name = None
+    ins_id = None
+    pub_ip = None
+    for each in each_running['Instances']:
+      
+      key = '/root/.ssh/'+each['KeyName']+'.pem'
+      fqdn = each['PublicDnsName']
+      name = each['Tags'][0]['Value']
+      ins_id = each['InstanceId'] 
+      pub_ip = each['PublicIpAddress']
+      private_dns_address = each['PrivateDnsName']
+      break
+
+    bootstrap = {'pub_ip': pub_ip, 'private_dns': private_dns_address, 'key':key, 'fqdn':fqdn,'name':name, 'ins_id':ins_id}
+    all_bootstraps.append(bootstrap)
+  print all_bootstraps
+  return all_bootstraps
+
+
+def run_master_test(ssh_data_file,controls):
+  if os.path.isfile(ssh_data_file):
+    ssh_info = json.load(open(ssh_data_file))
+    os.chdir("/opt/mu/lib/test")
+    cmd = "inspec exec mu-master-test --controls=%s -t ssh://root@%s -i %s" % (controls, ssh_info[0]['fqdn'],ssh_info[0]['key'])
+    exit = os.system(cmd)
+    return exit
+  else:
+    raise Exception("ssh file does not exist: %s" % ssh_data_file)
+
+def dump_ssh_info(data):
+  new_file = open(ssh_data_file,'w')
+  json.dump(data, new_file)
+
+
+def rm_chef_node_json_frm_target(user,host,key,file_to_rm="/tmp/chef_node.json"):
+  if user != None and host != None and os.path.isfile(key):
+    cmd = "ssh -oStrictHostKeyChecking -i %s %s@%s rm -rf %s" %(key,user,host,file_to_rm)
+    exit = os.system(cmd)
+    return exit
+  else:
+    raise Exception("Check user, host and make sure key file exists on this machine")
+
+
+def clean_up(ins_ids,bucket_name):
+  ec2 = boto3.resource('ec2')
+  s3 = boto3.resource('s3')
+  if s3.Bucket(bucket_name) in s3.buckets.all():
+    print bucket_name
+    master_bucket = s3.Bucket(bucket_name)
+    master_bucket.object_versions.all().delete()
+    master_bucket.delete()
+  
+  print "*********************************************"
+  print "INFO: CLEANED UP: %s" % ins_ids
+  print "*********************************************"
+  ec2.instances.filter(InstanceIds=ins_ids).terminate()  
+  termination_wait(ins_ids)
+
+def termination_wait(ins_ids):
+  ec2 = boto3.client('ec2')
+  wait = ec2.get_waiter('instance_terminated')
+  wait.wait(InstanceIds=ins_ids)
+
+
+######## Main 
+instance_ids = []
+instance_ids.append(create_instance())
+eip_id = allocat_associate_eip(instance_ids[0])
+controls = base_controls()
+controls_spaced_out = ' '.join(controls)
+data = desc_instances(instance_ids)
+dump_ssh_info(data)
+
+if os.path.isfile(ssh_data_file):
+  ssh_info = json.load(open(ssh_data_file))    
+  run_installer_over_ssh('root',ssh_info[0]['fqdn'],ssh_info[0]['key'],'sh /tmp/installer')
+  time.sleep(10)
+  status = run_master_test(ssh_data_file, controls_spaced_out) 
+  #if status != 0:
+  #  raise Exception("Mu Master Installation Failed")
+  
+  # why would I use this when I am terminating the instance .... 
+  #rm_chef_node_json_frm_target('root',ssh_info[0]['fqdn'],ssh_info[0]['key'])
+  cleanup_ids = []
+  for each in ssh_info:
+    cleanup_ids.append(each['ins_id']) 
+  clean_up(cleanup_ids, ssh_info[0]['private_dns'])
+  release_eip(eip_id)
+else:
+  print("Nothing to do! Instance Data file does not exists: %s" % ssh_data_file)
+
+
+os.system("rm -rf "+ssh_data_file)

data/test/exec_retry.py

@@ -0,0 +1,44 @@
+#!/bin/python
+
+import os, json, subprocess, sys, re, glob, time, yaml
+
+workspace = os.environ['WORKSPACE']
+test = workspace+'/test'
+all_retries = '/tmp/inspec_retries'
+
+
+def get_all_yml_files():
+  return os.listdir(all_retries)
+
+## To run base again or to not? for now just run it
+def base_controls():
+  return ['base_repositories', 'set_mu_hostname', 'disable-requiretty', 'set_local_fw', 'rsyslog', 'nrpe']
+
+os.chdir(test)
+files = get_all_yml_files()
+if len(files) != 0:
+  for x in files:
+      f = yaml.load(open(all_retries+'/'+x,'r'))
+
+      
+      
+      print ('=============== INFO: Auto Retry in 60 secs============== \nNode: %s \nBOK: %s \nProfile: %s \nControl: %s\nRetry File: %s===================================' % (f['server_name'],f['bok'],f['profile'],f['controls'],x))
+      
+      
+      
+      ssh = 'ssh://%s@%s' % (f['ssh_user'], f['fqdn'])
+      ssh_file = str(f['ssh_file'])
+      ctrls = (base_controls()+f['controls'])
+      ctrls_spaced_out = ' '.join(ctrls)
+      
+      
+      cmd = "inspec exec %s --controls=%s -t %s -i %s" %(f['profile'],ctrls_spaced_out,ssh,ssh_file)
+      time.sleep(60) 
+      stat = os.system(cmd)
+      if stat != 0:
+        raise Exception("Retry Failed on %s " % f['server_name'])
+      else:
+        os.system('rm -rf /tmp/inspec_retries/%s' % x)
+else:
+  print "All Tests passed! No retry files found!!!"
+

data/test/mu-master-test/README.md

@@ -0,0 +1,3 @@
+# Example InSpec Profile
+
+This example shows the implementation of an InSpec profile.