prepd 0.1.1 → 0.3.0

data/files/machine/json.rb

@@ -0,0 +1,26 @@
+#!/usr/bin/env ruby
+require 'json'
+
+# path, prefix, name, version = ARGV
+prefix, name, version = ARGV
+path = "#{Dir.pwd}/boxes"
+# prefix = 'prepd'
+# name = 'debian-stretch-amd64-developer'
+# version = '9.9.9'
+
+json = {
+  name: "#{prefix}/#{name}",
+  versions: [
+    {
+      version: version,
+      providers: [
+        {
+          name: 'virtualbox',
+          url: "file://#{path}/#{name}.box"
+        }
+      ]
+    }
+  ]
+}
+
+File.open("#{path}/#{name}.json", 'w') { |f| f.write json.to_json }

data/files/machine/push.json

@@ -0,0 +1,56 @@
+{
+  "variables": {
+    "aws_profile": "{{env `AWS_PROFILE`}}",
+    "vm_base_name": "{{env `VM_BASE_NAME`}}",
+    "vm_name": "{{env `VM_BASE_NAME`}}-{{env `VM_OUTPUT`}}",
+    "vm_input": "{{env `VM_INPUT`}}",
+    "vm_output": "{{env `VM_OUTPUT`}}",
+    "box_version": "{{env `BOX_VERSION`}}",
+    "s3_bucket": "{{env `S3_BUCKET`}}",
+    "s3_region": "{{env `S3_REGION`}}",
+    "s3_box_dir": "{{ env `S3_BOX_DIR`}}"
+  },
+
+  "builders": [
+    {
+      "guest_additions_mode": "disable",
+      "headless": true,
+      "output_directory": "images_cache/{{user `vm_output`}}",
+      "shutdown_command": "echo 'halt -p' > shutdown.sh; echo 'vagrant'|sudo -S sh 'shutdown.sh'",
+      "source_path": "images/{{user `vm_input`}}/{{user `vm_base_name`}}-{{user `vm_input`}}.ovf",
+      "ssh_username": "vagrant",
+      "ssh_password": "vagrant",
+      "ssh_wait_timeout": "30s",
+      "type": "virtualbox-ovf",
+      "vm_name": "{{user `vm_name`}}"
+    }
+  ],
+
+  "post-processors": [
+    [
+      {
+        "keep_input_artifact": true,
+        "output": "{{user `s3_box_dir`}}/{{user `vm_name`}}.box",
+        "type": "vagrant"
+      },
+      {
+        "type":     "vagrant-s3",
+        "box_dir":  "{{user `s3_box_dir`}}",
+        "box_name": "{{user `vm_name`}}",
+        "bucket":   "{{user `s3_bucket`}}",
+        "manifest": "{{user `vm_name`}}.json",
+        "profile": "{{user `aws_profile`}}",
+        "region": "{{user `s3_region`}}",
+        "version":  "{{ user `box_version` }}"
+      },
+      {
+        "type": "shell-local",
+        "inline": [
+          "rm -rf images/{{user `vm_input`}}",
+          "mv images_cache/{{user `vm_input`}} images",
+          "rmdir images_cache"
+        ]
+      }
+    ]
+  ]
+}

data/files/machine/rebuild.json

@@ -0,0 +1,60 @@
+{
+  "variables": {
+    "box_namespace": "{{env `BOX_NAMESPACE`}}",
+    "box_version": "{{env `BOX_VERSION`}}",
+    "playbook_file": "{{env `PLAYBOOK_FILE`}}",
+    "vm_base_name": "{{env `VM_BASE_NAME`}}",
+    "vm_name": "{{env `VM_BASE_NAME`}}-{{env `VM_OUTPUT`}}",
+    "vm_input": "{{env `VM_INPUT`}}",
+    "vm_output": "{{env `VM_OUTPUT`}}",
+    "json_rb_file": "{{env `JSON_RB_FILE`}}"
+  },
+
+  "builders": [
+    {
+      "type": "virtualbox-ovf",
+      "guest_additions_mode": "disable",
+      "headless": true,
+      "output_directory": "images_cache/{{user `vm_output`}}",
+      "shutdown_command": "echo 'halt -p' > shutdown.sh; echo 'vagrant'|sudo -S sh 'shutdown.sh'",
+      "source_path": "images/{{user `vm_input`}}/{{user `vm_base_name`}}-{{user `vm_input`}}.ovf",
+      "ssh_username": "vagrant",
+      "ssh_password": "vagrant",
+      "ssh_wait_timeout": "30s",
+      "vm_name": "{{user `vm_name`}}"
+    }
+  ],
+
+  "provisioners": [
+    {
+      "type": "ansible",
+      "groups": [ "{{user `vm_output`}}" ],
+      "playbook_file": "{{user `playbook_file`}}",
+      "user" : "vagrant"
+    }
+  ],
+
+  "post-processors": [
+    [
+      {
+        "type": "vagrant",
+        "keep_input_artifact": true,
+        "output": "boxes/{{user `vm_name`}}.box"
+      },
+      {
+        "type": "shell-local",
+        "inline": [
+          "rm -rf images/{{user `vm_input`}}",
+          "mv images_cache/{{user `vm_input`}} images",
+          "rmdir images_cache"
+        ]
+      },
+      {
+        "type": "shell-local",
+        "inline": [
+          "{{user `json_rb_file`}} {{user `box_namespace`}} {{user `vm_name`}} {{user `box_version`}}"
+        ]
+      }
+    ]
+  ]
+}

data/files/project/provision.yml

@@ -0,0 +1,20 @@
+#!/usr/bin/env ansible-playbook
+---
+- hosts: development
+  tasks:
+    - include_role:
+        name: prepd/machine
+
+    - name: Include vars
+      include_vars:
+        file: vars.yml
+        name: projects
+
+    - include_role:
+        name: prepd/project/setup
+        tasks_from: clone
+      vars:
+        projects_dir: '{{ prepd_machine.dirs.projects }}'
+      with_dict: '{{ projects }}'
+      loop_control:
+        loop_var: project

data/files/project/vars.yml

@@ -0,0 +1,5 @@
+---
+example:
+  repo:
+    name: devops
+    url: git@github.com:rjayroach/prepd-project.git

data/files/setup.yml

@@ -0,0 +1,16 @@
+# NOTE: Custom values for the developer's machines
+# TODO:
+# move the machine hash to the machine's vars/setup.yml
+# shell and aliases_file are properties of the developer so put them under a 'developer' hash key
+---
+machine_time_zone: 'Asia/Singapore'
+machine_shell: zsh
+machine_aliases_file: '{{ ansible_env.HOME }}/.zsh.after/aliases.zsh'
+machine_projects_dir: '{{ ansible_env.HOME }}/dev'
+
+machine:
+  time_zone: 'Asia/Singapore'
+  shell: zsh
+  aliases_file: '{{ ansible_env.HOME }}/.zsh.after/aliases.zsh'
+
+machine_env: '{{ machine }}'

data/files/setup/README.md

@@ -0,0 +1,21 @@
+# prepd
+
+## Getting Started with a new Host
+
+First, modify vars.yml to
+
+- include your git name and email
+- the list of extra packages to install
+
+Next, run the setup.yml playbook. It will fail on the first attempt to install VirtualBox
+
+After installation fails go to security settings and enable kernel extensions from Oracle Amercia
+
+Then rerun setup.yml
+
+
+```bash
+prepd setup
+```
+
+Full documentation is available [here](https://github.com/rjayroach/prepd/tree/master/docs)

data/files/setup/ansible.cfg

@@ -0,0 +1,4 @@
+[defaults]
+inventory = hosts
+retry_files_enabled = no
+# vault_password_file = ~/prepd/config/developer/vault-keys/password.txt

data/files/setup/hosts

@@ -0,0 +1 @@
+localhost ansible_connection=local

data/files/setup/setup.yml

@@ -0,0 +1,19 @@
+#!/usr/bin/env ansible-playbook
+# NOTE: run setup.yml with -K to prompt for sudo password
+# NOTE: the order of the roles is important: virtualbox is first b/c it will fail, yadr is next b/c it sets the shell
+# roles installed after that often depend on the shell type which is zsh in the case of yadr
+---
+- hosts: localhost
+  vars_files:
+    - vars.yml
+  roles:
+    - prepd/virtualbox
+    - prepd/yadr
+    - prepd/vagrant
+    - prepd/packer
+    - prepd/extras
+
+# TODO: put the extra role still has some legacy code; refactor that when installing on linux laptop
+# TODO: cordova needs some work
+# TODO: developer/setup role contains the list of packages to install, but is not invoked here yet
+# TODO: when the installation is 'standarized' then put all the roles into a summary role in prepd

data/files/setup/vars.yml

@@ -0,0 +1,20 @@
+---
+git_user:
+  name: John Doe Developer
+  email: email@example.com
+
+extra_items:
+  - google-chrome
+  - dropbox
+  - evernote
+  - iterm2
+  - firefox
+  - kindle
+  - postman
+  - skype
+  - slack
+  - spotify
+  - transmission
+
+extra_items_not_installing:
+  - insomniax

data/files/workspace/.gitignore

@@ -0,0 +1,12 @@
+# See https://help.github.com/articles/ignoring-files for more about ignoring files.
+
+/developer/vault-password.txt
+
+/machines/*
+!/machines/*.yml
+
+/projects/*
+!/projects/*.yml
+
+/data/*
+!/data/.keep

data/files/workspace/README.md

@@ -0,0 +1,11 @@
+# prepd
+
+## Getting Started
+
+Create a VirtualBox cluster of machines managed by Vagrant:
+
+```bash
+prepd new cluster my-cluster
+```
+
+Full documentation is available [here](https://github.com/rjayroach/prepd/tree/master/docs)

data/files/workspace/clusters/prepd.yml

@@ -0,0 +1,17 @@
+# included as prepd_machine by prepd/machine role
+---
+dirs:
+  data: '{{ ansible_env.HOME }}/data'
+  developer: '{{ ansible_env.HOME }}/developer'
+  machine: '{{ ansible_env.HOME }}/machine'
+  projects: '{{ ansible_env.HOME }}/projects'
+  share: '{{ ansible_env.HOME }}/share'
+
+dns:
+  name: "{{ hostname.stdout.split('.')[1] }}"
+  domainname: "{{ hostname.stdout.split('.')[-2:] | join('.') }}"
+
+cluster:
+  ssh: n1
+  host: "node1.{{ hostname.stdout.split('.')[-2:] | join('.') }}"
+  registry_port: 30005

data/files/workspace/clusters/provision.yml

@@ -0,0 +1,73 @@
+#!/usr/bin/env ansible-playbook
+# vagrant machine provisioner (based on host's ansible.groups values set in Vagrantfile)
+---
+- hosts: all
+  tasks:
+    - file:
+        path: '{{ ansible_env.HOME }}/{{ item }}'
+        state: absent
+      with_items: [shutdown.sh, yankring_history_v2.txt]
+
+# TODO: move terraform/terragrunt binary installation can be moved to packer.
+- hosts: kubectl:!minikube
+  # NOTE: These vars need to be accessible to the project, e.g. utils.yml -e@utils/build.yml needs the cluster_host and local_registry_port in order to tag the image to push it
+  # This should go into the machine's prepd.yml config file which also has directory locations. Anything that is a machine wide configuration goes here
+  # This would include machine's timezone, docker daemon config, perhaps PG installation and so on. The project could still configure this stuff, but doesn't have to
+  # vars:
+  #   cluster_host: node1
+  #   cluster_ssh: n1
+  #   local_registry_port: 30005
+
+  tasks:
+    - include_role:
+        name: prepd/machine
+
+    - name: Insert aliases
+      include_role:
+        name: prepd/aliases
+      vars:
+        aliases_marker: host-ssh
+        aliases_block: |
+          {{ prepd_machine.cluster.ssh }}() { ssh {{ prepd_machine.cluster.host }} $@ }
+
+    - name: Use scp to copy minikube credentials from cluster
+      include_role:
+        name: prepd/k8s/kubectl
+        tasks_from: remote
+      vars:
+        k8s_kubectl_cluster_host: '{{ prepd_machine.cluster.host }}'
+
+    - name: Initialize helm
+      command: helm init
+
+    - name: Wait for tiller to be available
+      pause:
+        seconds: 30
+
+    - name: Install local registry in local cluster
+      command: 'helm install stable/docker-registry --name local-registry --set service.type=NodePort,service.nodePort={{ prepd_machine.cluster.registry_port }}'
+
+    - name: Configure docker to allow push to insecure registry running in local cluster
+      include_role:
+        name: prepd/docker
+        tasks_from: insecure-registry
+      vars:
+        registry: '{{ prepd_machine.cluster.host }}:{{ prepd_machine.cluster.registry_port }}'
+
+- hosts: minikube
+  tasks:
+    - command: 'minikube status |grep minikube'
+      register: mks
+      ignore_errors: yes
+      changed_when: false
+
+    - set_fact:
+        minikube_running: "{{ mks.stdout_lines[0].split(' ')[-1] == 'Running' }}"
+
+    - include_role:
+        name: prepd/k8s/minikube
+        tasks_from: start
+      vars:
+        # The physical interface enp0s8 is specific to this hardware setup
+        minikube_spoof_intf: ansible_enp0s8
+      when: not minikube_running

data/files/workspace/clusters/vagrant.rb

@@ -0,0 +1,106 @@
+# Load configuration yaml
+require 'yaml'
+require 'erb'
+
+# Apply deep_merge method to Hash class
+class ::Hash
+  def deep_merge(second)
+    merger = proc { |key, v1, v2| Hash === v1 && Hash === v2 ? v1.merge(v2, &merger) : v2 }
+    self.merge(second, &merger)
+  end
+end
+
+module Prepd
+  module Vagrant
+    class Config
+      # NOTE: paths a relative to the location of the Vagrantfile from which this file is included
+      WORKSPACE_CONFIG_FILE = '../../prepd-workspace.yml'
+      BASE_CLUSTER_CONFIG_FILE = '../vagrant.yml'
+      CLUSTER_CONFIG_FILE = 'vagrant.yml'
+
+      attr_accessor :workspace, :base, :cluster
+
+      def workspace
+        @workspace ||= YAML.load(ERB.new(File.read(WORKSPACE_CONFIG_FILE)).result(binding))
+      end
+
+      def base
+        return @base if @base
+        settings = {}
+        config = YAML.load(ERB.new(File.read(BASE_CLUSTER_CONFIG_FILE)).result(binding))
+        settings = config['settings']
+        @base = YAML.load(ERB.new(File.read(BASE_CLUSTER_CONFIG_FILE)).result(binding))
+      end
+
+      def cluster
+        return @cluster if @cluster
+        settings = base['settings']
+        boxes = base['boxes']
+        config = YAML.load(ERB.new(File.read(CLUSTER_CONFIG_FILE)).result(binding))
+        config['machines'].each do |k, v|
+          config['machines'][k] = config['defaults'].deep_merge(v)
+        end
+        @cluster = config
+      end
+
+      def machines
+        cluster['machines'].keys
+      end
+
+      def machine_config(key)
+        cluster['machines'][key]
+      end
+    end
+
+    class Machine
+      attr_accessor :name, :config
+
+      def initialize(name = 'node1')
+        @name = name
+      end
+
+      def config
+        @config ||= Config.new.machine_config(name)
+      end
+
+      def autostart
+        config['vm']['autostart'] || false
+      end
+    
+      def host_name
+        "#{config['name'] || name}.#{config['domain']}"
+      end
+
+      def vm_box
+        config['vm']['box']
+      end
+
+      def vm_box_url
+        config['vm']['box_url']
+      end
+
+      def mounts
+        config['mounts'] || {}
+      end
+      
+      def port_forwards
+        config['vm']['port_forwards'] || {}
+      end
+
+      def ssh_interface
+        config['ssh']['interface']
+      end
+      
+      def ansible_groups
+        config['ansible_groups'] || {}
+      end
+    end
+  end
+end
+
+# Testing
+unless defined? PREPD_VAGRANT
+  m = Prepd::Vagrant::Machine.new('node0')
+  require 'pry'
+  binding.pry
+end