prepd 0.1.1 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. checksums.yaml +5 -5
  2. data/bin/console +2 -0
  3. data/files/cluster/Vagrantfile +118 -0
  4. data/files/cluster/vagrant.yml +52 -0
  5. data/files/developer/cluster/provision.yml +2 -0
  6. data/files/machine/build.json +52 -0
  7. data/files/machine/debian/stretch/iso.json +101 -0
  8. data/files/machine/debian/stretch/preseed.cfg +404 -0
  9. data/files/machine/json.rb +26 -0
  10. data/files/machine/push.json +56 -0
  11. data/files/machine/rebuild.json +60 -0
  12. data/files/project/provision.yml +20 -0
  13. data/files/project/vars.yml +5 -0
  14. data/files/setup.yml +16 -0
  15. data/files/setup/README.md +21 -0
  16. data/files/setup/ansible.cfg +4 -0
  17. data/files/setup/hosts +1 -0
  18. data/files/setup/setup.yml +19 -0
  19. data/files/setup/vars.yml +20 -0
  20. data/files/workspace/.gitignore +12 -0
  21. data/files/workspace/README.md +11 -0
  22. data/files/workspace/clusters/prepd.yml +17 -0
  23. data/files/workspace/clusters/provision.yml +73 -0
  24. data/files/workspace/clusters/vagrant.rb +106 -0
  25. data/files/workspace/clusters/vagrant.yml +18 -0
  26. data/files/workspace/data/.keep +0 -0
  27. data/files/workspace/developer/ansible.cfg +4 -0
  28. data/files/workspace/developer/credentials/.keep +0 -0
  29. data/files/workspace/developer/hosts +7 -0
  30. data/files/workspace/developer/machines/provision.yml +9 -0
  31. data/files/workspace/developer/provision.yml +15 -0
  32. data/files/workspace/machines/build.yml +34 -0
  33. data/files/workspace/machines/provision.yml +36 -0
  34. data/lib/prepd.rb +95 -34
  35. data/lib/prepd/cli.rb +27 -4
  36. data/lib/prepd/cli/commands.rb +64 -25
  37. data/lib/prepd/cli/options_parser.rb +42 -16
  38. data/lib/prepd/models.rb +7 -261
  39. data/lib/prepd/models/base.rb +124 -0
  40. data/lib/prepd/models/cluster.rb +255 -0
  41. data/lib/prepd/models/data.rb +5 -0
  42. data/lib/prepd/models/developer.rb +129 -0
  43. data/lib/prepd/models/machine.rb +146 -0
  44. data/lib/prepd/models/project.rb +94 -0
  45. data/lib/prepd/models/setup.rb +48 -0
  46. data/lib/prepd/models/workspace.rb +51 -0
  47. data/lib/prepd/version.rb +1 -1
  48. data/prepd.gemspec +4 -6
  49. metadata +47 -37
  50. data/TODO.md +0 -17
  51. data/lib/prepd/schema.rb +0 -23
@@ -0,0 +1,5 @@
1
+ module Prepd
2
+ class Data < Base
3
+ # Stuff for encyrpting data and so forth
4
+ end
5
+ end
@@ -0,0 +1,129 @@
1
+ module Prepd
2
+ class Developer < Base
3
+ WORK_DIR = 'developer'
4
+ include Prepd::Component
5
+ end
6
+ end
7
+
8
+ =begin
9
+ #
10
+ # Create AWS credential files for Terraform and Ansible, ssh keys and and ansible-vault encryption key
11
+ # NOTE: The path to credentials is used in the ansible-role prepd
12
+ #
13
+ def generate_credentials
14
+ # self.tf_creds = '/Users/rjayroach/Documents/c2p4/aws/legos-terraform.csv'
15
+ # self.ansible_creds = '/Users/rjayroach/Documents/c2p4/aws/legos-ansible.csv'
16
+ generate_tf_creds
17
+ generate_ansible_creds
18
+ generate_ssh_keys
19
+ generate_vault_password
20
+ end
21
+
22
+ def generate_tf_creds
23
+ self.tf_key, self.tf_secret = CSV.read(tf_creds).last.slice(2,2) if tf_creds
24
+ unless tf_key and tf_secret
25
+ STDOUT.puts 'tf_key and tf_secret need to be set (or set tf_creds to path to CSV file)'
26
+ return
27
+ end
28
+ require 'csv'
29
+ Dir.chdir(path) do
30
+ File.open('.terraform-vars.txt', 'w') do |f|
31
+ f.puts("aws_access_key_id = \"#{tf_key}\"")
32
+ f.puts("aws_secret_access_key = \"#{tf_secret}\"")
33
+ end
34
+ end
35
+ end
36
+
37
+ def generate_ansible_creds
38
+ self.ansible_key, self.ansible_secret = CSV.read(ansible_creds).last.slice(2,2) if ansible_creds
39
+ unless ansible_key and ansible_secret
40
+ STDOUT.puts 'ansible_key and ansible_secret need to be set (or set ansible_creds to path to CSV file)'
41
+ return
42
+ end
43
+ Dir.chdir(path) do
44
+ File.open('.boto', 'w') do |f|
45
+ f.puts('[Credentials]')
46
+ f.puts("aws_access_key_id = #{ansible_key}")
47
+ f.puts("aws_secret_access_key = #{ansible_secret}")
48
+ end
49
+ end
50
+ end
51
+
52
+ #
53
+ # Generate a key pair to be used as the EC2 key pair
54
+ #
55
+ def generate_ssh_keys(file_name = '.id_rsa')
56
+ Dir.chdir(path) { system("ssh-keygen -b 2048 -t rsa -f #{file_name} -q -N '' -C 'ansible@#{name}.#{client.name}.local'") }
57
+ end
58
+
59
+ #
60
+ # Use ansible-vault to encrypt the inventory group_vars
61
+ #
62
+ def encrypt_vault_files
63
+ Dir.chdir("#{path}/ansible") do
64
+ %w(all development local production staging).each do |env|
65
+ system("ansible-vault encrypt inventory/group_vars/#{env}/vault")
66
+ end
67
+ end
68
+ end
69
+
70
+ def encrypt(mode = :vault)
71
+ return unless executable?('gpg')
72
+ Dir.chdir(path) do
73
+ system "tar cf #{archive(:credentials)} #{file_list(mode)}"
74
+ end
75
+ system "gpg -c #{archive(:credentials)}"
76
+ FileUtils.rm(archive(:credentials))
77
+ "File created: #{archive(:credentials)}.gpg"
78
+ end
79
+
80
+ def encrypt_data
81
+ return unless executable?('gpg')
82
+ archive_path = "#{path}/#{client.name}-#{name}-data.tar"
83
+ Dir.chdir(path) do
84
+ system "tar cf #{archive_path} data"
85
+ end
86
+ system "gpg -c #{archive_path}"
87
+ FileUtils.rm(archive_path)
88
+ FileUtils.mv("#{archive_path}.gpg", "#{archive(:data)}.gpg")
89
+ "File created: #{archive(:data)}.gpg"
90
+ end
91
+
92
+ def decrypt(type = :credentials)
93
+ return unless %i(credentials data).include? type
94
+ return unless executable?('gpg')
95
+ unless File.exists?("#{archive(type)}.gpg")
96
+ STDOUT.puts "File not found: #{archive(type)}.gpg"
97
+ return
98
+ end
99
+ system "gpg #{archive(type)}.gpg"
100
+ Dir.chdir(path) do
101
+ system "tar xf #{archive(type)}"
102
+ end
103
+ FileUtils.rm(archive(type))
104
+ "File processed: #{archive(type)}.gpg"
105
+ end
106
+
107
+ def executable?(name = 'gpg')
108
+ require 'mkmf'
109
+ rv = find_executable(name)
110
+ STDOUT.puts "#{name} executable not found" unless rv
111
+ FileUtils.rm('mkmf.log')
112
+ rv
113
+ end
114
+
115
+ def file_list(mode)
116
+ return ".boto .id_rsa .id_rsa.pub .terraform-vars.txt .vault-password.txt" if mode.eql?(:all)
117
+ ".vault-password.txt"
118
+ end
119
+
120
+ def archive(type = :credentials)
121
+ "#{data_path}/#{client.name}-#{name}-#{type}.tar"
122
+ end
123
+
124
+ def data_path
125
+ "#{path}/data"
126
+ end
127
+ end
128
+ end
129
+ =end
@@ -0,0 +1,146 @@
1
+ require 'yaml'
2
+ require 'erb'
3
+ require 'json'
4
+
5
+ module Prepd
6
+ class Machine < Base
7
+ WORK_DIR = 'machines'
8
+ USER_CONFIG_FILE = 'build.yml'
9
+ VALID_BUMP_NAMES = %w(major minor patch)
10
+
11
+ include Prepd::Component
12
+
13
+ attr_accessor :bump
14
+
15
+ before_validation :set_defaults
16
+
17
+ validates :bump, inclusion: { in: VALID_BUMP_NAMES, message: "must be one of #{VALID_BUMP_NAMES.join(', ')}" }
18
+ validate :name_included_in_yaml
19
+
20
+ after_create :perform
21
+
22
+ def set_defaults
23
+ self.bump ||= Prepd.config.bump || 'patch'
24
+ end
25
+
26
+ def name_included_in_yaml
27
+ return if valid_build_names.include? name
28
+ errors.add(:invalid_name, "valid names are #{valid_build_names.join(', ')}")
29
+ end
30
+
31
+ def valid_build_names
32
+ yml['images'].keys
33
+ end
34
+
35
+ def yml
36
+ return @yml if @yml
37
+ in_component_root do
38
+ @yml = YAML.load(ERB.new(File.read("#{workspace_root}/prepd-workspace.yml")).result(binding)).merge(
39
+ YAML.load(ERB.new(File.read("#{component_root}/#{USER_CONFIG_FILE}")).result(binding)))
40
+ end
41
+ end
42
+
43
+ #
44
+ # Execute the builder
45
+ #
46
+ def perform
47
+ # return "#{build_action}\n#{build_env}" if Prepd.config.no_op
48
+ in_component_root do
49
+ # binding.pry
50
+ # TODO: remove the preseed copy when running packer command from the gem directory
51
+ FileUtils.cp("#{Prepd.files_dir}/machine/#{os_env['base_dir']}/preseed.cfg", '.')
52
+ system(build_env, "packer build #{action_file}")
53
+ FileUtils.rm('preseed.cfg')
54
+ end
55
+ end
56
+
57
+ def action_file
58
+ return "#{Prepd.files_dir}/machine/#{build_action}.json" unless build_action.eql?(:iso)
59
+ "#{Prepd.files_dir}/machine/#{os_env['base_dir']}/iso.json"
60
+ end
61
+
62
+ def build
63
+ yml['images'][name]
64
+ end
65
+
66
+ # Get references to current build, the os build and the os env
67
+ def os_build
68
+ os_build = build
69
+ loop do
70
+ break if os_build['source'].has_key?('os_image')
71
+ os_build = yml['images'][os_build['source']['image']]
72
+ end
73
+ os_build
74
+ end
75
+
76
+ def os_env
77
+ yml['os_images'][os_build['source']['os_image']]
78
+ end
79
+
80
+ #
81
+ # Derive values for image and box
82
+ #
83
+ def base_dir; os_env['base_dir'] end
84
+ # def build_image_dir; "#{yml['name']}/images/#{name}" end
85
+ def build_image_dir; "images/#{name}" end
86
+ def build_image_name; "#{os_env['base_name']}-#{name}" end
87
+ # def build_box_dir; "#{Dir.pwd}/#{yml['name']}/boxes" end
88
+ def build_box_dir; "#{Dir.pwd}/boxes" end
89
+ def box_json_file; "#{build_box_dir}/#{build_image_name}.json" end
90
+
91
+ #
92
+ # Caclulate the packer builder to run: :iso, :build, :rebuild or :push
93
+ #
94
+ def build_action
95
+ return :push if Prepd.config.push
96
+ return :rebuild if File.exists?("#{build_image_dir}/#{build_image_name}-disk1.vmdk")
97
+ build['source'].has_key?('os_image') ? :iso : :build
98
+ end
99
+
100
+ #
101
+ # Setup env vars for the builder
102
+ #
103
+ def build_env
104
+ xbuild_env = {
105
+ 'VM_BASE_NAME' => os_env['base_name'],
106
+ 'VM_INPUT' => build_action.eql?(:build) ? build['source']['image'] : name,
107
+ 'VM_OUTPUT' => name,
108
+ 'BOX_NAMESPACE' => yml['name'],
109
+ 'BOX_VERSION' => box_version,
110
+ 'PLAYBOOK_FILE' => build['provisioner'],
111
+ 'JSON_RB_FILE' => "#{Prepd.files_dir}/machine/json.rb"
112
+ }
113
+
114
+ xbuild_env.merge!({
115
+ 'ISO_CHECKSUM' => os_env['iso_checksum'],
116
+ 'ISO_URL' => os_env['iso_url']
117
+ }) if build_action.eql?(:iso)
118
+
119
+ xbuild_env.merge!({
120
+ 'AWS_PROFILE' => yml['aws']['profile'],
121
+ 'S3_BUCKET' => yml['aws']['s3_bucket'],
122
+ 'S3_REGION' => yml['aws']['s3_region'],
123
+ 'S3_BOX_DIR' => "#{yml['aws']['box_dir']}/#{yml['namesapce']}"
124
+ }) if build_action.eql?(:push)
125
+ xbuild_env
126
+ end
127
+
128
+ # Calculate the next box version
129
+ def box_version
130
+ return '0.0.1' unless File.exists?(box_json_file)
131
+ json = JSON.parse(File.read(box_json_file))
132
+ current_version = json['versions'].first['version']
133
+ return current_version if build_action.eql?(:push)
134
+ inc(current_version, type: bump)
135
+ end
136
+
137
+ def inc(version, type: 'patch')
138
+ idx = { 'major' => 0, 'minor' => 1, 'patch' => 2 }
139
+ ver = version.split('.')
140
+ ver[idx['patch']] = 0 if %w(major minor).include? type
141
+ ver[idx['minor']] = 0 if %w(major).include? type
142
+ ver[idx[type]] = ver[idx[type]].to_i + 1
143
+ ver.join('.')
144
+ end
145
+ end
146
+ end
@@ -0,0 +1,94 @@
1
+ module Prepd
2
+ class Project < Base
3
+ WORK_DIR = 'projects'
4
+ REPOSITORY_VERSION = '0.1.1'.freeze
5
+ REPOSITORY_NAME = 'prepd-project'.freeze
6
+
7
+ include Prepd::Component
8
+ # If production? then remove the .git directory in order to start with a clean repository
9
+ # If development? then clone the master branch and return
10
+ end
11
+ end
12
+
13
+ =begin
14
+ module Prepd
15
+ class Project < Base
16
+
17
+ has_many :machine_projects
18
+ has_many :machines, through: :machine_projects
19
+
20
+ after_save :write_config
21
+
22
+ after_create :set_machine, :clone_repository, :write_password_file
23
+ validates :name, presence: true, uniqueness: true # "You must supply APP_PATH" unless name
24
+
25
+ # Gets the machine that this project is created on and add it to the array and then save
26
+ # Both the project and the machine are saved which triggers an update of both machine and project YAML files
27
+ def set_machine
28
+ Machine.ref.projects << self
29
+ Machine.ref.save
30
+ end
31
+
32
+ #
33
+ # TODO: project's ansible.cfg needs a custom path to vault_password_file
34
+ # TODO: dir hierarchy for new projects:
35
+ # ~/projects/hashapp/apps/devops/app/ansible
36
+ # ~/projects/hashapp/config/vault
37
+ # ~/prepd/config/projects/hashapp/data
38
+ # ~/prepd/config/projects/hashapp/vars/setup.yml
39
+ # ~/prepd/config/projects/hashapp/vault-keys
40
+ #
41
+ # TODO: so prepd new <project_name> does this:
42
+ # mdkir ~/projects/<project_name>/app/ansible
43
+ # mdkir ~/projects/<project_name>/code
44
+ # mdkir ~/projects/<project_name>/config/vault
45
+ #
46
+ def clone_repository
47
+ Dir.mkdir(name)
48
+ Dir.chdir(name) do
49
+ Prepd.log('cloning git project') if config.no_op
50
+ system("git clone #{Prepd.git_log} git@github.com:rjayroach/#{self.class::REPOSITORY_NAME}.git .") unless config.no_op
51
+ if config.production?
52
+ Prepd.log("checking out version v#{self.class::REPOSITORY_VERSION}") if config.no_op
53
+ tag_checkout_ok = system("git checkout #{Prepd.git_log} -b v#{self.class::REPOSITORY_VERSION} tags/v#{self.class::REPOSITORY_VERSION}") unless config.no_op
54
+ fail "Could not checkout out tag v#{self.class::REPOSITORY_VERSION}" unless tag_checkout_ok or config.no_op
55
+ Prepd.log('initializing new .git repository') if config.no_op
56
+ FileUtils.rm_rf('.git') unless config.no_op
57
+ system("git init #{Prepd.git_log}") unless config.no_op
58
+ Prepd.log('adding all files to the first commit') if config.no_op
59
+ system('git add .') unless config.no_op
60
+ system("git commit #{Prepd.git_log} -m 'First commit from Prepd'") unless config.no_op
61
+ end
62
+ end
63
+ nil
64
+ end
65
+
66
+ def write_password_file
67
+ Prepd.create_password_file(config_dir)
68
+ end
69
+
70
+ # TODO: Projects have a client/project path, not just a name
71
+ def config_dir
72
+ "#{config.prepd_dir}/config/projects/#{name}"
73
+ end
74
+
75
+ # as_json with machines array included
76
+ def for_yaml
77
+ as_json
78
+ end
79
+
80
+ # If credentials=host then
81
+ # Make a directory on the host for this project's credentials
82
+ # Link to the host's config directory
83
+ # def setup_config
84
+ # if config.credentials.eql?('host')
85
+ # dir = "#{config.host_dir}/#{config.app_name}/credentials".gsub('~', Dir.home)
86
+ # FileUtils.mkdir_p(dir)
87
+ # Dir.chdir('config') do
88
+ # FileUtils.ln_s(dir, 'credentials')
89
+ # end
90
+ # end
91
+ # end
92
+ end
93
+ end
94
+ =end
@@ -0,0 +1,48 @@
1
+ module Prepd
2
+ class Setup < Base
3
+ validate :machine_is_host, :directory_cannot_exist
4
+
5
+ after_create :initialize_setup, :clone_ansible_roles
6
+
7
+ def machine_is_host
8
+ return if Prepd.config.machine_type.host?
9
+ errors.add(:machine_type, 'Setup can only run on the host machine')
10
+ end
11
+
12
+ def directory_cannot_exist
13
+ return if Prepd.config.force
14
+ errors.add(:directory_exists, requested_dir) if Dir.exists?(requested_dir)
15
+ end
16
+
17
+ def requested_dir
18
+ "#{Prepd.config_dir}/setup"
19
+ end
20
+
21
+ def initialize_setup
22
+ FileUtils.mkdir_p(requested_dir)
23
+ Dir.chdir(requested_dir) do
24
+ FileUtils.cp_r("#{Prepd.files_dir}/setup/.", '.')
25
+ end
26
+ Prepd.config.working_dir = Prepd.config_dir
27
+ Workspace.new(name: 'share').create
28
+ end
29
+
30
+ # TODO: add OS detection
31
+ # TODO: Externalize next two values to a yaml file?
32
+ ANSIBLE_ROLES_PATH = "#{Dir.home}/.ansible/roles".freeze
33
+ ANSIBLE_ROLES = {'prepd-roles' => 'prepd', 'terraplate' => 'terraplate', 'terraplate-components' => 'terraplate-components' }.freeze
34
+
35
+ #
36
+ # Clone Ansible roles
37
+ #
38
+ def clone_ansible_roles
39
+ FileUtils.mkdir_p(ANSIBLE_ROLES_PATH) unless Dir.exists? ANSIBLE_ROLES_PATH
40
+ Dir.chdir(ANSIBLE_ROLES_PATH) do
41
+ ANSIBLE_ROLES.each do |key, value|
42
+ next if Dir.exists? "#{ANSIBLE_ROLES_PATH}/#{value}"
43
+ system("git clone #{Prepd.git_log} git@github.com:rjayroach/#{key} #{value}")
44
+ end
45
+ end
46
+ end
47
+ end
48
+ end
@@ -0,0 +1,51 @@
1
+ module Prepd
2
+ class Workspace < Base
3
+ attr_accessor :name
4
+
5
+ validates :name, presence: true
6
+ validate :directory_cannot_exist
7
+
8
+ after_create :create_workspace, :initialize_workspace
9
+
10
+ def directory_cannot_exist
11
+ return if Prepd.config.force
12
+ errors.add(:directory_exists, requested_dir) if Dir.exists?(requested_dir)
13
+ end
14
+
15
+ def requested_dir
16
+ "#{Prepd.config.working_dir}/#{name}"
17
+ end
18
+
19
+ def create_workspace
20
+ Dir.chdir(Prepd.config.working_dir) do
21
+ FileUtils.rm_rf(name) if Prepd.config.force
22
+ FileUtils.mkdir_p(name)
23
+ end
24
+ end
25
+
26
+ def initialize_workspace
27
+ Dir.chdir(requested_dir) do
28
+ File.open('prepd-workspace.yml', 'w') { |f| f.write("---\nname: #{name}\n") }
29
+ Prepd.register_workspace(Dir.pwd)
30
+ FileUtils.cp_r("#{Prepd.files_dir}/workspace/.", '.')
31
+ Dir.chdir('developer') do
32
+ File.open('vars.yml', 'w') do |f|
33
+ f.puts("---\ngit_user:")
34
+ f.puts(" name: #{`git config --get user.name`.chomp}")
35
+ f.puts(" email: #{`git config --get user.email`.chomp}")
36
+ end
37
+ Prepd.write_password_file('vault-password.txt')
38
+ FileUtils.touch('vault.yml')
39
+ system('ansible-vault encrypt vault.yml')
40
+ end
41
+ # NOTE: remove after testing
42
+ # Dir.chdir('machines') do
43
+ # FileUtils.mkdir('packer_cache')
44
+ # Dir.chdir('packer_cache') do
45
+ # FileUtils.cp('/tmp/50e697ab8edda5b0ac5ba2482c07003d2ff037315c7910af66efd3c28d23ed51.iso', '.')
46
+ # end
47
+ # end
48
+ end
49
+ end
50
+ end
51
+ end