formatron 0.1.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (104) hide show
  1. checksums.yaml +7 -0
  2. data/.coveralls.yml +1 -0
  3. data/.gitignore +12 -0
  4. data/.rspec +2 -0
  5. data/.rubocop.yml +3 -0
  6. data/.simplecov +7 -0
  7. data/.travis.yml +17 -0
  8. data/CODE_OF_CONDUCT.md +13 -0
  9. data/Gemfile +6 -0
  10. data/Guardfile +16 -0
  11. data/LICENSE.txt +21 -0
  12. data/README.md +93 -0
  13. data/Rakefile +16 -0
  14. data/bin/console +14 -0
  15. data/bin/setup +7 -0
  16. data/exe/formatron +20 -0
  17. data/formatron.gemspec +52 -0
  18. data/lib/formatron.rb +357 -0
  19. data/lib/formatron/aws.rb +197 -0
  20. data/lib/formatron/chef.rb +156 -0
  21. data/lib/formatron/chef/berkshelf.rb +55 -0
  22. data/lib/formatron/chef/keys.rb +48 -0
  23. data/lib/formatron/chef/knife.rb +169 -0
  24. data/lib/formatron/chef_clients.rb +73 -0
  25. data/lib/formatron/cli.rb +33 -0
  26. data/lib/formatron/cli/completion.rb +26 -0
  27. data/lib/formatron/cli/deploy.rb +57 -0
  28. data/lib/formatron/cli/destroy.rb +57 -0
  29. data/lib/formatron/cli/generators/bootstrap.rb +250 -0
  30. data/lib/formatron/cli/generators/credentials.rb +100 -0
  31. data/lib/formatron/cli/generators/instance.rb +118 -0
  32. data/lib/formatron/cli/provision.rb +59 -0
  33. data/lib/formatron/cloud_formation.rb +54 -0
  34. data/lib/formatron/cloud_formation/resources/cloud_formation.rb +27 -0
  35. data/lib/formatron/cloud_formation/resources/ec2.rb +336 -0
  36. data/lib/formatron/cloud_formation/resources/iam.rb +94 -0
  37. data/lib/formatron/cloud_formation/resources/route53.rb +54 -0
  38. data/lib/formatron/cloud_formation/scripts.rb +128 -0
  39. data/lib/formatron/cloud_formation/template.rb +114 -0
  40. data/lib/formatron/cloud_formation/template/parameters.rb +20 -0
  41. data/lib/formatron/cloud_formation/template/vpc.rb +181 -0
  42. data/lib/formatron/cloud_formation/template/vpc/subnet.rb +187 -0
  43. data/lib/formatron/cloud_formation/template/vpc/subnet/acl.rb +147 -0
  44. data/lib/formatron/cloud_formation/template/vpc/subnet/bastion.rb +66 -0
  45. data/lib/formatron/cloud_formation/template/vpc/subnet/chef_server.rb +205 -0
  46. data/lib/formatron/cloud_formation/template/vpc/subnet/instance.rb +162 -0
  47. data/lib/formatron/cloud_formation/template/vpc/subnet/instance/policy.rb +74 -0
  48. data/lib/formatron/cloud_formation/template/vpc/subnet/instance/security_group.rb +117 -0
  49. data/lib/formatron/cloud_formation/template/vpc/subnet/instance/setup.rb +68 -0
  50. data/lib/formatron/cloud_formation/template/vpc/subnet/nat.rb +94 -0
  51. data/lib/formatron/completion.rb +26 -0
  52. data/lib/formatron/completion/completion.sh.erb +35 -0
  53. data/lib/formatron/config.rb +31 -0
  54. data/lib/formatron/config/reader.rb +29 -0
  55. data/lib/formatron/dsl.rb +15 -0
  56. data/lib/formatron/dsl/formatron.rb +25 -0
  57. data/lib/formatron/dsl/formatron/global.rb +19 -0
  58. data/lib/formatron/dsl/formatron/global/ec2.rb +17 -0
  59. data/lib/formatron/dsl/formatron/vpc.rb +17 -0
  60. data/lib/formatron/dsl/formatron/vpc/subnet.rb +27 -0
  61. data/lib/formatron/dsl/formatron/vpc/subnet/acl.rb +18 -0
  62. data/lib/formatron/dsl/formatron/vpc/subnet/chef_server.rb +32 -0
  63. data/lib/formatron/dsl/formatron/vpc/subnet/chef_server/organization.rb +22 -0
  64. data/lib/formatron/dsl/formatron/vpc/subnet/instance.rb +29 -0
  65. data/lib/formatron/dsl/formatron/vpc/subnet/instance/chef.rb +22 -0
  66. data/lib/formatron/dsl/formatron/vpc/subnet/instance/policy.rb +21 -0
  67. data/lib/formatron/dsl/formatron/vpc/subnet/instance/policy/statement.rb +23 -0
  68. data/lib/formatron/dsl/formatron/vpc/subnet/instance/security_group.rb +21 -0
  69. data/lib/formatron/dsl/formatron/vpc/subnet/instance/setup.rb +22 -0
  70. data/lib/formatron/dsl/formatron/vpc/subnet/instance/setup/variable.rb +23 -0
  71. data/lib/formatron/external.rb +61 -0
  72. data/lib/formatron/external/dsl.rb +171 -0
  73. data/lib/formatron/external/outputs.rb +25 -0
  74. data/lib/formatron/generators/bootstrap.rb +90 -0
  75. data/lib/formatron/generators/bootstrap/config.rb +62 -0
  76. data/lib/formatron/generators/bootstrap/ec2.rb +17 -0
  77. data/lib/formatron/generators/bootstrap/formatronfile.rb +52 -0
  78. data/lib/formatron/generators/bootstrap/formatronfile/Formatronfile.erb +79 -0
  79. data/lib/formatron/generators/bootstrap/ssl.rb +35 -0
  80. data/lib/formatron/generators/credentials.rb +17 -0
  81. data/lib/formatron/generators/instance.rb +64 -0
  82. data/lib/formatron/generators/instance/config.rb +47 -0
  83. data/lib/formatron/generators/instance/formatronfile.rb +47 -0
  84. data/lib/formatron/generators/instance/formatronfile/Formatronfile.erb +16 -0
  85. data/lib/formatron/generators/util.rb +14 -0
  86. data/lib/formatron/generators/util/cookbook.rb +65 -0
  87. data/lib/formatron/generators/util/gitignore.rb +16 -0
  88. data/lib/formatron/generators/util/readme.rb +18 -0
  89. data/lib/formatron/logger.rb +8 -0
  90. data/lib/formatron/s3/chef_server_cert.rb +85 -0
  91. data/lib/formatron/s3/chef_server_keys.rb +103 -0
  92. data/lib/formatron/s3/cloud_formation_template.rb +61 -0
  93. data/lib/formatron/s3/configuration.rb +58 -0
  94. data/lib/formatron/s3/path.rb +30 -0
  95. data/lib/formatron/util/dsl.rb +107 -0
  96. data/lib/formatron/util/shell.rb +20 -0
  97. data/lib/formatron/util/vpc.rb +15 -0
  98. data/lib/formatron/version.rb +4 -0
  99. data/support/cloudformation_describe_stacks_response.rb +36 -0
  100. data/support/dsl_test.rb +123 -0
  101. data/support/route53_get_hosted_zone_response.rb +21 -0
  102. data/support/s3_get_object_response.rb +21 -0
  103. data/support/template_test.rb +41 -0
  104. metadata +414 -0
@@ -0,0 +1,22 @@
1
+ require 'formatron/util/dsl'
2
+ require_relative 'setup/variable'
3
+
4
+ class Formatron
5
+ class DSL
6
+ class Formatron
7
+ class VPC
8
+ class Subnet
9
+ class Instance
10
+ # Instance setup scripts
11
+ class Setup
12
+ extend Util::DSL
13
+ dsl_initialize_block
14
+ dsl_hash :variable, 'Variable'
15
+ dsl_array :script
16
+ end
17
+ end
18
+ end
19
+ end
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,23 @@
1
+ require 'formatron/util/dsl'
2
+
3
+ class Formatron
4
+ class DSL
5
+ class Formatron
6
+ class VPC
7
+ class Subnet
8
+ class Instance
9
+ # Instance setup scripts
10
+ class Setup
11
+ # Instance setup variables
12
+ class Variable
13
+ extend Util::DSL
14
+ dsl_initialize_hash
15
+ dsl_property :value
16
+ end
17
+ end
18
+ end
19
+ end
20
+ end
21
+ end
22
+ end
23
+ end
@@ -0,0 +1,61 @@
1
+ require 'json'
2
+ require 'deep_merge'
3
+ require 'formatron/s3/configuration'
4
+ require 'formatron/dsl/formatron'
5
+ require_relative 'external/dsl'
6
+ require_relative 'external/outputs'
7
+
8
+ class Formatron
9
+ # downloads and merges config from dependencies
10
+ class External
11
+ DSL_KEY = 'dsl'
12
+ CONFIG_KEY = 'config'
13
+ OUTPUTS_KEY = 'outputs'
14
+
15
+ attr_reader(
16
+ :formatron,
17
+ :outputs
18
+ )
19
+
20
+ def initialize(aws:, target:, config:)
21
+ @aws = aws
22
+ @target = target
23
+ @config = config
24
+ @local_config = Marshal.load Marshal.dump(@config)
25
+ @formatron = Formatron::DSL::Formatron.new external: nil
26
+ @outputs = Outputs.new aws: @aws, target: @target
27
+ end
28
+
29
+ # rubocop:disable Metrics/MethodLength
30
+ def merge(bucket:, dependency:)
31
+ configuration = S3::Configuration.get(
32
+ aws: @aws,
33
+ bucket: bucket,
34
+ name: dependency,
35
+ target: @target
36
+ )
37
+ DSL.merge(
38
+ formatron: @formatron,
39
+ configuration: configuration[DSL_KEY]
40
+ )
41
+ @config.deep_merge! configuration[CONFIG_KEY]
42
+ @config.deep_merge! @local_config
43
+ @outputs.merge(
44
+ dependency: dependency,
45
+ configuration: configuration[OUTPUTS_KEY]
46
+ )
47
+ end
48
+ # rubocop:enable Metrics/MethodLength
49
+
50
+ def export(formatron:)
51
+ dsl = DSL.export formatron: @formatron
52
+ local_dsl = DSL.export formatron: formatron
53
+ dsl.deep_merge! local_dsl
54
+ {
55
+ CONFIG_KEY => @config,
56
+ DSL_KEY => dsl,
57
+ OUTPUTS_KEY => @outputs.hash
58
+ }
59
+ end
60
+ end
61
+ end
@@ -0,0 +1,171 @@
1
+ require 'deep_merge'
2
+ require 'formatron/s3/configuration'
3
+ require 'formatron/dsl/formatron'
4
+
5
+ class Formatron
6
+ class External
7
+ # merges the given configuration into a formatron object
8
+ # rubocop:disable Metrics/ModuleLength
9
+ module DSL
10
+ # rubocop:disable Metrics/MethodLength
11
+ # rubocop:disable Metrics/AbcSize
12
+ # rubocop:disable Metrics/CyclomaticComplexity
13
+ # rubocop:disable Metrics/PerceivedComplexity
14
+ def self.merge(formatron:, configuration:)
15
+ new_global = configuration['global']
16
+ formatron.global do |global|
17
+ global.protect(
18
+ new_global['protect']
19
+ ) unless new_global['protect'].nil?
20
+ global.kms_key(
21
+ new_global['kms_key']
22
+ ) unless new_global['kms_key'].nil?
23
+ global.databag_secret(
24
+ new_global['databag_secret']
25
+ ) unless new_global['databag_secret'].nil?
26
+ global.hosted_zone_id(
27
+ new_global['hosted_zone_id']
28
+ ) unless new_global['hosted_zone_id'].nil?
29
+ new_ec2 = new_global['ec2']
30
+ global.ec2 do |ec2|
31
+ ec2.key_pair(
32
+ new_ec2['key_pair']
33
+ ) unless new_ec2['key_pair'].nil?
34
+ ec2.private_key(
35
+ new_ec2['private_key']
36
+ ) unless new_ec2['private_key'].nil?
37
+ end unless new_ec2.nil?
38
+ end unless new_global.nil?
39
+ new_vpcs = configuration['vpcs']
40
+ new_vpcs.each do |vpc_key, new_vpc|
41
+ formatron.vpc vpc_key do |vpc|
42
+ vpc.guid new_vpc['guid']
43
+ vpc.cidr new_vpc['cidr']
44
+ new_subnets = new_vpc['subnets']
45
+ new_subnets.each do |subnet_key, new_subnet|
46
+ vpc.subnet subnet_key do |subnet|
47
+ subnet.guid new_subnet['guid']
48
+ subnet.availability_zone new_subnet['availability_zone']
49
+ subnet.gateway new_subnet['gateway']
50
+ new_nats = new_subnet['nats']
51
+ new_nats.each do |nat_key, new_nat|
52
+ subnet.nat nat_key do |nat|
53
+ nat.guid new_nat['guid']
54
+ end
55
+ end
56
+ new_bastions = new_subnet['bastions']
57
+ new_bastions.each do |bastion_key, new_bastion|
58
+ subnet.bastion bastion_key do |bastion|
59
+ bastion.guid new_bastion['guid']
60
+ bastion.sub_domain new_bastion['sub_domain']
61
+ end
62
+ end
63
+ new_chef_servers = new_subnet['chef_servers']
64
+ new_chef_servers.each do |chef_server_key, new_chef_server|
65
+ subnet.chef_server chef_server_key do |chef_server|
66
+ chef_server.guid new_chef_server['guid']
67
+ chef_server.username new_chef_server['username']
68
+ chef_server.ssl_verify new_chef_server['ssl_verify']
69
+ chef_server.sub_domain new_chef_server['sub_domain']
70
+ chef_server.organization do |organization|
71
+ organization.short_name(
72
+ new_chef_server['organization_short_name']
73
+ )
74
+ end
75
+ chef_server.stack new_chef_server['stack']
76
+ end
77
+ end
78
+ end
79
+ end
80
+ end
81
+ end
82
+ end
83
+ # rubocop:enable Metrics/PerceivedComplexity
84
+ # rubocop:enable Metrics/CyclomaticComplexity
85
+ # rubocop:enable Metrics/AbcSize
86
+ # rubocop:enable Metrics/MethodLength
87
+
88
+ # rubocop:disable Metrics/MethodLength
89
+ # rubocop:disable Metrics/AbcSize
90
+ # rubocop:disable Metrics/CyclomaticComplexity
91
+ # rubocop:disable Metrics/PerceivedComplexity
92
+ def self.export(formatron:)
93
+ name = formatron.name
94
+ global = formatron.global
95
+ vpcs = formatron.vpc
96
+ configuration = {
97
+ 'vpcs' => {}
98
+ }
99
+ unless global.nil?
100
+ configuration_global = configuration['global'] = {}
101
+ configuration_global['protect'] =
102
+ global.protect unless global.protect.nil?
103
+ configuration_global['kms_key'] =
104
+ global.kms_key unless global.kms_key.nil?
105
+ configuration_global['databag_secret'] =
106
+ global.databag_secret unless global.databag_secret.nil?
107
+ configuration_global['hosted_zone_id'] =
108
+ global.hosted_zone_id unless global.hosted_zone_id.nil?
109
+ ec2 = global.ec2
110
+ unless ec2.nil?
111
+ configuration_ec2 = configuration_global['ec2'] = {}
112
+ configuration_ec2['key_pair'] =
113
+ ec2.key_pair unless ec2.key_pair.nil?
114
+ configuration_ec2['private_key'] =
115
+ ec2.private_key unless ec2.private_key.nil?
116
+ end
117
+ end
118
+ vpcs.each do |vpc_key, vpc|
119
+ vpc_configuration = configuration['vpcs'][vpc_key] = {
120
+ 'subnets' => {},
121
+ 'guid' => vpc.guid,
122
+ 'cidr' => vpc.cidr
123
+ }
124
+ subnets = vpc.subnet
125
+ subnets.each do |subnet_key, subnet|
126
+ subnet_configuration =
127
+ vpc_configuration['subnets'][subnet_key] = {
128
+ 'nats' => {},
129
+ 'bastions' => {},
130
+ 'chef_servers' => {},
131
+ 'guid' => subnet.guid,
132
+ 'availability_zone' => subnet.availability_zone,
133
+ 'gateway' => subnet.gateway
134
+ }
135
+ nats = subnet.nat
136
+ bastions = subnet.bastion
137
+ chef_servers = subnet.chef_server
138
+ nats.each do |nat_key, nat|
139
+ subnet_configuration['nats'][nat_key] = {
140
+ 'guid' => nat.guid
141
+ }
142
+ end
143
+ bastions.each do |bastion_key, bastion|
144
+ subnet_configuration['bastions'][bastion_key] = {
145
+ 'guid' => bastion.guid,
146
+ 'sub_domain' => bastion.sub_domain
147
+ }
148
+ end
149
+ chef_servers.each do |chef_server_key, chef_server|
150
+ subnet_configuration['chef_servers'][chef_server_key] = {
151
+ 'guid' => chef_server.guid,
152
+ 'username' => chef_server.username,
153
+ 'ssl_verify' => chef_server.ssl_verify,
154
+ 'sub_domain' => chef_server.sub_domain,
155
+ 'organization_short_name' =>
156
+ chef_server.organization.short_name,
157
+ 'stack' => chef_server.stack || name
158
+ }
159
+ end
160
+ end
161
+ end
162
+ configuration
163
+ end
164
+ # rubocop:enable Metrics/PerceivedComplexity
165
+ # rubocop:enable Metrics/CyclomaticComplexity
166
+ # rubocop:enable Metrics/AbcSize
167
+ # rubocop:enable Metrics/MethodLength
168
+ end
169
+ # rubocop:enable Metrics/ModuleLength
170
+ end
171
+ end
@@ -0,0 +1,25 @@
1
+ require 'formatron/cloud_formation'
2
+
3
+ class Formatron
4
+ class External
5
+ # queries and merges CloudFormation outputs for external stacks
6
+ class Outputs
7
+ attr_reader :hash
8
+
9
+ def initialize(aws:, target:)
10
+ @aws = aws
11
+ @target = target
12
+ @hash = {}
13
+ end
14
+
15
+ def merge(dependency:, configuration:)
16
+ @hash.merge! configuration
17
+ @hash.merge! CloudFormation.outputs(
18
+ aws: @aws,
19
+ name: dependency,
20
+ target: @target
21
+ )
22
+ end
23
+ end
24
+ end
25
+ end
@@ -0,0 +1,90 @@
1
+ require_relative 'bootstrap/formatronfile'
2
+ require_relative 'bootstrap/config'
3
+ require_relative 'bootstrap/ssl'
4
+ require_relative 'util/cookbook'
5
+ require_relative 'util/readme'
6
+ require_relative 'util/gitignore'
7
+ require_relative 'bootstrap/ec2'
8
+
9
+ class Formatron
10
+ module Generators
11
+ # generates a bootstrap configuration
12
+ module Bootstrap
13
+ def self.validate_target_params(targets)
14
+ targets.each do |_, params|
15
+ fail 'target should have :protect parameter' if params[:protect].nil?
16
+ end
17
+ end
18
+
19
+ def self.validate_hash_params(hash, params)
20
+ params.each do |param|
21
+ fail "params should contain #{param}" if hash[param].nil?
22
+ end
23
+ end
24
+
25
+ # rubocop:disable Metrics/MethodLength
26
+ def self.validate_params(params)
27
+ validate_hash_params params, [
28
+ :name,
29
+ :s3_bucket,
30
+ :kms_key,
31
+ :ec2_key_pair,
32
+ :hosted_zone_id,
33
+ :hosted_zone_id,
34
+ :targets,
35
+ :availability_zone,
36
+ :chef_server
37
+ ]
38
+ validate_hash_params params[:chef_server], [
39
+ :organization,
40
+ :username,
41
+ :email,
42
+ :first_name,
43
+ :last_name,
44
+ :password
45
+ ]
46
+ validate_target_params params[:targets]
47
+ end
48
+ # rubocop:enable Metrics/MethodLength
49
+
50
+ def self.generate_targets(directory, targets, cookbooks_bucket_prefix)
51
+ targets.each do |target, params|
52
+ Config.write(
53
+ directory,
54
+ target,
55
+ params[:protect],
56
+ cookbooks_bucket_prefix
57
+ )
58
+ SSL.write directory, target
59
+ end
60
+ end
61
+
62
+ def self.generate_cookbooks(directory)
63
+ Util::Cookbook.write(
64
+ directory,
65
+ 'chef_server_instance',
66
+ 'Chef Server instance'
67
+ )
68
+ Util::Cookbook.write directory, 'nat_instance', 'NAT instance'
69
+ Util::Cookbook.write directory, 'bastion_instance', 'Bastion instance'
70
+ end
71
+
72
+ # rubocop:disable Metrics/MethodLength
73
+ def self.generate(directory, params)
74
+ validate_params params
75
+ Util::Readme.write directory, params[:name]
76
+ Util::Gitignore.write directory
77
+ Formatronfile.write directory, params
78
+ Config.write directory
79
+ EC2.write directory
80
+ generate_targets(
81
+ directory,
82
+ params[:targets],
83
+ params[:chef_server][:cookbooks_bucket_prefix]
84
+ )
85
+ generate_cookbooks directory
86
+ end
87
+ # rubocop:enable Metrics/MethodLength
88
+ end
89
+ end
90
+ end
@@ -0,0 +1,62 @@
1
+ class Formatron
2
+ module Generators
3
+ module Bootstrap
4
+ # generates an empty config
5
+ module Config
6
+ def self.write_default(file)
7
+ File.write file, <<-EOH.gsub(/^ {12}/, '')
8
+ {
9
+ }
10
+ EOH
11
+ end
12
+
13
+ # rubocop:disable Metrics/MethodLength
14
+ def self.write_target(file, target, protect, cookbooks_bucket_prefix)
15
+ File.write file, <<-EOH.gsub(/^ {12}/, '')
16
+ {
17
+ "protected": #{protect},
18
+ "bastion": {
19
+ "sub_domain": "bastion-#{target}"
20
+ },
21
+ "nat": {
22
+ "sub_domain": "nat-#{target}"
23
+ },
24
+ "chef_server": {
25
+ "sub_domain": "chef-#{target}",
26
+ "cookbooks_bucket": "#{cookbooks_bucket_prefix}-#{target}",
27
+ "ssl": {
28
+ "verify": true
29
+ }
30
+ }
31
+ }
32
+ EOH
33
+ end
34
+ # rubocop:enable Metrics/MethodLength
35
+
36
+ # rubocop:disable Metrics/MethodLength
37
+ def self.write(
38
+ directory,
39
+ target = nil,
40
+ protect = true,
41
+ cookbooks_bucket_prefix = nil
42
+ )
43
+ target_directory = File.join(
44
+ directory,
45
+ 'config',
46
+ target.nil? ? '_default' : target.to_s
47
+ )
48
+ FileUtils.mkdir_p target_directory
49
+ file = File.join target_directory, '_default.json'
50
+ write_default(file) if target.nil?
51
+ write_target(
52
+ file,
53
+ target,
54
+ protect,
55
+ cookbooks_bucket_prefix
56
+ ) unless target.nil?
57
+ end
58
+ # rubocop:enable Metrics/MethodLength
59
+ end
60
+ end
61
+ end
62
+ end