cloud-mu 3.5.0 → 3.6.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (245) hide show
  1. checksums.yaml +4 -4
  2. data/Berksfile +5 -2
  3. data/Berksfile.lock +135 -0
  4. data/ansible/roles/mu-base/README.md +33 -0
  5. data/ansible/roles/mu-base/defaults/main.yml +2 -0
  6. data/ansible/roles/mu-base/files/check_apm.cfg +1 -0
  7. data/ansible/roles/mu-base/files/check_apm.sh +18 -0
  8. data/ansible/roles/mu-base/files/check_disk.cfg +1 -0
  9. data/ansible/roles/mu-base/files/check_elastic_shards.cfg +1 -0
  10. data/ansible/roles/mu-base/files/check_elastic_shards.sh +12 -0
  11. data/ansible/roles/mu-base/files/check_logstash.cfg +1 -0
  12. data/ansible/roles/mu-base/files/check_logstash.sh +14 -0
  13. data/ansible/roles/mu-base/files/check_mem.cfg +1 -0
  14. data/ansible/roles/mu-base/files/check_updates.cfg +1 -0
  15. data/ansible/roles/mu-base/files/logrotate.conf +35 -0
  16. data/ansible/roles/mu-base/files/nrpe-apm-sudo +1 -0
  17. data/ansible/roles/mu-base/files/nrpe-elasticshards-sudo +2 -0
  18. data/ansible/roles/mu-base/handlers/main.yml +5 -0
  19. data/ansible/roles/mu-base/meta/main.yml +53 -0
  20. data/ansible/roles/mu-base/tasks/main.yml +113 -0
  21. data/ansible/roles/mu-base/templates/nrpe.cfg.j2 +231 -0
  22. data/ansible/roles/mu-base/tests/inventory +2 -0
  23. data/ansible/roles/mu-base/tests/test.yml +5 -0
  24. data/ansible/roles/mu-base/vars/main.yml +1 -0
  25. data/ansible/roles/mu-compliance/README.md +33 -0
  26. data/ansible/roles/mu-compliance/defaults/main.yml +2 -0
  27. data/ansible/roles/mu-compliance/files/U_MS_Windows_Server_2016_V2R1_STIG_SCAP_1-2_Benchmark.xml +15674 -0
  28. data/ansible/roles/mu-compliance/files/U_MS_Windows_Server_2019_V2R1_STIG_SCAP_1-2_Benchmark.xml +17553 -0
  29. data/ansible/roles/mu-compliance/handlers/main.yml +2 -0
  30. data/ansible/roles/mu-compliance/meta/main.yml +53 -0
  31. data/ansible/roles/mu-compliance/tasks/main.yml +45 -0
  32. data/ansible/roles/mu-compliance/tests/inventory +2 -0
  33. data/ansible/roles/mu-compliance/tests/test.yml +5 -0
  34. data/ansible/roles/mu-compliance/vars/main.yml +4 -0
  35. data/ansible/roles/mu-elastic/README.md +51 -0
  36. data/ansible/roles/mu-elastic/defaults/main.yml +2 -0
  37. data/ansible/roles/mu-elastic/files/jvm.options +93 -0
  38. data/ansible/roles/mu-elastic/handlers/main.yml +10 -0
  39. data/ansible/roles/mu-elastic/meta/main.yml +52 -0
  40. data/ansible/roles/mu-elastic/tasks/main.yml +186 -0
  41. data/ansible/roles/mu-elastic/templates/elasticsearch.yml.j2 +110 -0
  42. data/ansible/roles/mu-elastic/templates/kibana.yml.j2 +131 -0
  43. data/ansible/roles/mu-elastic/templates/password_set.expect.j2 +19 -0
  44. data/ansible/roles/mu-elastic/tests/inventory +2 -0
  45. data/ansible/roles/mu-elastic/tests/test.yml +5 -0
  46. data/ansible/roles/mu-elastic/vars/main.yml +2 -0
  47. data/ansible/roles/mu-logstash/README.md +51 -0
  48. data/ansible/roles/mu-logstash/defaults/main.yml +2 -0
  49. data/ansible/roles/mu-logstash/files/02-beats-input.conf +5 -0
  50. data/ansible/roles/mu-logstash/files/10-rails-filter.conf +16 -0
  51. data/ansible/roles/mu-logstash/files/jvm.options +84 -0
  52. data/ansible/roles/mu-logstash/files/logstash.yml +304 -0
  53. data/ansible/roles/mu-logstash/handlers/main.yml +20 -0
  54. data/ansible/roles/mu-logstash/meta/main.yml +52 -0
  55. data/ansible/roles/mu-logstash/tasks/main.yml +254 -0
  56. data/ansible/roles/mu-logstash/templates/20-cloudtrail.conf.j2 +28 -0
  57. data/ansible/roles/mu-logstash/templates/30-elasticsearch-output.conf.j2 +19 -0
  58. data/ansible/roles/mu-logstash/templates/apm-server.yml.j2 +33 -0
  59. data/ansible/roles/mu-logstash/templates/heartbeat.yml.j2 +29 -0
  60. data/ansible/roles/mu-logstash/templates/nginx/apm.conf.j2 +25 -0
  61. data/ansible/roles/mu-logstash/templates/nginx/default.conf.j2 +56 -0
  62. data/ansible/roles/mu-logstash/templates/nginx/elastic.conf.j2 +27 -0
  63. data/ansible/roles/mu-logstash/tests/inventory +2 -0
  64. data/ansible/roles/mu-logstash/tests/test.yml +5 -0
  65. data/ansible/roles/mu-logstash/vars/main.yml +2 -0
  66. data/ansible/roles/mu-rdp/README.md +33 -0
  67. data/ansible/roles/mu-rdp/meta/main.yml +53 -0
  68. data/ansible/roles/mu-rdp/tasks/main.yml +9 -0
  69. data/ansible/roles/mu-rdp/tests/inventory +2 -0
  70. data/ansible/roles/mu-rdp/tests/test.yml +5 -0
  71. data/ansible/roles/mu-windows/tasks/main.yml +3 -0
  72. data/bin/mu-ansible-secret +1 -1
  73. data/bin/mu-aws-setup +4 -3
  74. data/bin/mu-azure-setup +5 -5
  75. data/bin/mu-configure +25 -17
  76. data/bin/mu-firewall-allow-clients +1 -0
  77. data/bin/mu-gcp-setup +3 -3
  78. data/bin/mu-load-config.rb +1 -0
  79. data/bin/mu-node-manage +66 -33
  80. data/bin/mu-self-update +2 -2
  81. data/bin/mu-upload-chef-artifacts +6 -1
  82. data/bin/mu-user-manage +1 -1
  83. data/cloud-mu.gemspec +25 -23
  84. data/cookbooks/firewall/CHANGELOG.md +417 -224
  85. data/cookbooks/firewall/LICENSE +202 -0
  86. data/cookbooks/firewall/README.md +153 -126
  87. data/cookbooks/firewall/TODO.md +6 -0
  88. data/cookbooks/firewall/attributes/firewalld.rb +7 -0
  89. data/cookbooks/firewall/attributes/iptables.rb +3 -3
  90. data/cookbooks/firewall/chefignore +115 -0
  91. data/cookbooks/firewall/libraries/helpers.rb +5 -0
  92. data/cookbooks/firewall/libraries/helpers_firewalld.rb +1 -1
  93. data/cookbooks/firewall/libraries/helpers_firewalld_dbus.rb +72 -0
  94. data/cookbooks/firewall/libraries/helpers_iptables.rb +3 -3
  95. data/cookbooks/firewall/libraries/helpers_nftables.rb +170 -0
  96. data/cookbooks/firewall/libraries/helpers_ufw.rb +7 -0
  97. data/cookbooks/firewall/libraries/helpers_windows.rb +8 -9
  98. data/cookbooks/firewall/libraries/provider_firewall_firewalld.rb +9 -9
  99. data/cookbooks/firewall/libraries/provider_firewall_iptables.rb +7 -7
  100. data/cookbooks/firewall/libraries/provider_firewall_iptables_ubuntu.rb +12 -8
  101. data/cookbooks/firewall/libraries/provider_firewall_iptables_ubuntu1404.rb +13 -9
  102. data/cookbooks/firewall/libraries/provider_firewall_rule.rb +1 -1
  103. data/cookbooks/firewall/libraries/provider_firewall_ufw.rb +5 -5
  104. data/cookbooks/firewall/libraries/provider_firewall_windows.rb +4 -4
  105. data/cookbooks/firewall/libraries/resource_firewall_rule.rb +3 -3
  106. data/cookbooks/firewall/metadata.json +40 -1
  107. data/cookbooks/firewall/metadata.rb +15 -0
  108. data/cookbooks/firewall/recipes/default.rb +7 -7
  109. data/cookbooks/firewall/recipes/disable_firewall.rb +1 -1
  110. data/cookbooks/firewall/recipes/firewalld.rb +87 -0
  111. data/cookbooks/firewall/renovate.json +18 -0
  112. data/cookbooks/firewall/resources/firewalld.rb +28 -0
  113. data/cookbooks/firewall/resources/firewalld_config.rb +39 -0
  114. data/cookbooks/firewall/resources/firewalld_helpers.rb +106 -0
  115. data/cookbooks/firewall/resources/firewalld_icmptype.rb +88 -0
  116. data/cookbooks/firewall/resources/firewalld_ipset.rb +104 -0
  117. data/cookbooks/firewall/resources/firewalld_policy.rb +115 -0
  118. data/cookbooks/firewall/resources/firewalld_service.rb +98 -0
  119. data/cookbooks/firewall/resources/firewalld_zone.rb +118 -0
  120. data/cookbooks/firewall/resources/nftables.rb +71 -0
  121. data/cookbooks/firewall/resources/nftables_rule.rb +113 -0
  122. data/cookbooks/mu-activedirectory/Berksfile +1 -1
  123. data/cookbooks/mu-activedirectory/metadata.rb +1 -1
  124. data/cookbooks/mu-firewall/metadata.rb +2 -2
  125. data/cookbooks/mu-master/Berksfile +4 -3
  126. data/cookbooks/mu-master/attributes/default.rb +5 -2
  127. data/cookbooks/mu-master/files/default/check_elastic.sh +761 -0
  128. data/cookbooks/mu-master/files/default/check_kibana.rb +45 -0
  129. data/cookbooks/mu-master/libraries/mu.rb +24 -0
  130. data/cookbooks/mu-master/metadata.rb +5 -5
  131. data/cookbooks/mu-master/recipes/default.rb +31 -20
  132. data/cookbooks/mu-master/recipes/firewall-holes.rb +5 -0
  133. data/cookbooks/mu-master/recipes/init.rb +58 -19
  134. data/cookbooks/mu-master/recipes/update_nagios_only.rb +251 -178
  135. data/cookbooks/mu-master/templates/default/nagios.conf.erb +5 -11
  136. data/cookbooks/mu-master/templates/default/web_app.conf.erb +3 -0
  137. data/cookbooks/mu-php54/Berksfile +1 -1
  138. data/cookbooks/mu-php54/metadata.rb +2 -2
  139. data/cookbooks/mu-tools/Berksfile +2 -3
  140. data/cookbooks/mu-tools/attributes/default.rb +3 -4
  141. data/cookbooks/mu-tools/files/amazon/etc/bashrc +90 -0
  142. data/cookbooks/mu-tools/files/amazon/etc/login.defs +292 -0
  143. data/cookbooks/mu-tools/files/amazon/etc/profile +77 -0
  144. data/cookbooks/mu-tools/files/amazon/etc/security/limits.conf +63 -0
  145. data/cookbooks/mu-tools/files/amazon/etc/sysconfig/init +19 -0
  146. data/cookbooks/mu-tools/files/amazon/etc/sysctl.conf +82 -0
  147. data/cookbooks/mu-tools/files/amazon-2023/etc/login.defs +294 -0
  148. data/cookbooks/mu-tools/files/default/logrotate.conf +35 -0
  149. data/cookbooks/mu-tools/files/default/nrpe_conf_d.pp +0 -0
  150. data/cookbooks/mu-tools/libraries/helper.rb +21 -9
  151. data/cookbooks/mu-tools/metadata.rb +4 -4
  152. data/cookbooks/mu-tools/recipes/apply_security.rb +3 -2
  153. data/cookbooks/mu-tools/recipes/aws_api.rb +23 -5
  154. data/cookbooks/mu-tools/recipes/base_repositories.rb +4 -1
  155. data/cookbooks/mu-tools/recipes/gcloud.rb +56 -56
  156. data/cookbooks/mu-tools/recipes/nagios.rb +1 -1
  157. data/cookbooks/mu-tools/recipes/nrpe.rb +20 -2
  158. data/cookbooks/mu-tools/recipes/rsyslog.rb +12 -1
  159. data/cookbooks/mu-tools/recipes/set_local_fw.rb +1 -1
  160. data/data_bags/nagios_services/apm_backend_connect.json +5 -0
  161. data/data_bags/nagios_services/apm_listen.json +5 -0
  162. data/data_bags/nagios_services/elastic_shards.json +5 -0
  163. data/data_bags/nagios_services/logstash.json +5 -0
  164. data/data_bags/nagios_services/rhel7_updates.json +8 -0
  165. data/extras/image-generators/AWS/centos7.yaml +1 -0
  166. data/extras/image-generators/AWS/rhel7.yaml +21 -0
  167. data/extras/image-generators/AWS/win2k12r2.yaml +1 -0
  168. data/extras/image-generators/AWS/win2k16.yaml +1 -0
  169. data/extras/image-generators/AWS/win2k19.yaml +1 -0
  170. data/extras/list-stock-amis +0 -0
  171. data/extras/ruby_rpm/muby.spec +8 -5
  172. data/extras/vault_tools/export_vaults.sh +1 -1
  173. data/extras/vault_tools/recreate_vaults.sh +0 -0
  174. data/extras/vault_tools/test_vaults.sh +0 -0
  175. data/install/deprecated-bash-library.sh +1 -1
  176. data/install/installer +4 -2
  177. data/modules/mommacat.ru +3 -1
  178. data/modules/mu/adoption.rb +1 -1
  179. data/modules/mu/cloud/dnszone.rb +2 -2
  180. data/modules/mu/cloud/machine_images.rb +26 -25
  181. data/modules/mu/cloud/resource_base.rb +213 -182
  182. data/modules/mu/cloud/server_pool.rb +1 -1
  183. data/modules/mu/cloud/ssh_sessions.rb +7 -5
  184. data/modules/mu/cloud/wrappers.rb +2 -2
  185. data/modules/mu/cloud.rb +1 -1
  186. data/modules/mu/config/bucket.rb +1 -1
  187. data/modules/mu/config/function.rb +6 -1
  188. data/modules/mu/config/loadbalancer.rb +24 -2
  189. data/modules/mu/config/ref.rb +12 -0
  190. data/modules/mu/config/role.rb +1 -1
  191. data/modules/mu/config/schema_helpers.rb +42 -9
  192. data/modules/mu/config/server.rb +43 -27
  193. data/modules/mu/config/tail.rb +19 -10
  194. data/modules/mu/config.rb +6 -5
  195. data/modules/mu/defaults/AWS.yaml +78 -114
  196. data/modules/mu/deploy.rb +9 -2
  197. data/modules/mu/groomer.rb +12 -4
  198. data/modules/mu/groomers/ansible.rb +104 -20
  199. data/modules/mu/groomers/chef.rb +15 -6
  200. data/modules/mu/master.rb +9 -4
  201. data/modules/mu/mommacat/daemon.rb +4 -2
  202. data/modules/mu/mommacat/naming.rb +1 -2
  203. data/modules/mu/mommacat/storage.rb +7 -2
  204. data/modules/mu/mommacat.rb +33 -6
  205. data/modules/mu/providers/aws/database.rb +161 -8
  206. data/modules/mu/providers/aws/dnszone.rb +11 -6
  207. data/modules/mu/providers/aws/endpoint.rb +81 -6
  208. data/modules/mu/providers/aws/firewall_rule.rb +254 -172
  209. data/modules/mu/providers/aws/function.rb +65 -3
  210. data/modules/mu/providers/aws/loadbalancer.rb +39 -28
  211. data/modules/mu/providers/aws/log.rb +2 -1
  212. data/modules/mu/providers/aws/role.rb +25 -7
  213. data/modules/mu/providers/aws/server.rb +36 -12
  214. data/modules/mu/providers/aws/server_pool.rb +237 -127
  215. data/modules/mu/providers/aws/storage_pool.rb +7 -1
  216. data/modules/mu/providers/aws/user.rb +1 -1
  217. data/modules/mu/providers/aws/userdata/linux.erb +6 -2
  218. data/modules/mu/providers/aws/userdata/windows.erb +7 -5
  219. data/modules/mu/providers/aws/vpc.rb +49 -25
  220. data/modules/mu/providers/aws.rb +13 -8
  221. data/modules/mu/providers/azure/container_cluster.rb +1 -1
  222. data/modules/mu/providers/azure/loadbalancer.rb +2 -2
  223. data/modules/mu/providers/azure/server.rb +5 -2
  224. data/modules/mu/providers/azure/userdata/linux.erb +1 -1
  225. data/modules/mu/providers/azure.rb +11 -8
  226. data/modules/mu/providers/cloudformation/dnszone.rb +1 -1
  227. data/modules/mu/providers/google/container_cluster.rb +15 -2
  228. data/modules/mu/providers/google/folder.rb +2 -1
  229. data/modules/mu/providers/google/function.rb +130 -4
  230. data/modules/mu/providers/google/habitat.rb +2 -1
  231. data/modules/mu/providers/google/loadbalancer.rb +407 -160
  232. data/modules/mu/providers/google/role.rb +16 -3
  233. data/modules/mu/providers/google/server.rb +5 -1
  234. data/modules/mu/providers/google/user.rb +25 -18
  235. data/modules/mu/providers/google/userdata/linux.erb +1 -1
  236. data/modules/mu/providers/google/vpc.rb +53 -7
  237. data/modules/mu/providers/google.rb +39 -39
  238. data/modules/mu.rb +8 -8
  239. data/modules/tests/elk.yaml +46 -0
  240. data/test/mu-master-test/controls/all_in_one.rb +1 -1
  241. metadata +207 -112
  242. data/cookbooks/firewall/CONTRIBUTING.md +0 -2
  243. data/cookbooks/firewall/MAINTAINERS.md +0 -19
  244. data/cookbooks/firewall/libraries/matchers.rb +0 -30
  245. data/extras/image-generators/AWS/rhel71.yaml +0 -17
@@ -0,0 +1,131 @@
1
+ # Kibana is served by a back end server. This setting specifies the port to use.
2
+ #server.port: 5601
3
+
4
+ # Specifies the address to which the Kibana server will bind. IP addresses and host names are both valid values.
5
+ # The default is 'localhost', which usually means remote machines will not be able to connect.
6
+ # To allow connections from remote users, set this parameter to a non-loopback address.
7
+ server.host: "0.0.0.0"
8
+
9
+ # Enables you to specify a path to mount Kibana at if you are running behind a proxy.
10
+ # Use the `server.rewriteBasePath` setting to tell Kibana if it should remove the basePath
11
+ # from requests it receives, and to prevent a deprecation warning at startup.
12
+ # This setting cannot end in a slash.
13
+ #server.basePath: ""
14
+
15
+ # Specifies whether Kibana should rewrite requests that are prefixed with
16
+ # `server.basePath` or require that they are rewritten by your reverse proxy.
17
+ # This setting was effectively always `false` before Kibana 6.3 and will
18
+ # default to `true` starting in Kibana 7.0.
19
+ #server.rewriteBasePath: false
20
+
21
+ # Specifies the public URL at which Kibana is available for end users. If
22
+ # `server.basePath` is configured this URL should end with the same basePath.
23
+ #server.publicBaseUrl: ""
24
+
25
+ # The maximum payload size in bytes for incoming server requests.
26
+ #server.maxPayloadBytes: 1048576
27
+
28
+ # The Kibana server's name. This is used for display purposes.
29
+ #server.name: "your-hostname"
30
+
31
+ # The URLs of the Elasticsearch instances to use for all your queries.
32
+ #elasticsearch.hosts: ["https://node.elasticsearch.com:9200"]
33
+ #elasticsearch.hosts: ["http://localhost:9200"]
34
+ # Kibana uses an index in Elasticsearch to store saved searches, visualizations and
35
+ # dashboards. Kibana creates a new index if the index doesn't already exist.
36
+ #kibana.index: ".kibana"
37
+
38
+ # The default application to load.
39
+ #kibana.defaultAppId: "home"
40
+
41
+ # If your Elasticsearch is protected with basic authentication, these settings provide
42
+ # the username and password that the Kibana server uses to perform maintenance on the Kibana
43
+ # index at startup. Your Kibana users still need to authenticate with Elasticsearch, which
44
+ # is proxied through the Kibana server.
45
+ #elasticsearch.username: "kibana_system"
46
+ #elasticsearch.password: "123456"
47
+
48
+ # Enables SSL and paths to the PEM-format SSL certificate and SSL key files, respectively.
49
+ # These settings enable SSL for outgoing requests from the Kibana server to the browser.
50
+ #server.ssl.enabled: false
51
+ #server.ssl.certificate: /path/to/your/server.crt
52
+ #server.ssl.key: /path/to/your/server.key
53
+
54
+ # Optional settings that provide the paths to the PEM-format SSL certificate and key files.
55
+ # These files are used to verify the identity of Kibana to Elasticsearch and are required when
56
+ # xpack.security.http.ssl.client_authentication in Elasticsearch is set to required.
57
+ #elasticsearch.ssl.certificate: /path/to/your/client.crt
58
+ #elasticsearch.ssl.key: /path/to/your/client.key
59
+
60
+ # Optional setting that enables you to specify a path to the PEM file for the certificate
61
+ # authority for your Elasticsearch instance.
62
+ #elasticsearch.ssl.certificateAuthorities: [ "/path/to/your/CA.pem" ]
63
+
64
+ # To disregard the validity of SSL certificates, change this setting's value to 'none'.
65
+ #elasticsearch.ssl.verificationMode: full
66
+
67
+ # Time in milliseconds to wait for Elasticsearch to respond to pings. Defaults to the value of
68
+ # the elasticsearch.requestTimeout setting.
69
+ #elasticsearch.pingTimeout: 1500
70
+
71
+ # Time in milliseconds to wait for responses from the back end or Elasticsearch. This value
72
+ # must be a positive integer.
73
+ #elasticsearch.requestTimeout: 30000
74
+
75
+ # List of Kibana client-side headers to send to Elasticsearch. To send *no* client-side
76
+ # headers, set this value to [] (an empty list).
77
+ #elasticsearch.requestHeadersWhitelist: [ authorization ]
78
+
79
+ # Header names and values that are sent to Elasticsearch. Any custom headers cannot be overwritten
80
+ # by client-side headers, regardless of the elasticsearch.requestHeadersWhitelist configuration.
81
+ #elasticsearch.customHeaders: {}
82
+
83
+ # Time in milliseconds for Elasticsearch to wait for responses from shards. Set to 0 to disable.
84
+ #elasticsearch.shardTimeout: 30000
85
+
86
+ # Logs queries sent to Elasticsearch. Requires logging.verbose set to true.
87
+ #elasticsearch.logQueries: false
88
+
89
+ # Specifies the path where Kibana creates the process ID file.
90
+ #pid.file: /run/kibana/kibana.pid
91
+
92
+ # Enables you to specify a file where Kibana stores log output.
93
+ #logging.dest: stdout
94
+
95
+ # Set the value of this setting to true to suppress all logging output.
96
+ #logging.silent: false
97
+
98
+ # Set the value of this setting to true to suppress all logging output other than error messages.
99
+ #logging.quiet: false
100
+
101
+ # Set the value of this setting to true to log all events, including system usage information
102
+ # and all requests.
103
+ #logging.verbose: false
104
+
105
+ # Set the interval in milliseconds to sample system and process performance
106
+ # metrics. Minimum is 100ms. Defaults to 5000.
107
+ #ops.interval: 5000
108
+
109
+ # Specifies locale to be used for all localizable strings, dates and number formats.
110
+ # Supported languages are the following: English - en , by default , Chinese - zh-CN .
111
+ #i18n.locale: "en"
112
+
113
+ xpack.encryptedSavedObjects.encryptionKey: 'Q!:V9p#,N\6AHE-dt!!{S,j-@GX@DADpsG'
114
+ #xpack.security.session.idleTimeout: "1h"
115
+ #xpack.security.session.lifespan: "30d"
116
+ #elasticsearch.ssl.certificateAuthorities: ["/etc/kibana/config/elasticsearch-ca.pem"]
117
+ ##elasticsearch.ssl.verificationMode: "none"
118
+
119
+ elasticsearch.hosts: {% for node,meta in mu_deployment['servers']['backend'].items() %}
120
+ {%- for k,v in meta.items() %}
121
+ {%- if k in ["private_ip_address"] %}
122
+ ["https://{{ v }}:9200"]
123
+ {%- endif %}
124
+ {%- endfor %}
125
+ {%- if not loop.last %},{%- endif %}
126
+ {%- endfor %}
127
+
128
+ elasticsearch.username: "kibana_system"
129
+ elasticsearch.password: "{{ elasticpw }}"
130
+ elasticsearch.ssl.certificateAuthorities: [ "/etc/kibana/elasticsearch-ca.pem" ]
131
+ elasticsearch.ssl.verificationMode: none
@@ -0,0 +1,19 @@
1
+ #!/usr/bin/expect
2
+
3
+ set pw {% raw %}{{% endraw %}{{ elasticpw }}{% raw %}}{% endraw %}
4
+
5
+ spawn /usr/share/elasticsearch/bin/elasticsearch-setup-passwords interactive
6
+ expect "y/N" {send "y\r"}
7
+ expect "Enter password for \\\[elastic\\\]" {send "$pw\r"}
8
+ expect "Reenter password for \\\[elastic\\\]" {send "$pw\r"}
9
+ expect "Enter password for \\\[apm_system\\\]" {send "$pw\r"}
10
+ expect "Reenter password for \\\[apm_system\\\]" {send "$pw\r"}
11
+ expect "Enter password for \\\[kibana_system\\\]" {send "$pw\r"}
12
+ expect "Reenter password for \\\[kibana_system\\\]" {send "$pw\r"}
13
+ expect "Enter password for \\\[logstash_system\\\]" {send "$pw\r"}
14
+ expect "Reenter password for \\\[logstash_system\\\]" {send "$pw\r"}
15
+ expect "Enter password for \\\[beats_system\\\]" {send "$pw\r"}
16
+ expect "Reenter password for \\\[beats_system\\\]" {send "$pw\r"}
17
+ expect "Enter password for \\\[remote_monitoring_user\\\]" {send "$pw\r"}
18
+ expect "Reenter password for \\\[remote_monitoring_user\\\]" {send "$pw\r"}
19
+ interact
@@ -0,0 +1,2 @@
1
+ localhost
2
+
@@ -0,0 +1,5 @@
1
+ ---
2
+ - hosts: localhost
3
+ remote_user: root
4
+ roles:
5
+ - mu-elastic
@@ -0,0 +1,2 @@
1
+ ---
2
+ # vars file for mu-elastic
@@ -0,0 +1,51 @@
1
+ Role Name
2
+ =========
3
+
4
+ Set up a log auditing service using Logstash and APM. Depends on having Elastic and Kibana somewhere.
5
+
6
+ Requirements
7
+ ------------
8
+
9
+ Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required.
10
+
11
+ Role Variables
12
+ --------------
13
+
14
+ A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well.
15
+
16
+ Dependencies
17
+ ------------
18
+
19
+ A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles.
20
+
21
+ Example Playbook
22
+ ----------------
23
+
24
+ Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too:
25
+
26
+ - hosts: servers
27
+ roles:
28
+ - { role: username.rolename, x: 42 }
29
+
30
+ License
31
+ -------
32
+
33
+ Copyright:: Copyright (c) 2020 eGlobalTech, Inc., all rights reserved
34
+
35
+ Licensed under the BSD-3 license (the "License");
36
+ you may not use this file except in compliance with the License.
37
+ You may obtain a copy of the License in the root of the project or at
38
+
39
+ http://egt-labs.com/mu/LICENSE.html
40
+
41
+ Unless required by applicable law or agreed to in writing, software
42
+ distributed under the License is distributed on an "AS IS" BASIS,
43
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
44
+ See the License for the specific language governing permissions and
45
+ limitations under the License.
46
+
47
+
48
+ Author Information
49
+ ------------------
50
+
51
+ An optional section for the role authors to include contact information, or a website (HTML is not allowed).
@@ -0,0 +1,2 @@
1
+ ---
2
+ # defaults file for mu-logstash
@@ -0,0 +1,5 @@
1
+ input {
2
+ beats {
3
+ port => 5044
4
+ }
5
+ }
@@ -0,0 +1,16 @@
1
+ filter {
2
+ if [application] == "ruby" {
3
+ json {
4
+ source => "message"
5
+ }
6
+
7
+ #useragent {
8
+ # source => "browser"
9
+ #}
10
+
11
+ geoip {
12
+ source => "remote_ip"
13
+ target => "geoip"
14
+ }
15
+ }
16
+ }
@@ -0,0 +1,84 @@
1
+ ## JVM configuration
2
+
3
+ # Xms represents the initial size of total heap space
4
+ # Xmx represents the maximum size of total heap space
5
+
6
+ -Xms1g
7
+ -Xmx1g
8
+
9
+ ################################################################
10
+ ## Expert settings
11
+ ################################################################
12
+ ##
13
+ ## All settings below this section are considered
14
+ ## expert settings. Don't tamper with them unless
15
+ ## you understand what you are doing
16
+ ##
17
+ ################################################################
18
+
19
+ ## GC configuration
20
+ -XX:+UseConcMarkSweepGC
21
+ -XX:CMSInitiatingOccupancyFraction=75
22
+ -XX:+UseCMSInitiatingOccupancyOnly
23
+
24
+ ## Locale
25
+ # Set the locale language
26
+ #-Duser.language=en
27
+
28
+ # Set the locale country
29
+ #-Duser.country=US
30
+
31
+ # Set the locale variant, if any
32
+ #-Duser.variant=
33
+
34
+ ## basic
35
+
36
+ # set the I/O temp directory
37
+ #-Djava.io.tmpdir=$HOME
38
+
39
+ # set to headless, just in case
40
+ -Djava.awt.headless=true
41
+
42
+ # ensure UTF-8 encoding by default (e.g. filenames)
43
+ -Dfile.encoding=UTF-8
44
+
45
+ # use our provided JNA always versus the system one
46
+ #-Djna.nosys=true
47
+
48
+ # Turn on JRuby invokedynamic
49
+ -Djruby.compile.invokedynamic=true
50
+ # Force Compilation
51
+ -Djruby.jit.threshold=0
52
+ # Make sure joni regexp interruptability is enabled
53
+ -Djruby.regexp.interruptible=true
54
+
55
+ ## heap dumps
56
+
57
+ # generate a heap dump when an allocation from the Java heap fails
58
+ # heap dumps are created in the working directory of the JVM
59
+ -XX:+HeapDumpOnOutOfMemoryError
60
+
61
+ # specify an alternative path for heap dumps
62
+ # ensure the directory exists and has sufficient space
63
+ #-XX:HeapDumpPath=${LOGSTASH_HOME}/heapdump.hprof
64
+
65
+ ## GC logging
66
+ #-XX:+PrintGCDetails
67
+ #-XX:+PrintGCTimeStamps
68
+ #-XX:+PrintGCDateStamps
69
+ #-XX:+PrintClassHistogram
70
+ #-XX:+PrintTenuringDistribution
71
+ #-XX:+PrintGCApplicationStoppedTime
72
+
73
+ # log GC status to a file with time stamps
74
+ # ensure the directory exists
75
+ #-Xloggc:${LS_GC_LOG_FILE}
76
+
77
+ # Entropy source for randomness
78
+ -Djava.security.egd=file:/dev/urandom
79
+
80
+ # Copy the logging context from parent threads to children
81
+ -Dlog4j2.isThreadContextMapInheritable=true
82
+
83
+ # CVE-2021-44228
84
+ -Dlog4j2.formatMsgNoLookups=true
@@ -0,0 +1,304 @@
1
+ # Settings file in YAML
2
+ #
3
+ # Settings can be specified either in hierarchical form, e.g.:
4
+ #
5
+ # pipeline:
6
+ # batch:
7
+ # size: 125
8
+ # delay: 5
9
+ #
10
+ # Or as flat keys:
11
+ #
12
+ # pipeline.batch.size: 125
13
+ # pipeline.batch.delay: 5
14
+ #
15
+ # ------------ Node identity ------------
16
+ #
17
+ # Use a descriptive name for the node:
18
+ #
19
+ # node.name: test
20
+ #
21
+ # If omitted the node name will default to the machine's host name
22
+ #
23
+ # ------------ Data path ------------------
24
+ #
25
+ # Which directory should be used by logstash and its plugins
26
+ # for any persistent needs. Defaults to LOGSTASH_HOME/data
27
+ #
28
+ path.data: /var/lib/logstash
29
+ #
30
+ # ------------ Pipeline Settings --------------
31
+ #
32
+ # The ID of the pipeline.
33
+ #
34
+ # pipeline.id: main
35
+ #
36
+ # Set the number of workers that will, in parallel, execute the filters+outputs
37
+ # stage of the pipeline.
38
+ #
39
+ # This defaults to the number of the host's CPU cores.
40
+ #
41
+ # pipeline.workers: 2
42
+ #
43
+ # How many events to retrieve from inputs before sending to filters+workers
44
+ #
45
+ # pipeline.batch.size: 125
46
+ #
47
+ # How long to wait in milliseconds while polling for the next event
48
+ # before dispatching an undersized batch to filters+outputs
49
+ #
50
+ # pipeline.batch.delay: 50
51
+ #
52
+ # Force Logstash to exit during shutdown even if there are still inflight
53
+ # events in memory. By default, logstash will refuse to quit until all
54
+ # received events have been pushed to the outputs.
55
+ #
56
+ # WARNING: enabling this can lead to data loss during shutdown
57
+ #
58
+ # pipeline.unsafe_shutdown: false
59
+ #
60
+ # Set the pipeline event ordering. Options are "auto" (the default), "true" or "false".
61
+ # "auto" will automatically enable ordering if the 'pipeline.workers' setting
62
+ # is also set to '1'.
63
+ # "true" will enforce ordering on the pipeline and prevent logstash from starting
64
+ # if there are multiple workers.
65
+ # "false" will disable any extra processing necessary for preserving ordering.
66
+ #
67
+ pipeline.ordered: auto
68
+ #
69
+ # ------------ Pipeline Configuration Settings --------------
70
+ #
71
+ # Where to fetch the pipeline configuration for the main pipeline
72
+ #
73
+ # path.config:
74
+ #
75
+ # Pipeline configuration string for the main pipeline
76
+ #
77
+ # config.string:
78
+ #
79
+ # At startup, test if the configuration is valid and exit (dry run)
80
+ #
81
+ # config.test_and_exit: false
82
+ #
83
+ # Periodically check if the configuration has changed and reload the pipeline
84
+ # This can also be triggered manually through the SIGHUP signal
85
+ #
86
+ # config.reload.automatic: false
87
+ #
88
+ # How often to check if the pipeline configuration has changed (in seconds)
89
+ # Note that the unit value (s) is required. Values without a qualifier (e.g. 60)
90
+ # are treated as nanoseconds.
91
+ # Setting the interval this way is not recommended and might change in later versions.
92
+ #
93
+ # config.reload.interval: 3s
94
+ #
95
+ # Show fully compiled configuration as debug log message
96
+ # NOTE: --log.level must be 'debug'
97
+ #
98
+ #config.debug: true
99
+ #
100
+ # When enabled, process escaped characters such as \n and \" in strings in the
101
+ # pipeline configuration files.
102
+ #
103
+ # config.support_escapes: false
104
+ #
105
+ # ------------ HTTP API Settings -------------
106
+ # Define settings related to the HTTP API here.
107
+ #
108
+ # The HTTP API is enabled by default. It can be disabled, but features that rely
109
+ # on it will not work as intended.
110
+ # http.enabled: true
111
+ #
112
+ # By default, the HTTP API is bound to only the host's local loopback interface,
113
+ # ensuring that it is not accessible to the rest of the network. Because the API
114
+ # includes neither authentication nor authorization and has not been hardened or
115
+ # tested for use as a publicly-reachable API, binding to publicly accessible IPs
116
+ # should be avoided where possible.
117
+ #
118
+ # http.host: 0
119
+ #
120
+ # The HTTP API web server will listen on an available port from the given range.
121
+ # Values can be specified as a single port (e.g., `9600`), or an inclusive range
122
+ # of ports (e.g., `9600-9700`).
123
+ #
124
+ # http.port: 9600-9700
125
+ #
126
+ # ------------ Module Settings ---------------
127
+ # Define modules here. Modules definitions must be defined as an array.
128
+ # The simple way to see this is to prepend each `name` with a `-`, and keep
129
+ # all associated variables under the `name` they are associated with, and
130
+ # above the next, like this:
131
+ #
132
+ # modules:
133
+ # - name: MODULE_NAME
134
+ # var.PLUGINTYPE1.PLUGINNAME1.KEY1: VALUE
135
+ # var.PLUGINTYPE1.PLUGINNAME1.KEY2: VALUE
136
+ # var.PLUGINTYPE2.PLUGINNAME1.KEY1: VALUE
137
+ # var.PLUGINTYPE3.PLUGINNAME3.KEY1: VALUE
138
+ #
139
+ # Module variable names must be in the format of
140
+ #
141
+ # var.PLUGIN_TYPE.PLUGIN_NAME.KEY
142
+ #
143
+ # modules:
144
+ #
145
+ # ------------ Cloud Settings ---------------
146
+ # Define Elastic Cloud settings here.
147
+ # Format of cloud.id is a base64 value e.g. dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyRub3RhcmVhbCRpZGVudGlmaWVy
148
+ # and it may have an label prefix e.g. staging:dXMtZ...
149
+ # This will overwrite 'var.elasticsearch.hosts' and 'var.kibana.host'
150
+ # cloud.id: <identifier>
151
+ #
152
+ # Format of cloud.auth is: <user>:<pass>
153
+ # This is optional
154
+ # If supplied this will overwrite 'var.elasticsearch.username' and 'var.elasticsearch.password'
155
+ # If supplied this will overwrite 'var.kibana.username' and 'var.kibana.password'
156
+ # cloud.auth: elastic:<password>
157
+ #
158
+ # ------------ Queuing Settings --------------
159
+ #
160
+ # Internal queuing model, "memory" for legacy in-memory based queuing and
161
+ # "persisted" for disk-based acked queueing. Defaults is memory
162
+ #
163
+ # queue.type: memory
164
+ #
165
+ # If using queue.type: persisted, the directory path where the data files will be stored.
166
+ # Default is path.data/queue
167
+ #
168
+ # path.queue:
169
+ #
170
+ # If using queue.type: persisted, the page data files size. The queue data consists of
171
+ # append-only data files separated into pages. Default is 64mb
172
+ #
173
+ # queue.page_capacity: 64mb
174
+ #
175
+ # If using queue.type: persisted, the maximum number of unread events in the queue.
176
+ # Default is 0 (unlimited)
177
+ #
178
+ # queue.max_events: 0
179
+ #
180
+ # If using queue.type: persisted, the total capacity of the queue in number of bytes.
181
+ # If you would like more unacked events to be buffered in Logstash, you can increase the
182
+ # capacity using this setting. Please make sure your disk drive has capacity greater than
183
+ # the size specified here. If both max_bytes and max_events are specified, Logstash will pick
184
+ # whichever criteria is reached first
185
+ # Default is 1024mb or 1gb
186
+ #
187
+ # queue.max_bytes: 1024mb
188
+ #
189
+ # If using queue.type: persisted, the maximum number of acked events before forcing a checkpoint
190
+ # Default is 1024, 0 for unlimited
191
+ #
192
+ # queue.checkpoint.acks: 1024
193
+ #
194
+ # If using queue.type: persisted, the maximum number of written events before forcing a checkpoint
195
+ # Default is 1024, 0 for unlimited
196
+ #
197
+ # queue.checkpoint.writes: 1024
198
+ #
199
+ # If using queue.type: persisted, the interval in milliseconds when a checkpoint is forced on the head page
200
+ # Default is 1000, 0 for no periodic checkpoint.
201
+ #
202
+ # queue.checkpoint.interval: 1000
203
+ #
204
+ # ------------ Dead-Letter Queue Settings --------------
205
+ # Flag to turn on dead-letter queue.
206
+ #
207
+ # dead_letter_queue.enable: false
208
+
209
+ # If using dead_letter_queue.enable: true, the maximum size of each dead letter queue. Entries
210
+ # will be dropped if they would increase the size of the dead letter queue beyond this setting.
211
+ # Default is 1024mb
212
+ # dead_letter_queue.max_bytes: 1024mb
213
+
214
+ # If using dead_letter_queue.enable: true, the interval in milliseconds where if no further events eligible for the DLQ
215
+ # have been created, a dead letter queue file will be written. A low value here will mean that more, smaller, queue files
216
+ # may be written, while a larger value will introduce more latency between items being "written" to the dead letter queue, and
217
+ # being available to be read by the dead_letter_queue input when items are are written infrequently.
218
+ # Default is 5000.
219
+ #
220
+ # dead_letter_queue.flush_interval: 5000
221
+
222
+ # If using dead_letter_queue.enable: true, the directory path where the data files will be stored.
223
+ # Default is path.data/dead_letter_queue
224
+ #
225
+ # path.dead_letter_queue:
226
+ #
227
+ # ------------ Metrics Settings --------------
228
+ #
229
+ # Bind address for the metrics REST endpoint
230
+ #
231
+ #http.host: "0.0.0.0"
232
+ #
233
+ # Bind port for the metrics REST endpoint, this option also accept a range
234
+ # (9600-9700) and logstash will pick up the first available ports.
235
+ #
236
+ # http.port: 9600-9700
237
+ #
238
+ # ------------ Debugging Settings --------------
239
+ #
240
+ # Options for log.level:
241
+ # * fatal
242
+ # * error
243
+ # * warn
244
+ # * info (default)
245
+ # * debug
246
+ # * trace
247
+ #
248
+ log.level: info
249
+ path.logs: /var/log/logstash
250
+ #
251
+ # ------------ Other Settings --------------
252
+ #
253
+ # Where to find custom plugins
254
+ # path.plugins: []
255
+ #
256
+ # Flag to output log lines of each pipeline in its separate log file. Each log filename contains the pipeline.name
257
+ # Default is false
258
+ # pipeline.separate_logs: false
259
+ #
260
+ # ------------ X-Pack Settings (not applicable for OSS build)--------------
261
+ #
262
+ # X-Pack Monitoring
263
+ # https://www.elastic.co/guide/en/logstash/current/monitoring-logstash.html
264
+ #xpack.monitoring.enabled: true
265
+ #xpack.monitoring.elasticsearch.username: logstash_system
266
+ #xpack.monitoring.elasticsearch.password: 123456
267
+ #xpack.monitoring.elasticsearch.proxy: ["http://proxy:port"]
268
+ #xpack.monitoring.elasticsearch.hosts: ["https://localhost:9200"]
269
+ # an alternative to hosts + username/password settings is to use cloud_id/cloud_auth
270
+ #xpack.monitoring.elasticsearch.cloud_id: monitoring_cluster_id:xxxxxxxxxx
271
+ #xpack.monitoring.elasticsearch.cloud_auth: logstash_system:password
272
+ # another authentication alternative is to use an Elasticsearch API key
273
+ #xpack.monitoring.elasticsearch.api_key: "id:api_key"
274
+ #xpack.monitoring.elasticsearch.ssl.certificate_authority: [ "/etc/logstash/elasticsearch-ca.pem" ]
275
+ #xpack.monitoring.elasticsearch.ssl.truststore.path: path/to/file
276
+ #xpack.monitoring.elasticsearch.ssl.truststore.password: password
277
+ #xpack.monitoring.elasticsearch.ssl.keystore.path: /path/to/file
278
+ #xpack.monitoring.elasticsearch.ssl.keystore.password: password
279
+ xpack.monitoring.elasticsearch.ssl.verification_mode: none
280
+ #xpack.monitoring.elasticsearch.sniffing: false
281
+ #xpack.monitoring.collection.interval: 10s
282
+ #xpack.monitoring.collection.pipeline.details.enabled: true
283
+ #
284
+ # X-Pack Management
285
+ # https://www.elastic.co/guide/en/logstash/current/logstash-centralized-pipeline-management.html
286
+ #xpack.management.enabled: false
287
+ #xpack.management.pipeline.id: ["main", "apache_logs"]
288
+ #xpack.management.elasticsearch.username: logstash_admin_user
289
+ #xpack.management.elasticsearch.password: password
290
+ #xpack.management.elasticsearch.proxy: ["http://proxy:port"]
291
+ #xpack.management.elasticsearch.hosts: ["https://es1:9200", "https://es2:9200"]
292
+ # an alternative to hosts + username/password settings is to use cloud_id/cloud_auth
293
+ #xpack.management.elasticsearch.cloud_id: management_cluster_id:xxxxxxxxxx
294
+ #xpack.management.elasticsearch.cloud_auth: logstash_admin_user:password
295
+ # another authentication alternative is to use an Elasticsearch API key
296
+ #xpack.management.elasticsearch.api_key: "id:api_key"
297
+ #xpack.management.elasticsearch.ssl.certificate_authority: [ "/path/to/ca.crt" ]
298
+ #xpack.management.elasticsearch.ssl.truststore.path: /path/to/file
299
+ #xpack.management.elasticsearch.ssl.truststore.password: password
300
+ #xpack.management.elasticsearch.ssl.keystore.path: /path/to/file
301
+ #xpack.management.elasticsearch.ssl.keystore.password: password
302
+ #xpack.management.elasticsearch.ssl.verification_mode: certificate
303
+ #xpack.management.elasticsearch.sniffing: false
304
+ #xpack.management.logstash.poll_interval: 5s
@@ -0,0 +1,20 @@
1
+ ---
2
+ - name: Restart nginx
3
+ service:
4
+ name: nginx
5
+ state: restarted
6
+
7
+ - name: Restart logstash
8
+ service:
9
+ name: logstash
10
+ state: restarted
11
+
12
+ - name: Restart heartbeat-elastic
13
+ service:
14
+ name: heartbeat-elastic
15
+ state: restarted
16
+
17
+ - name: Restart apm-server
18
+ service:
19
+ name: apm-server
20
+ state: restarted