cloud-mu 3.5.0 → 3.6.3

data/ansible/roles/mu-compliance/handlers/main.yml

@@ -0,0 +1,2 @@
+---
+# handlers file for mu-windows

data/ansible/roles/mu-compliance/meta/main.yml

@@ -0,0 +1,53 @@
+galaxy_info:
+  author: your name
+  description: your description
+  company: your company (optional)
+
+  # If the issue tracker for your role is not on github, uncomment the
+  # next line and provide a value
+  # issue_tracker_url: http://example.com/issue/tracker
+
+  # Choose a valid license ID from https://spdx.org - some suggested licenses:
+  # - BSD-3-Clause (default)
+  # - MIT
+  # - GPL-2.0-or-later
+  # - GPL-3.0-only
+  # - Apache-2.0
+  # - CC-BY-4.0
+  license: license (GPL-2.0-or-later, MIT, etc)
+
+  min_ansible_version: 2.4
+
+  # If this a Container Enabled role, provide the minimum Ansible Container version.
+  # min_ansible_container_version:
+
+  #
+  # Provide a list of supported platforms, and for each platform a list of versions.
+  # If you don't wish to enumerate all versions for a particular platform, use 'all'.
+  # To view available platforms and versions (or releases), visit:
+  # https://galaxy.ansible.com/api/v1/platforms/
+  #
+  # platforms:
+  # - name: Fedora
+  #   versions:
+  #   - all
+  #   - 25
+  # - name: SomePlatform
+  #   versions:
+  #   - all
+  #   - 1.0
+  #   - 7
+  #   - 99.99
+
+  galaxy_tags: []
+    # List tags for your role here, one per line. A tag is a keyword that describes
+    # and categorizes the role. Users find roles by searching for tags. Be sure to
+    # remove the '[]' above, if you add tags to this list.
+    #
+    # NOTE: A tag is limited to a single word comprised of alphanumeric characters.
+    #       Maximum 20 tags per role.
+
+dependencies: []
+  # List your role dependencies here, one per line. Be sure to remove the '[]' above,
+  # if you add dependencies to this list.
+  

data/ansible/roles/mu-compliance/tasks/main.yml

@@ -0,0 +1,45 @@
+---
+- name: Ensure existence of c:/temp
+  win_file:
+    path: c:/temp
+    state: directory
+
+- name: Check for existence of OpenSCAP
+  win_shell: Get-ChildItem -Name "c:/Program Files (x86)/OpenSCAP {{ OSCAP_VERSION }}/"
+  ignore_errors: true
+  no_log: true
+  register: openscap_exists
+
+- name: Install vcredist
+  win_package:
+    path: https://aka.ms/vs/16/release/vc_redist.x86.exe
+    arguments:
+    - /install
+    - /passive
+    - /norestart
+
+- name: Download OpenSCAP installer
+  when: openscap_exists is failed
+  win_get_url:
+    url: "{{ OSCAP_URL }}"
+    dest: "c:/temp/{{ OSCAP_INSTALLER }}"
+
+- name: "Install OpenSCAP"
+  when: openscap_exists is failed
+  win_package:
+    path: "c:/temp/{{ OSCAP_INSTALLER }}"
+    arguments:
+    - /quiet
+    - /norestart
+
+- name: "Copy scan benchmarks"
+  win_copy:
+    src: U_MS_Windows_Server_2016_V2R1_STIG_SCAP_1-2_Benchmark.xml
+    dest: "c:/temp"
+
+- debug: var=ansible_facts
+
+- name: "run a scan, why not {{ ansible_distribution }} -- {{ ansible_distribution_version }} -- {{ ansible_distribution_major_version  }} -- {{ ansible_distribution_release  }}"
+  win_shell: |
+    cd c:/temp
+    oscap --report report.txt xccdf eval U_MS_Windows_Server_2016_V2R1_STIG_SCAP_1-2_Benchmark.xml

data/ansible/roles/mu-compliance/tests/inventory

@@ -0,0 +1,2 @@
+localhost
+

data/ansible/roles/mu-compliance/tests/test.yml

@@ -0,0 +1,5 @@
+---
+- hosts: localhost
+  remote_user: root
+  roles:
+    - mu-compliance

data/ansible/roles/mu-compliance/vars/main.yml

@@ -0,0 +1,4 @@
+---
+OSCAP_VERSION: "1.3.0"
+OSCAP_INSTALLER: "OpenSCAP-{{ OSCAP_VERSION }}-win32.msi"
+OSCAP_URL: "https://github.com/OpenSCAP/openscap/releases/download/{{ OSCAP_VERSION }}/{{ OSCAP_INSTALLER }}"

data/ansible/roles/mu-elastic/README.md

@@ -0,0 +1,51 @@
+Role Name
+=========
+
+Set up an Elastic & Kibana front with an Nginx proxy. Assumes Logstash and APM exist somewhere.
+
+Requirements
+------------
+
+Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required.
+
+Role Variables
+--------------
+
+A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well.
+
+Dependencies
+------------
+
+A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles.
+
+Example Playbook
+----------------
+
+Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too:
+
+    - hosts: servers
+      roles:
+         - { role: username.rolename, x: 42 }
+
+License
+-------
+
+Copyright:: Copyright (c) 2020 eGlobalTech, Inc., all rights reserved
+
+Licensed under the BSD-3 license (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License in the root of the project or at
+
+    http://egt-labs.com/mu/LICENSE.html
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+
+Author Information
+------------------
+
+An optional section for the role authors to include contact information, or a website (HTML is not allowed).

data/ansible/roles/mu-elastic/defaults/main.yml

@@ -0,0 +1,2 @@
+---
+# defaults file for mu-elastic

data/ansible/roles/mu-elastic/files/jvm.options

@@ -0,0 +1,93 @@
+################################################################
+##
+## JVM configuration
+##
+################################################################
+##
+## WARNING: DO NOT EDIT THIS FILE. If you want to override the
+## JVM options in this file, or set any additional options, you
+## should create one or more files in the jvm.options.d
+## directory containing your adjustments.
+##
+## See https://www.elastic.co/guide/en/elasticsearch/reference/current/jvm-options.html
+## for more information.
+##
+################################################################
+
+# CVE-2021-44228
+-Dlog4j2.formatMsgNoLookups=true
+
+################################################################
+## IMPORTANT: JVM heap size
+################################################################
+##
+## The heap size is automatically configured by Elasticsearch
+## based on the available memory in your system and the roles
+## each node is configured to fulfill. If specifying heap is
+## required, it should be done through a file in jvm.options.d,
+## and the min and max should be set to the same value. For
+## example, to set the heap to 4 GB, create a new file in the
+## jvm.options.d directory containing these lines:
+##
+-Xms5g
+-Xmx5g
+##
+## See https://www.elastic.co/guide/en/elasticsearch/reference/current/heap-size.html
+## for more information
+##
+################################################################
+
+
+################################################################
+## Expert settings
+################################################################
+##
+## All settings below here are considered expert settings. Do
+## not adjust them unless you understand what you are doing. Do
+## not edit them in this file; instead, create a new file in the
+## jvm.options.d directory containing your adjustments.
+##
+################################################################
+
+## GC configuration
+8-13:-XX:+UseConcMarkSweepGC
+8-13:-XX:CMSInitiatingOccupancyFraction=75
+8-13:-XX:+UseCMSInitiatingOccupancyOnly
+
+## G1GC Configuration
+# NOTE: G1 GC is only supported on JDK version 10 or later
+# to use G1GC, uncomment the next two lines and update the version on the
+# following three lines to your version of the JDK
+# 10-13:-XX:-UseConcMarkSweepGC
+# 10-13:-XX:-UseCMSInitiatingOccupancyOnly
+14-:-XX:+UseG1GC
+
+## JVM temporary directory
+-Djava.io.tmpdir=${ES_TMPDIR}
+
+## heap dumps
+
+# generate a heap dump when an allocation from the Java heap fails; heap dumps
+# are created in the working directory of the JVM unless an alternative path is
+# specified
+-XX:+HeapDumpOnOutOfMemoryError
+
+# specify an alternative path for heap dumps; ensure the directory exists and
+# has sufficient space
+-XX:HeapDumpPath=/var/lib/elasticsearch
+
+# specify an alternative path for JVM fatal error logs
+-XX:ErrorFile=/var/log/elasticsearch/hs_err_pid%p.log
+
+## JDK 8 GC logging
+8:-XX:+PrintGCDetails
+8:-XX:+PrintGCDateStamps
+8:-XX:+PrintTenuringDistribution
+8:-XX:+PrintGCApplicationStoppedTime
+8:-Xloggc:/var/log/elasticsearch/gc.log
+8:-XX:+UseGCLogFileRotation
+8:-XX:NumberOfGCLogFiles=32
+8:-XX:GCLogFileSize=64m
+
+# JDK 9+ GC logging
+9-:-Xlog:gc*,gc+age=trace,safepoint:file=/var/log/elasticsearch/gc.log:utctime,pid,tags:filecount=32,filesize=64m

data/ansible/roles/mu-elastic/handlers/main.yml

@@ -0,0 +1,10 @@
+---
+- name: Restart elasticsearch
+  service:
+    name: elasticsearch
+    state: restarted
+
+- name: Restart kibana
+  service:
+    name: kibana
+    state: restarted

data/ansible/roles/mu-elastic/meta/main.yml

@@ -0,0 +1,52 @@
+galaxy_info:
+  author: your name
+  description: your role description
+  company: your company (optional)
+
+  # If the issue tracker for your role is not on github, uncomment the
+  # next line and provide a value
+  # issue_tracker_url: http://example.com/issue/tracker
+
+  # Choose a valid license ID from https://spdx.org - some suggested licenses:
+  # - BSD-3-Clause (default)
+  # - MIT
+  # - GPL-2.0-or-later
+  # - GPL-3.0-only
+  # - Apache-2.0
+  # - CC-BY-4.0
+  license: license (GPL-2.0-or-later, MIT, etc)
+
+  min_ansible_version: 2.1
+
+  # If this a Container Enabled role, provide the minimum Ansible Container version.
+  # min_ansible_container_version:
+
+  #
+  # Provide a list of supported platforms, and for each platform a list of versions.
+  # If you don't wish to enumerate all versions for a particular platform, use 'all'.
+  # To view available platforms and versions (or releases), visit:
+  # https://galaxy.ansible.com/api/v1/platforms/
+  #
+  # platforms:
+  # - name: Fedora
+  #   versions:
+  #   - all
+  #   - 25
+  # - name: SomePlatform
+  #   versions:
+  #   - all
+  #   - 1.0
+  #   - 7
+  #   - 99.99
+
+  galaxy_tags: []
+    # List tags for your role here, one per line. A tag is a keyword that describes
+    # and categorizes the role. Users find roles by searching for tags. Be sure to
+    # remove the '[]' above, if you add tags to this list.
+    #
+    # NOTE: A tag is limited to a single word comprised of alphanumeric characters.
+    #       Maximum 20 tags per role.
+
+dependencies: []
+  # List your role dependencies here, one per line. Be sure to remove the '[]' above,
+  # if you add dependencies to this list.

data/ansible/roles/mu-elastic/tasks/main.yml

@@ -0,0 +1,186 @@
+---
+
+- name: remove firewalld
+  package:
+    name: firewalld
+    state: absent
+
+- name: make sure iptables is available
+  package:
+    name: iptables-services
+    state: present
+
+- name: allow inbound for internal services
+  iptables:
+    chain: INPUT
+    source: "0.0.0.0/0"
+    destination_port: "{{ item }}"
+    protocol: tcp
+    jump: ACCEPT
+  with_items:
+  - "5601" # Kibana
+  - "8200" # APM
+  - "9200" # Elastic
+  - "9300" # Elastic
+
+- name: add yum repo for ElasticSearch
+  yum_repository:
+    name: elasticsearch
+    gpgkey: https://artifacts.elastic.co/GPG-KEY-elasticsearch
+    baseurl: https://artifacts.elastic.co/packages/7.x/yum
+    description: Elasticsearch repository for 7.x packages
+
+- name: install elasticsearch and related packages
+  package:
+    name: "{{ item }}"
+    state: present
+  with_items:
+  - elasticsearch
+  - kibana
+  - filebeat
+  - httpd-tools
+  - expect
+
+# XXX (this insecure convolution belongs in some kind of shared library)
+# We have to go through this tempfile dance because jinja doesn't actually see
+# decrypted vault data, apparently, so as soon as we try to do anything other
+# than write the whole decrypted blob to a file it fails to decrypt. That's
+# even if we try the various workarounds from:
+# https://github.com/ansible/ansible/issues/24425
+- name: create Elastic password temp file (ugh)
+  tempfile:
+    state: file
+    suffix: temp
+  register: elasticpw_tmpfile
+- name: "Write Elastic password to temp file"
+  copy:
+    dest: "{{ elasticpw_tmpfile.path }}"
+    content: "{{ mu_vaults[mu_deploy_id]['elasticpw'] }}"
+- name: "Load Elastic password from temp file"
+  slurp:
+    src: "{{ elasticpw_tmpfile.path }}"
+  register: elasticpw_yaml
+- name: From tmp YAML to dict
+  set_fact:
+    elasticpw_dict: "{{ elasticpw_yaml.content | b64decode | from_yaml }}"
+
+- name: decrypt elastic password
+  set_fact:
+    elasticpw: "{{ elasticpw_dict['password'] }}"
+
+- name: ElasticSearch config files in /etc/elasticsearch
+  copy:
+    dest: "/etc/elasticsearch/{{ item }}"
+    src: "{{ item }}"
+    mode: 0660
+    owner: root
+    group: elasticsearch
+  become: yes
+  with_items:
+  - jvm.options
+  notify:
+  - Restart elasticsearch
+
+- name: /etc/elasticsearch/elasticsearch.yml
+  template:
+    src: elasticsearch.yml.j2
+    dest: /etc/elasticsearch/elasticsearch.yml
+    mode: 0660
+    owner: root
+    group: elasticsearch
+  become: yes
+  notify:
+  - Restart elasticsearch
+
+- name: Copy ElasticSearch certificate store into place
+  copy:
+    dest: "/etc/elasticsearch/{{ item }}"
+    src: "/opt/mu/var/ssl/{{ inventory_hostname }}.pfx"
+    mode: 0640
+    group: elasticsearch
+  become: yes
+  with_items:
+  - elastic-certificates.p12
+  - http.p12
+  notify:
+  - Restart elasticsearch
+
+- name: "Enable and start ElasticSearch"
+  service:
+    name: elasticsearch
+    state: started
+
+- name: see if Elastic system passwords have been set
+  shell:
+    cmd: "/usr/share/elasticsearch/bin/elasticsearch-setup-passwords -s interactive 2>&1 | grep 'user has already been changed'"
+  ignore_errors: true
+  async: 30
+  poll: 5
+  no_log: true
+  register: passwords_set
+
+- name: Create password-setter script
+  template:
+    src: password_set.expect.j2
+    dest: "/root/password_set.expect"
+    mode: 0700
+    owner: root
+  become: yes
+  when: passwords_set is failed
+
+- name: set elastic system passwords
+  command:
+    cmd: "/root/password_set.expect"
+  no_log: true
+  become: yes
+  when: passwords_set is failed
+
+- name: Remove password-setter scripts
+  file:
+    path: /root/password_set.expect
+    state: absent
+
+- name: Copy Mu's CA
+  copy:
+    dest: "/etc/{{ item }}/elasticsearch-ca.pem"
+    src: /opt/mu/var/ssl/Mu_CA.pem
+    mode: 0644
+  become: yes
+  notify:
+  - Restart kibana
+  with_items:
+  - kibana
+
+- name: Kibana config files in /etc/kibana
+  template:
+    dest: /etc/kibana/kibana.yml
+    src: kibana.yml.j2
+    mode: 0640
+  become: yes
+  notify:
+  - Restart kibana
+
+- name: "Enable and start Kibana"
+  service:
+    name: kibana
+    state: started
+
+- name: Check whether ElasticSearch S3 backup plugin is installed
+  shell: |
+    /usr/share/elasticsearch/bin/elasticsearch-plugin list | grep '^repository-s3$'
+  ignore_errors: true
+  register: s3_present
+  no_log: true
+  become: yes
+
+- name: Install ElasticSearch S3 backup plugin
+  shell: "/usr/share/elasticsearch/bin/elasticsearch-plugin install -b repository-s3"
+  become: yes
+  when: s3_present is failed
+  notify:
+  - Restart elasticsearch
+
+- name: "Force set a reasonable number of shards {{ mu_deployment['servers']['backend'] | json_query('[*].private_ip_address') }}"
+  shell: |
+    /bin/curl --user "elastic:{{ mu_vaults[mu_deploy_id]['elasticpw'] }}" -k -X PUT "https://{{ mu_deployment['servers']['backend'][inventory_hostname]['private_ip_address'] }}:9200/_cluster/settings" -H "Content-Type: application/json" -d '{ "persistent": { "cluster.max_shards_per_node": "10000" } }'
+  become: yes

data/ansible/roles/mu-elastic/templates/elasticsearch.yml.j2

@@ -0,0 +1,110 @@
+# ======================== Elasticsearch Configuration =========================
+#
+# NOTE: Elasticsearch comes with reasonable defaults for most settings.
+#       Before you set out to tweak and tune the configuration, make sure you
+#       understand what are you trying to accomplish and the consequences.
+#
+# The primary way of configuring a node is via this file. This template lists
+# the most important settings you may want to configure for a production cluster.
+#
+# Please consult the documentation for further information on configuration options:
+# https://www.elastic.co/guide/en/elasticsearch/reference/index.html
+#
+# ---------------------------------- Cluster -----------------------------------
+#
+# Use a descriptive name for your cluster:
+#
+#cluster.name: icras-elastic
+#
+# ------------------------------------ Node ------------------------------------
+#
+# Use a descriptive name for the node:
+#
+#node.name: 0
+#
+# Add custom attributes to the node:
+#
+#node.attr.rack: r1
+#
+# ----------------------------------- Paths ------------------------------------
+#
+# Path to directory where to store the data (separate multiple locations by comma):
+#
+path.data: /var/lib/elasticsearch
+#
+# Path to log files:
+#
+path.logs: /var/log/elasticsearch
+#
+# ----------------------------------- Memory -----------------------------------
+#
+# Lock the memory on startup:
+#
+#bootstrap.memory_lock: true
+#
+# Make sure that the heap size is set to about half the memory available
+# on the system and that the owner of the process is allowed to use this
+# limit.
+#
+# Elasticsearch performs poorly when the system is swapping the memory.
+#
+# ---------------------------------- Network -----------------------------------
+#
+# Set the bind address to a specific IP (IPv4 or IPv6):
+#
+#network.host: 0
+#
+# Set a custom port for HTTP:
+#
+#http.port: 9200
+#
+# For more information, consult the network module documentation.
+#
+# --------------------------------- Discovery ----------------------------------
+#
+# Pass an initial list of hosts to perform discovery when this node is started:
+# The default list of hosts is ["127.0.0.1", "[::1]"]
+#
+#discovery.seed_hosts: ["host1", "host2"]
+#
+# Bootstrap the cluster using an initial set of master-eligible nodes:
+#
+#cluster.initial_master_nodes: ["node-1", "node-2"]
+#
+# For more information, consult the discovery and cluster formation module documentation.
+#
+# ---------------------------------- Gateway -----------------------------------
+#
+# Block initial recovery after a full cluster restart until N nodes are started:
+#
+#gateway.recover_after_nodes: 3
+#
+# For more information, consult the gateway module documentation.
+#
+# ---------------------------------- Various -----------------------------------
+#
+# Require explicit names when deleting indices:
+#
+#action.destructive_requires_name: true
+
+cluster.initial_master_nodes: {% for node,meta in mu_deployment['servers']['backend'].items() %}
+  {%- for k,v in meta.items() %}
+    {%- if k in ["private_ip_address"] %}
+{{ v }}
+    {%- endif %}
+  {%- endfor %}
+  {%- if not loop.last %},{%- endif %}
+{%- endfor %}
+
+xpack.security.enabled: true
+#discovery.type: single-node
+xpack.security.transport.ssl.enabled: true
+xpack.security.transport.ssl.verification_mode: certificate
+xpack.security.transport.ssl.keystore.path: elastic-certificates.p12
+xpack.security.transport.ssl.truststore.path: elastic-certificates.p12
+xpack.security.http.ssl.enabled: true
+xpack.security.http.ssl.keystore.path: "http.p12"
+#xpack.license.self_generated.type: basic
+#xpack.security.http.ssl.enabled: true
+#xpack.security.http.ssl.keystore.path: "/etc/elasticsearch/config/certs/elasticsearch/http.p12"
+network.host: _site_