cloud-mu 3.5.0 → 3.6.3

data/ansible/roles/mu-logstash/meta/main.yml

@@ -0,0 +1,52 @@
+galaxy_info:
+  author: your name
+  description: your role description
+  company: your company (optional)
+
+  # If the issue tracker for your role is not on github, uncomment the
+  # next line and provide a value
+  # issue_tracker_url: http://example.com/issue/tracker
+
+  # Choose a valid license ID from https://spdx.org - some suggested licenses:
+  # - BSD-3-Clause (default)
+  # - MIT
+  # - GPL-2.0-or-later
+  # - GPL-3.0-only
+  # - Apache-2.0
+  # - CC-BY-4.0
+  license: license (GPL-2.0-or-later, MIT, etc)
+
+  min_ansible_version: 2.1
+
+  # If this a Container Enabled role, provide the minimum Ansible Container version.
+  # min_ansible_container_version:
+
+  #
+  # Provide a list of supported platforms, and for each platform a list of versions.
+  # If you don't wish to enumerate all versions for a particular platform, use 'all'.
+  # To view available platforms and versions (or releases), visit:
+  # https://galaxy.ansible.com/api/v1/platforms/
+  #
+  # platforms:
+  # - name: Fedora
+  #   versions:
+  #   - all
+  #   - 25
+  # - name: SomePlatform
+  #   versions:
+  #   - all
+  #   - 1.0
+  #   - 7
+  #   - 99.99
+
+  galaxy_tags: []
+    # List tags for your role here, one per line. A tag is a keyword that describes
+    # and categorizes the role. Users find roles by searching for tags. Be sure to
+    # remove the '[]' above, if you add tags to this list.
+    #
+    # NOTE: A tag is limited to a single word comprised of alphanumeric characters.
+    #       Maximum 20 tags per role.
+
+dependencies: []
+  # List your role dependencies here, one per line. Be sure to remove the '[]' above,
+  # if you add dependencies to this list.

data/ansible/roles/mu-logstash/tasks/main.yml

@@ -0,0 +1,254 @@
+---
+
+- name: remove firewalld
+  package:
+    name: firewalld
+    state: absent
+
+- name: make sure iptables is available
+  package:
+    name: iptables-services
+    state: present
+
+- name: allow inbound for public traffic
+  iptables:
+    chain: INPUT
+    source: 0.0.0.0/0
+    destination_port: "{{ item }}"
+    protocol: tcp
+    jump: ACCEPT
+  with_items:
+  - "80"
+  - "443"
+  - "8080"
+  - "8008"
+  - "8200"
+  - "5044"
+
+- name: add yum repo for ElasticSearch
+  yum_repository:
+    name: elasticsearch
+    gpgkey: https://artifacts.elastic.co/GPG-KEY-elasticsearch
+    baseurl: https://artifacts.elastic.co/packages/7.x/yum
+    description: Elasticsearch repository for 7.x packages
+
+- name: install logstash and related packages
+  package:
+    name: "{{ item }}"
+    state: present
+  with_items:
+  - logstash
+  - heartbeat-elastic
+  - nginx
+  - apm-server
+  - httpd-tools
+  - policycoreutils-python
+
+# XXX (this insecure convolution belongs in some kind of shared library)
+# We have to go through this tempfile dance because jinja doesn't actually see
+# decrypted vault data, apparently, so as soon as we try to do anything other
+# than write the whole decrypted blob to a file it fails to decrypt. That's
+# even if we try the various workarounds from:
+# https://github.com/ansible/ansible/issues/24425
+- name: create Elastic password temp file (ugh)
+  tempfile:
+    state: file
+    suffix: temp
+  register: elasticpw_tmpfile
+- name: "Write Elastic password to temp file"
+  copy:
+    dest: "{{ elasticpw_tmpfile.path }}"
+    content: "{{ mu_vaults[mu_deploy_id]['elasticpw'] }}"
+- name: "Load Elastic password from temp file"
+  slurp:
+    src: "{{ elasticpw_tmpfile.path }}"
+  register: elasticpw_yaml
+- name: From tmp YAML to dict
+  set_fact:
+    elasticpw_dict: "{{ elasticpw_yaml.content | b64decode | from_yaml }}"
+
+- name: decrypt elastic password
+  set_fact:
+    elasticpw: "{{ elasticpw_dict['password'] }}"
+
+- name: Logstash config files in /etc/logstash
+  copy:
+    dest: "/etc/logstash/{{ item }}"
+    src: "{{ item }}"
+    mode: 0644
+  become: yes
+  with_items:
+  - jvm.options
+  - logstash.yml
+  notify:
+  - Restart logstash
+
+- name: Logstash config files in /etc/logstash/conf.d
+  copy:
+    dest: "/etc/logstash/conf.d/{{ item }}"
+    src: "{{ item }}"
+    mode: 0644
+  become: yes
+  with_items:
+  - 02-beats-input.conf
+  - 10-rails-filter.conf
+
+- name: Copy Mu's CA
+  copy:
+    dest: "/etc/{{ item }}/elasticsearch-ca.pem"
+    src: /opt/mu/var/ssl/Mu_CA.pem
+    mode: 0644
+  become: yes
+  notify:
+  - Restart logstash
+  - Restart apm-server
+  with_items:
+  - logstash
+  - apm-server
+
+- name: Logstash Elastic integration config
+  template:
+    src: 30-elasticsearch-output.conf.j2
+    dest: /etc/logstash/conf.d/30-elasticsearch-output.conf
+    mode: 0644
+
+- name: Logstash CloudTrail integration config
+  template:
+    src: 20-cloudtrail.conf.j2
+    dest: /etc/logstash/conf.d/20-cloudtrail.conf
+    mode: 0644
+
+- name: Elastic Heartbeat config
+  template:
+    src: heartbeat.yml.j2
+    dest: /etc/heartbeat/heartbeat.yml
+    mode: 0600
+  notify:
+  - Restart heartbeat-elastic
+
+- name: Copy Nginx certificate into place
+  copy:
+    dest: "/etc/ssl/certs/{{ inventory_hostname }}.crt"
+    src: "/opt/mu/var/ssl/{{ inventory_hostname }}.crt"
+    mode: 0644
+  become: yes
+  notify:
+  - Restart nginx
+
+- name: Make sure /etc/ssl/private exists
+  file:
+    path: /etc/ssl/private
+    mode: 0077
+    state: directory
+
+- name: Copy Nginx key into place
+  copy:
+    dest: "/etc/ssl/private/{{ inventory_hostname }}.key"
+    src: "/opt/mu/var/ssl/{{ inventory_hostname }}.key"
+    mode: 0644
+  become: yes
+  notify:
+  - Restart nginx
+
+- name: Nginx configs
+  template:
+    src: "nginx/{{ item }}.j2"
+    dest: "/etc/nginx/conf.d/{{ item }}"
+    mode: 0644
+  with_items:
+  - apm.conf
+  - default.conf
+  - elastic.conf
+  notify:
+  - Restart nginx
+
+- name: Enable and start logstash
+  service:
+    name: logstash
+    state: started
+
+- name: Enable and start Elastic Heartbeat
+  service:
+    name: heartbeat-elastic
+    state: started
+
+- name: Enable and start Nginx
+  service:
+    name: nginx
+    state: started
+
+- name: set elastic password
+  command:
+    cmd: "/bin/htpasswd -b -c /etc/nginx/htpasswd.users elastic \"{{ elasticpw }}\""
+  no_log: true
+  become: yes
+
+- name: fix permissions on /etc/nginx/htpasswd.users
+  file:
+    path: /etc/nginx/htpasswd.users
+    owner: nginx
+    mode: 0600
+
+- name: Check whether logstash CloudTrail plugin is installed
+  shell: "/usr/share/logstash/bin/logstash-plugin list logstash-codec-cloudtrail"
+  ignore_errors: true
+  register: cloudtrail_present
+  no_log: true
+  become: yes
+
+- name: Install logstash CloudTrail plugin
+  shell: /usr/share/logstash/bin/logstash-plugin install logstash-codec-cloudtrail
+  become: yes
+  when: cloudtrail_present is failed
+  notify:
+  - Restart logstash
+
+- name: Fix permissions on Logstash plugins
+  shell: |
+    find /usr/share/logstash/vendor/bundle/jruby/ -type d -exec chmod go+rx {} \;
+    find /usr/share/logstash/vendor/bundle/jruby/ -type f -exec chmod go+r {} \;
+  become: yes
+
+- name: Check whether Kibana port is allowed
+  shell: "/usr/sbin/semanage port -l | grep ^http_port_t | grep 5601"
+  ignore_errors: true
+  register: kibana_allowed
+  no_log: true
+  become: yes
+
+- name: Allow Nginx to connect to Kibana
+  command: "/usr/sbin/semanage port -a -t http_port_t -p tcp 5601"
+  become: yes
+  when: kibana_allowed is failed
+  notify:
+  - Restart nginx
+
+- name: Check whether Elastic port is allowed
+  shell: "/usr/sbin/semanage port -l | grep ^http_port_t | grep 9200"
+  ignore_errors: true
+  register: elastic_allowed
+  no_log: true
+  become: yes
+
+- name: Allow Nginx to connect to Elastic
+  command: "/usr/sbin/semanage port -m -t http_port_t -p tcp 9200"
+  ignore_errors: true
+  become: yes
+  when: elastic_allowed is failed
+  notify:
+  - Restart nginx
+
+- name: APM Server config
+  template:
+    src: apm-server.yml.j2
+    dest: /etc/apm-server/apm-server.yml
+    owner: root
+    group: apm-server
+    mode: 0644
+  notify:
+  - Restart apm-server
+
+- name: Enable and start APM Server
+  service:
+    name: apm-server
+    state: started

data/ansible/roles/mu-logstash/templates/20-cloudtrail.conf.j2

@@ -0,0 +1,28 @@
+{%- if application_attributes is defined and "cloudtrail_sources" in application_attributes: %}
+{% for trail in application_attributes["cloudtrail_sources"]: %}
+input {
+  s3 {
+    bucket => "{{ trail['bucket'] }}"
+    prefix => "AWSLogs/"
+    codec => cloudtrail {}
+    ecs_compatibility => v1
+    id => "cloudtrail{{ trail['tag'] }}"
+{% if "role_arn" in trail %}
+    role_arn => "{{ trail["role_arn"] }}"
+{% endif %}
+    tags => ["AWS", "cloudtrail", "{{ trail['tag'] }}"]
+    type => "cloudtrail"
+  }
+}
+{%- endfor %}
+{%- endif %}
+
+filter {
+  if [type] == "cloudtrail" {
+    geoip {
+      source => "sourceIPAddress"
+      ecs_compatibility => v1
+      target => "geoip"
+    }
+  }
+}

data/ansible/roles/mu-logstash/templates/30-elasticsearch-output.conf.j2

@@ -0,0 +1,19 @@
+output {
+  elasticsearch {
+    hosts => {%- for node,meta in mu_deployment['servers']['backend'].items() %}
+  {%- for k,v in meta.items() %}
+    {%- if k in ["private_ip_address"] %} "https://{{ v }}:9200"
+    {%- endif %}
+  {%- endfor %}
+  {%- if not loop.last %},{%- endif %}
+{%- endfor %}
+
+    ssl => true
+    ssl_certificate_verification => false
+    user => "elastic"
+    password => "{{ elasticpw }}"
+    cacert => "/etc/logstash/elasticsearch-ca.pem"
+    manage_template => false
+    index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
+  }
+}

data/ansible/roles/mu-logstash/templates/apm-server.yml.j2

@@ -0,0 +1,33 @@
+apm-server:
+  host: "0.0.0.0:8200"
+  concurrent_requests: 5
+  rum:
+    enabled: true
+  kibana:
+    enabled: true
+    username: "elastic"
+    password: "{{ elasticpw }}"
+
+output.elasticsearch:
+  hosts: [
+{%- for node,meta in mu_deployment['servers']['backend'].items() %}
+  {%- for k,v in meta.items() %}
+    {%- if k in ["private_ip_address"] %}
+"https://{{ v }}:9200"
+    {%- endif %}
+  {%- endfor %}
+  {%- if not loop.last %},{%- endif %}
+{%- endfor %}]
+  username: "elastic"
+  password: "{{ elasticpw }}"
+  protocol: "https"
+  worker: 2
+  ssl:
+    enabled: true
+    verification_mode: none
+    certificate_authorities: ["/etc/apm-server/elasticsearch-ca.pem"]
+    supported_protocols: ["TLSv1.2"]
+
+#queue.mem.events: 4096
+
+max_procs: 4

data/ansible/roles/mu-logstash/templates/heartbeat.yml.j2

@@ -0,0 +1,29 @@
+heartbeat.monitors:
+  - type: http
+    id: elk
+    name: elk
+    ports: [5601, 9200]
+    schedule: '@every 10s'
+    urls: {%- for node,meta in mu_deployment['servers']['frontend'].items() %}
+  {%- for k,v in meta.items() %}
+    {%- if k in ["private_ip_address"] %} ["https://{{ v }}"]
+    {%- endif %}
+  {%- endfor %}
+  {%- if not loop.last %},{%- endif %}
+{%- endfor %}
+
+processors:
+  - add_cloud_metadata: ~
+  - add_host_metadata: ~
+
+output.logstash:
+  hosts: ["localhost:5044"]
+
+setup:kibana:
+  host: {%- for node,meta in mu_deployment['servers']['frontend'].items() %}
+  {%- for k,v in meta.items() %}
+    {%- if k in ["public_dns_name"] %} ["{{ v }}/"]
+    {%- endif %}
+  {%- endfor %}
+  {%- if not loop.last %},{%- endif %}
+{%- endfor %}

data/ansible/roles/mu-logstash/templates/nginx/apm.conf.j2

@@ -0,0 +1,25 @@
+server {
+  listen 8008 ssl;
+  listen [::]:8008 ssl;
+
+  server_name {{ inventory_hostname }} {{ ec2['public_dns_name'] }} {{ ec2['private_dns_name'] }};
+  ssl_certificate /etc/ssl/certs/{{ inventory_hostname }}.crt;
+  ssl_certificate_key /etc/ssl/private/{{ inventory_hostname }}.key;
+#  ssl_dhparam /etc/ssl/certs/dhparam.pem;
+
+  location / {
+     proxy_pass {%- for node,meta in mu_deployment['servers']['backend'].items() %}
+  {%- for k,v in meta.items() %}
+    {%- if k in ["private_ip_address"] %} https://{{ v }}:8200;
+    {%- endif %}
+  {%- endfor %}
+  {%- if not loop.last %},{%- endif %}
+{%- endfor %}
+
+     proxy_http_version 1.1;
+     proxy_set_header Upgrade $http_upgrade;
+     proxy_set_header Connection 'upgrade';
+     proxy_set_header Host $host;
+     proxy_cache_bypass $http_upgrade;
+  }
+}

data/ansible/roles/mu-logstash/templates/nginx/default.conf.j2

@@ -0,0 +1,56 @@
+server {
+    listen 443 http2 ssl;
+    listen [::]:443 http2 ssl;
+
+    server_name {{ inventory_hostname }} {{ ec2['public_dns_name'] }} {{ ec2['private_dns_name'] }};
+
+    #auth_basic "Restricted Access";
+    #auth_basic_user_file /etc/nginx/htpasswd.users;
+
+    ssl_certificate /etc/ssl/certs/{{ inventory_hostname }}.crt;
+    ssl_certificate_key /etc/ssl/private/{{ inventory_hostname }}.key;
+#    ssl_dhparam /etc/ssl/certs/dhparam.pem;
+
+    location / {
+        auth_basic "Restricted Access";
+        auth_basic_user_file /etc/nginx/htpasswd.users;
+        proxy_pass {%- for node,meta in mu_deployment['servers']['backend'].items() %}
+  {%- for k,v in meta.items() %}
+    {%- if k in ["private_ip_address"] %} http://{{ v }}:5601;
+    {%- endif %}
+  {%- endfor %}
+  {%- if not loop.last %},{%- endif %}
+{%- endfor %}
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection 'upgrade';
+        proxy_set_header Host $host;
+        proxy_cache_bypass $http_upgrade;
+    }
+
+    #location /elastic {
+    #    set $proxy_port 9200;
+    #    proxy_pass http://localhost:9200;
+    #    proxy_http_version 1.1;
+    #    proxy_set_header Upgrade $http_upgrade;
+    #    proxy_set_header Connection 'upgrade';
+    #    proxy_set_header Host $host;
+    #    proxy_cache_bypass $http_upgrade;
+    #}
+
+    #location /logstash {
+    #    proxy_pass http://localhost:5044;
+    #    proxy_http_version 1.1;
+    #    proxy_set_header Upgrade $http_upgrade;
+    #    proxy_set_header Connection 'upgrade';
+    #    proxy_set_header Host $host;
+    #    proxy_cache_bypass $http_upgrade;
+    #}
+
+    error_page 404 /404.html;
+    location = /404.html {
+    }
+    error_page 500 502 503 504 /50x.html;
+    location = /50x.html {
+    }
+}

data/ansible/roles/mu-logstash/templates/nginx/elastic.conf.j2

@@ -0,0 +1,27 @@
+server {
+  listen 8080 ssl;
+  listen [::]:8080 ssl;
+
+  server_name {{ inventory_hostname }} {{ ec2['public_dns_name'] }} {{ ec2['private_dns_name'] }};
+  ssl_certificate /etc/ssl/certs/{{ inventory_hostname }}.crt;
+  ssl_certificate_key /etc/ssl/private/{{ inventory_hostname }}.key;
+#  ssl_dhparam /etc/ssl/certs/dhparam.pem;
+
+  location / {
+     proxy_pass {%- for node,meta in mu_deployment['servers']['backend'].items() %}
+  {%- for k,v in meta.items() %}
+    {%- if k in ["private_ip_address"] %} https://{{ v }}:9200;
+    {%- endif %}
+  {%- endfor %}
+  {%- if not loop.last %},{%- endif %}
+{%- endfor %}
+
+     #proxy_http_version 1.1;
+     #proxy_set_header Upgrade $http_upgrade;
+     #proxy_set_header Connection 'upgrade';
+     #proxy_set_header Host $host;
+     #proxy_cache_bypass $http_upgrade;
+     #proxy_ssl_trusted_certificate /etc/;
+     proxy_ssl_verify off;
+  }
+}

data/ansible/roles/mu-logstash/tests/inventory

@@ -0,0 +1,2 @@
+localhost
+

data/ansible/roles/mu-logstash/tests/test.yml

@@ -0,0 +1,5 @@
+---
+- hosts: localhost
+  remote_user: root
+  roles:
+    - mu-logstash

data/ansible/roles/mu-logstash/vars/main.yml

@@ -0,0 +1,2 @@
+---
+# vars file for mu-logstash

data/ansible/roles/mu-rdp/README.md

@@ -0,0 +1,33 @@
+Role Name
+=========
+
+Allow local-auth interactive remote logins to Windows nodes.
+
+Requirements
+------------
+
+Windows host with internet connectivity
+
+License
+-------
+
+Copyright:: Copyright (c) 2021 eGlobalTech, Inc., all rights reserved
+
+Licensed under the BSD-3 license (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License in the root of the project or at
+
+    http://egt-labs.com/mu/LICENSE.html
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Author Information
+------------------
+
+Current developers: John Stange
+
+egt-labs-admins@egt-labs.com

data/ansible/roles/mu-rdp/meta/main.yml

@@ -0,0 +1,53 @@
+galaxy_info:
+  author: your name
+  description: your description
+  company: your company (optional)
+
+  # If the issue tracker for your role is not on github, uncomment the
+  # next line and provide a value
+  # issue_tracker_url: http://example.com/issue/tracker
+
+  # Choose a valid license ID from https://spdx.org - some suggested licenses:
+  # - BSD-3-Clause (default)
+  # - MIT
+  # - GPL-2.0-or-later
+  # - GPL-3.0-only
+  # - Apache-2.0
+  # - CC-BY-4.0
+  license: license (GPL-2.0-or-later, MIT, etc)
+
+  min_ansible_version: 2.4
+
+  # If this a Container Enabled role, provide the minimum Ansible Container version.
+  # min_ansible_container_version:
+
+  #
+  # Provide a list of supported platforms, and for each platform a list of versions.
+  # If you don't wish to enumerate all versions for a particular platform, use 'all'.
+  # To view available platforms and versions (or releases), visit:
+  # https://galaxy.ansible.com/api/v1/platforms/
+  #
+  # platforms:
+  # - name: Fedora
+  #   versions:
+  #   - all
+  #   - 25
+  # - name: SomePlatform
+  #   versions:
+  #   - all
+  #   - 1.0
+  #   - 7
+  #   - 99.99
+
+  galaxy_tags: []
+    # List tags for your role here, one per line. A tag is a keyword that describes
+    # and categorizes the role. Users find roles by searching for tags. Be sure to
+    # remove the '[]' above, if you add tags to this list.
+    #
+    # NOTE: A tag is limited to a single word comprised of alphanumeric characters.
+    #       Maximum 20 tags per role.
+
+dependencies: []
+  # List your role dependencies here, one per line. Be sure to remove the '[]' above,
+  # if you add dependencies to this list.
+  

data/ansible/roles/mu-rdp/tasks/main.yml

@@ -0,0 +1,9 @@
+---
+- name: Allow traffic to port 3389
+  win_shell: Enable-NetFirewallRule -DisplayGroup "Remote Desktop"
+
+- name: Enable RDP
+  win_shell: Set-ItemProperty -Path 'HKLM:\System\CurrentControlSet\Control\Terminal Server' -name "fDenyTSConnections" -value 0
+
+- name: Allow RDP to use local user authentication
+  win_shell: Set-ItemProperty -Path 'HKLM:\System\CurrentControlSet\Control\Terminal Server\WinStations\RDP-Tcp' -name "UserAuthentication" -Value 0

data/ansible/roles/mu-rdp/tests/inventory

@@ -0,0 +1,2 @@
+localhost
+

data/ansible/roles/mu-rdp/tests/test.yml

@@ -0,0 +1,5 @@
+---
+- hosts: localhost
+  remote_user: root
+  roles:
+    - mu-rdp

data/ansible/roles/mu-windows/tasks/main.yml

@@ -34,3 +34,6 @@
 - name: "Tell EC2Launch to run on next boot (Windows 2016+)"
   when: ((ansible_facts['distribution_major_version'] | int) >= 10 and mu_build_image is defined and mu_build_image == True)
   win_shell: C:\ProgramData\Amazon\EC2-Windows\Launch\Scripts\InitializeInstance.ps1 -Schedule
+
+- name: Allow RDP to use local user authentication
+  win_shell: Set-ItemProperty -Path 'HKLM:\System\CurrentControlSet\Control\Terminal Server\WinStations\RDP-Tcp' -name "UserAuthentication" -Value 0

data/bin/mu-ansible-secret

@@ -56,7 +56,7 @@ if $opts[:string]
   else
     ARGV.shift
   end
-  MU::Groomer::Ansible.encryptString(namestr, $opts[:string])
+  puts MU::Groomer::Ansible.encryptString($opts[:string], namestr)
   exit
 end