cloud-mu 3.1.5 → 3.3.2

data/modules/mu/{clouds → providers}/google/vpc.rb

@@ -113,7 +113,7 @@ module MU
         # Describe this VPC
         # @return [Hash]
         def notify
-          base = MU.structToHash(cloud_desc)
+          base = MU.structToHash(cloud_desc, stringify_keys: true)
           base["cloud_id"] = @cloud_id
           base["project_id"] = habitat_id
           base.merge!(@config.to_h)
@@ -301,14 +301,10 @@ end
              @deploy.deployment["vpcs"][@config['name']]["subnets"] and
              @deploy.deployment["vpcs"][@config['name']]["subnets"].size > 0
             @deploy.deployment["vpcs"][@config['name']]["subnets"].each { |desc|
-              subnet = {}
-              subnet["ip_block"] = desc['ip_block']
-              subnet["name"] = desc["name"]
+              subnet = desc.clone
               subnet['mu_name'] = @config['scrub_mu_isms'] ? @cloud_id+subnet['name'].downcase : MU::Cloud::Google.nameStr(@deploy.getResourceName(subnet['name'], max_length: 61))
-              subnet["cloud_id"] = desc['cloud_id']
               subnet["cloud_id"] ||= desc['self_link'].gsub(/.*?\/([^\/]+)$/, '\1')
               subnet["cloud_id"] ||= subnet['mu_name']
-              subnet['az'] = desc["az"]
               subnet['az'] ||= desc["region"].gsub(/.*?\/([^\/]+)$/, '\1')
               @subnets << MU::Cloud::Google::VPC::Subnet.new(self, subnet, precache_description: false)
             }
@@ -541,16 +537,16 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
         # @param noop [Boolean]: If true, will only print what would be done
         # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
         # @return [void]
-        def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
-          flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
-          return if !MU::Cloud::Google::Habitat.isLive?(flags["project"], credentials)
+        def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {})
+          flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials)
+          return if !MU::Cloud.resourceClass("Google", "Habitat").isLive?(flags["habitat"], credentials)
           filter = %Q{(labels.mu-id = "#{MU.deploy_id.downcase}")}
           if !ignoremaster and MU.mu_public_ip
             filter += %Q{ AND (labels.mu-master-ip = "#{MU.mu_public_ip.gsub(/\./, "_")}")}
           end
           MU.log "Placeholder: Google VPC artifacts do not support labels, so ignoremaster cleanup flag has no effect", MU::DEBUG, details: filter
 
-          purge_subnets(noop, project: flags['project'], credentials: credentials)
+          purge_subnets(noop, project: flags['habitat'], credentials: credentials)
           ["route", "network"].each { |type|
 # XXX tagged routes aren't showing up in list, and the networks that own them
 # fail to delete silently
@@ -559,7 +555,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
             begin
               MU::Cloud::Google.compute(credentials: credentials).delete(
                 type,
-                flags["project"],
+                flags["habitat"],
                 nil,
                 noop
               )
@@ -569,13 +565,13 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
                   MU.log e.message, MU::WARN
                   if e.message.match(/Failed to delete network (.+)/)
                     network_name = Regexp.last_match[1]
-                    fwrules = MU::Cloud::Google::FirewallRule.find(project: flags['project'], credentials: credentials)
+                    fwrules = MU::Cloud.resourceClass("Google", "FirewallRule").find(project: flags['habitat'], credentials: credentials)
                     fwrules.reject! { |_name, desc|
                       !desc.network.match(/.*?\/#{Regexp.quote(network_name)}$/)
                     }
                     fwrules.keys.each { |name|
                       MU.log "Attempting to delete firewall rule #{name} so that VPC #{network_name} can be removed", MU::NOTICE
-                      MU::Cloud::Google.compute(credentials: credentials).delete_firewall(flags['project'], name)
+                      MU::Cloud::Google.compute(credentials: credentials).delete_firewall(flags['habitat'], name)
                     }
                   end
                 end
@@ -950,6 +946,41 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
           createRoute(route, network: @url, tags: [MU::Cloud::Google.nameStr(server.mu_name)])
         end
 
+        # Looks at existing subnets, and attempts to find the next available
+        # IP block that's roughly similar to the ones we already have. This
+        # checks against secondary IP ranges, as well as each subnet's primary
+        # CIDR block.
+        # @param exclude [Array<String>]: One or more CIDRs to treat as unavailable, in addition to those allocated to existing subnets
+        # @return [String]
+        def getUnusedAddressBlock(exclude: [], max_bits: 28)
+          used_ranges = exclude.map { |cidr| NetAddr::IPv4Net.parse(cidr) }
+          subnets.each { |s|
+            used_ranges << NetAddr::IPv4Net.parse(s.cloud_desc.ip_cidr_range)
+            if s.cloud_desc.secondary_ip_ranges
+              used_ranges.concat(s.cloud_desc.secondary_ip_ranges.map { |r| NetAddr::IPv4Net.parse(r.ip_cidr_range) })
+            end
+          }
+# XXX sort used_ranges
+          candidate = used_ranges.first.next_sib
+
+          begin
+            if candidate.netmask.prefix_len > max_bits
+              candidate = candidate.resize(max_bits)
+            end
+            try_again = false
+            used_ranges.each { |cidr|
+              if !cidr.rel(candidate).nil?
+                candidate = candidate.next_sib
+                try_again = true
+                break
+              end
+            }
+            try_again = false if candidate.nil?
+          end while try_again
+
+          candidate.to_s
+        end
+
         private
 
         def self.genStandardSubnetACLs(vpc_cidr, vpc_name, configurator, project, _publicroute = true, credentials: nil)
@@ -1120,7 +1151,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
           # Describe this VPC Subnet
           # @return [Hash]
           def notify
-            MU.structToHash(cloud_desc)
+            MU.structToHash(cloud_desc, stringify_keys: true)
           end
 
           # Return the +self_link+ to this subnet

data/modules/tests/aws-jobs-functions.yaml

@@ -0,0 +1,46 @@
+# clouds: AWS
+---
+appname: smoketest
+jobs:
+- name: event1
+  schedule:
+    minute: '0'
+    hour: '1'
+    day_of_month: '1'
+    month: "*"
+    day_of_week: "?"
+    year: "*"
+  targets:
+  - type: functions
+    name: python-function
+- name: event2
+  disabled: true
+  schedule:
+    minute: '0'
+    hour: '2'
+    day_of_month: '1'
+    month: "*"
+    day_of_week: "?"
+    year: "*"
+  targets:
+  - type: functions
+    name: node-function
+
+functions:
+- name: python-function
+  handler: lambda_function.lambda_handler
+  memory: 128
+  runtime: python3.6
+  timeout: 300
+  code:
+    path: functions/python-function
+  environment_variable:
+  - key: foo
+    value: bar
+- name: node-function
+  runtime: nodejs12.x
+  handler: lambda_function.lambda_handler
+  memory: 256
+  timeout: 60
+  code:
+    path: functions/node-function

data/modules/tests/centos6.yaml

@@ -0,0 +1,15 @@
+# groomers: Chef
+---
+appname: smoketest
+vpcs:
+- name: wrapper
+servers: 
+- name: centos6
+  vpc:
+    name: wrapper
+  platform: centos6
+  size: m3.medium
+  run_list:
+  - recipe[mu-tools::apply_security]
+  - recipe[mu-tools::updates]
+  - recipe[mu-tools::split_var_partitions]

data/modules/tests/centos7.yaml

@@ -0,0 +1,15 @@
+# groomers: Chef
+---
+appname: smoketest
+vpcs:
+- name: wrapper
+servers: 
+- name: centos7
+  platform: centos7
+  vpc:
+    name: wrapper
+  size: m3.medium
+  run_list:
+  - recipe[mu-tools::apply_security]
+  - recipe[mu-tools::updates]
+  - recipe[mu-tools::split_var_partitions]

data/modules/tests/centos8.yaml

@@ -0,0 +1,12 @@
+# groomers: Chef
+# clouds: Azure, Google
+---
+appname: smoketest
+servers: 
+- name: centos8
+  platform: centos8
+  size: m3.medium
+  run_list:
+  - recipe[mu-tools::apply_security]
+  - recipe[mu-tools::updates]
+  - recipe[mu-tools::split_var_partitions]

data/modules/tests/ecs.yaml

@@ -7,7 +7,7 @@ vpcs:
 container_clusters:
 - name: ecsplain
   flavor: ECS
-  instance_type: t2.medium
+  instance_type: t3.medium
   vpc:
     name: ecs
   containers:
@@ -15,7 +15,7 @@ container_clusters:
     image: "nginx:1.8"
 - name: ecsfargate
   flavor: Fargate
-  instance_type: t2.medium
+  instance_type: t3.medium
   vpc:
     name: ecs
   containers:

data/modules/tests/eks.yaml

@@ -10,7 +10,7 @@ container_clusters:
   vpc:
     vpc_name: eksvpc
   instance_count: 3
-  instance_type: t2.medium
+  instance_type: t3.medium
   kubernetes_resources:
   - apiVersion: apps/v1
     kind: Deployment

data/modules/tests/functions/node-function/lambda_function.js

@@ -0,0 +1,10 @@
+console.log('Loading function');
+
+exports.handler = async (event, context) => {
+    //console.log('Received event:', JSON.stringify(event, null, 2));
+    console.log('value1 =', event.key1);
+    console.log('value2 =', event.key2);
+    console.log('value3 =', event.key3);
+    return event.key1;  // Echo back the first key value
+    // throw new Error('Something went wrong');
+};

data/modules/tests/functions/python-function/lambda_function.py

@@ -0,0 +1,12 @@
+import json
+
+print('Loading function')
+
+
+def lambda_handler(event, context):
+    #print("Received event: " + json.dumps(event, indent=2))
+    print("value1 = " + event['key1'])
+    print("value2 = " + event['key2'])
+    print("value3 = " + event['key3'])
+    return event['key1']  # Echo back the first key value
+    #raise Exception('Something went wrong')

data/modules/tests/microservice_app.yaml

@@ -0,0 +1,288 @@
+# Old Sitemonitor, with serial numbers and code filed off. This will *only*
+# work on our own Labs sandbox, unless you feed it a different domain name to
+# play in.
+# clouds: AWS
+---
+appname: SMOKETEST
+parameters:
+- name: domain
+  default: "sandbox.egt-labs.com" # this must exist as a Route53 zone and have a corresponding wildcard ACM or IAM SSL certificate
+jobs:
+- name: clear-scan-data
+  schedule:
+    minute: '0'
+    hour: '1'
+    day_of_month: '*'
+    month: "*"
+    day_of_week: "?"
+    year: "*"
+  targets:
+  - type: functions
+    name: empty-out-table
+- name: run-scans
+  schedule:
+    minute: '0'
+    hour: '2'
+    day_of_month: '*'
+    month: "*"
+    day_of_week: "?"
+    year: "*"
+  targets:
+  - type: functions
+    name: queue-domains
+
+cdns:
+- name: front
+  origins:
+  - name: default
+    bucket:
+      name: bucket
+  certificate:
+    name: "*.<%= domain %>"
+  dns_records:
+  - zone:
+      name: <%= domain %>
+  behaviors:
+  - origin: default
+    forwarded_values:
+      headers:
+      - Origin
+      - Access-Control-Request-Headers
+      - Access-Control-Request-Method
+      - Access-Control-Allow-Origin
+
+roles:
+- name: dynamostream-to-es
+  can_assume:
+  - assume_method: basic
+    entity_type: service
+    entity_id: lambda.amazonaws.com
+  attachable_policies:
+  - id: AWSLambdaInvocation-DynamoDB
+  - id: AWSLambdaBasicExecutionRole
+  policies:
+  - name: allow_es_posting
+    permissions:
+    - es:ESHttpPost
+    targets:
+    - identifier: domains-scan-data
+      type: search_domain
+      path: "/*"
+- name: empty-out-table
+  can_assume:
+  - assume_method: basic
+    entity_type: service
+    entity_id: lambda.amazonaws.com
+  attachable_policies:
+  - id: AmazonDynamoDBFullAccess
+  - id: AWSLambdaBasicExecutionRole
+- name: on-demand-scanner
+  can_assume:
+  - assume_method: basic
+    entity_type: service
+    entity_id: lambda.amazonaws.com
+  attachable_policies:
+  - id: AmazonDynamoDBFullAccess
+  - id: AWSLambdaBasicExecutionRole
+- name: queue-domains
+  can_assume:
+  - assume_method: basic
+    entity_type: service
+    entity_id: lambda.amazonaws.com
+  attachable_policies:
+  - id: AmazonDynamoDBFullAccess
+  - id: AmazonSNSFullAccess
+  - id: AWSLambdaBasicExecutionRole
+- name: scheduled-scanner
+  can_assume:
+  - assume_method: basic
+    entity_type: service
+    entity_id: lambda.amazonaws.com
+  attachable_policies:
+  - id: AmazonDynamoDBFullAccess
+  - id: AWSLambdaBasicExecutionRole
+
+notifiers:
+- name: publish-domains
+  subscriptions:
+  - type: lambda
+    resource:
+      type: functions
+      name: scheduled-scanner
+
+functions:
+- name: dynamostream-to-es
+  handler: lambda_function.lambda_handler
+  memory: 128
+  runtime: python2.7
+  timeout: 900
+  code:
+    path: functions/python-function
+  role:
+    name: dynamostream-to-es
+    type: roles
+  triggers:
+  - service: dynamodb
+    name: scan-data
+  dependencies:
+  - type: search_domain
+    name: domains-scan-data
+    phase: groom
+- name: empty-out-table
+  handler: lambda_function.lambda_handler
+  memory: 128
+  runtime: python3.6
+  timeout: 300
+  code:
+    path: functions/python-function
+  environment_variable:
+  - key: table
+    value: scandata
+  role:
+    name: empty-out-table
+    type: roles
+  dependencies:
+  - type: nosqldb
+    name: scan-data
+  - type: nosqldb
+    name: domain-list
+- name: on-demand-scanner
+  handler: lambda_function.lambda_handler
+  memory: 128
+  runtime: python3.6
+  timeout: 900
+  code:
+    path: functions/python-function
+  role:
+    name: on-demand-scanner
+    type: roles
+  dependencies:
+  - type: nosqldb
+    name: scan-data
+  triggers:
+  - service: apigateway
+    name: api
+- name: queue-domains
+  handler: lambda_function.lambda_handler
+  memory: 128
+  runtime: python3.6
+  timeout: 900
+  code:
+    path: functions/python-function
+  role:
+    name: queue-domains
+    type: roles
+  invoke_on_completion:
+    invocation_type: "RequestResponse"
+  permissions:
+  - basic
+  - dynamo
+  dependencies:
+  - type: function
+    name: dynamostream-to-es
+  - type: nosqldb
+    name: domain-list
+  - type: nosqldb
+    name: scan-data
+  - type: notifier
+    name: publish-domains
+    phase: groom
+- name: scheduled-scanner
+  handler: lambda_function.lambda_handler
+  memory: 256
+  runtime: python3.6
+  timeout: 900
+  code:
+    path: functions/python-function
+  role:
+    name: scheduled-scanner
+    type: roles
+  dependencies:
+  - type: nosqldb
+    name: scan-data
+  triggers:
+  - service: sns
+    name: publish-domains
+
+endpoints:
+- name: api
+  deploy_to: production
+  log_requests: true
+  methods:
+  - path: "/"
+    type: POST
+    cors: "*"
+    responses:
+    - code: 200
+      body:
+      - is_error: false
+        content_type: application/json
+    integrate_with:
+      name: on-demand-scanner
+      type: functions
+      integration_http_method: POST
+      async: true
+      backend_http_method: POST
+      passthrough_behavior: WHEN_NO_MATCH
+  domain_names:
+  - dns_record:
+      zone:
+        name: <%= domain %>
+    certificate:
+      name: "*.<%= domain %>"
+
+buckets:
+- name: bucket
+  web: false
+  cors:
+  - allowed_methods:
+    - GET
+    - POST
+    allowed_origins:
+    - "*"
+  upload:
+#  - source: "code/build"
+  - source: "functions"
+    destination: "/"
+
+search_domains:
+- name: domains-scan-data
+  elasticsearch_version: '7.4'
+  instance_count: 1
+  instance_type: r5.large.elasticsearch
+  ebs_size: 10
+  ebs_type: gp2
+  access_policies:
+    Version: '2012-10-17'
+    Statement:
+    - Effect: Allow
+      Principal:
+        AWS: "*"
+      Action: es:ESHttp*
+nosqldbs:
+- name: scan-data
+  read_capacity: 25
+  write_capacity: 25
+  attributes:
+  - name: domain
+    type: S
+    primary_partition: true
+  - name: last_scanned_date
+    type: S
+    primary_sort: true
+  stream: NEW_IMAGE
+- name: domain-list
+  read_capacity: 100
+  write_capacity: 1
+  attributes:
+  - name: business_owner
+    type: S
+    primary_sort: true
+  - name: domain
+    type: S
+    primary_partition: true
+  populate:
+  - business_owner: TetraTech
+    staff_division: eGT
+    operational_division: eGTLabs
+    domain: egt-labs.com