cfn-vpn 1.3.4 → 1.4.0

data/lib/cfnvpn/templates/lambdas/auto_route_populator/app.py

@@ -1,26 +1,41 @@
+import os
 import socket
 import boto3
 from botocore.exceptions import ClientError
+from lib.slack import Slack
+from states import *
 import logging
+from quotas import increase_quota, AUTH_RULE_TABLE_QUOTA_CODE, ROUTE_TABLE_QUOTA_CODE
 
-logger = logging.getLogger()
+logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
 
+SLACK_USERNAME = 'CfnVpn Route Table Event'
 
 def delete_route(client, vpn_endpoint, subnet, cidr):
+  try:
     client.delete_client_vpn_route(
       ClientVpnEndpointId=vpn_endpoint,
       TargetVpcSubnetId=subnet,
       DestinationCidrBlock=cidr,
     )
+  except ClientError as e:
+    if e.response['Error']['Code'] == 'InvalidClientVpnEndpointAuthorizationRuleNotFound':
+      logger.info(f"route not found when deleting", exc_info=True)
+    else:
+      raise e
 
   
-def create_route(client, event, cidr):
+def create_route(client, event, cidr, target_subnet):
+  description = f"cfnvpn auto generated route for endpoint {event['Record']}."
+  if event['Description']:
+    description += f" {event['Description']}"
+
   client.create_client_vpn_route(
     ClientVpnEndpointId=event['ClientVpnEndpointId'],
     DestinationCidrBlock=cidr,
-    TargetVpcSubnetId=event['TargetSubnet'],
-    Description=f"cfnvpn auto generated route for endpoint {event['Record']}. {event['Description']}"
+    TargetVpcSubnetId=target_subnet,
+    Description=description
   )
 
 
@@ -34,15 +49,27 @@ def revoke_route_auth(client, event, cidr, group = None):
     args['RevokeAllGroups'] = True
   else:
     args['AccessGroupId'] = group
-    
-  client.revoke_client_vpn_ingress(**args)
+  
+  try:
+    client.revoke_client_vpn_ingress(**args)
+  except ClientError as e:
+    if e.response['Error']['Code'] == 'ConcurrentMutationLimitExceeded':
+      logger.warn(f"revoking auth is being rate limited", exc_info=True)
+    elif e.response['Error']['Code'] == 'InvalidClientVpnEndpointAuthorizationRuleNotFound':
+      logger.info(f"rule not found when revoking", exc_info=True)
+    else:
+      raise e
 
 
 def authorize_route(client, event, cidr, group = None):
+  description = f"cfnvpn auto generated authorization for endpoint {event['Record']}."
+  if event['Description']:
+    description += f" {event['Description']}"
+
   args = {
     'ClientVpnEndpointId': event['ClientVpnEndpointId'],
     'TargetNetworkCidr': cidr,
-    'Description': f"cfnvpn auto generated authorization for endpoint {event['Record']}. {event['Description']}"
+    'Description': description
   }
   
   if group is None:
@@ -54,7 +81,8 @@ def authorize_route(client, event, cidr, group = None):
 
 
 def get_routes(client, event):
-  response = client.describe_client_vpn_routes(
+  paginator = client.get_paginator('describe_client_vpn_routes')
+  response_iterator = paginator.paginate(
     ClientVpnEndpointId=event['ClientVpnEndpointId'],
     Filters=[
       {
@@ -63,113 +91,170 @@ def get_routes(client, event):
       }
     ]
   )
-  
-  routes = [route for route in response['Routes'] if event['Record'] in route['Description']]
-  logger.info(f"found {len(routes)} exisiting routes for {event['Record']}")
-  return routes
+ 
+  return [route for page in response_iterator 
+            for route in page['Routes'] 
+            if event['Record'] in route['Description']]
 
 
-def get_rules(client, vpn_endpoint, cidr):
-  response = client.describe_client_vpn_authorization_rules(
-    ClientVpnEndpointId=vpn_endpoint,
-    Filters=[
-        {
-            'Name': 'destination-cidr',
-            'Values': [cidr]
-        }
-    ]
+def get_auth_rules(client, event):
+  paginator = client.get_paginator('describe_client_vpn_authorization_rules')
+  response_iterator = paginator.paginate(
+    ClientVpnEndpointId=event['ClientVpnEndpointId']
   )
-  return response['AuthorizationRules']
+
+  return [rule for page in response_iterator 
+            for rule in page['AuthorizationRules']
+            if event['Record'] in rule['Description']]
+
+
+def expired_auth_rules(auth_rules, cidrs, groups):
+  for rule in auth_rules:
+    # if there is a rule for the record with an old cidr
+    if rule['DestinationCidr'] not in cidrs:
+      yield rule
+    # if there is a rule for a group that is no longer in the event
+    if groups and rule['GroupId'] not in groups:
+      yield rule
+    # if there is a rule for allow all but groups are in the event
+    if groups and rule['AccessAll']:
+      yield rule
+
+
+def expired_routes(routes, cidrs):
+  for route in routes:
+    if route['DestinationCidr'] not in cidrs:
+      yield route
 
 
 def handler(event,context):
+
+  logger.info(f"auto route populator triggered with event : {event}")
+  slack = Slack(username=SLACK_USERNAME)
   
   # DNS lookup on the dns record and return all IPS for the endpoint
   try:
     cidrs = [ ip + "/32" for ip in socket.gethostbyname_ex(event['Record'])[-1]]
     logger.info(f"resolved endpoint {event['Record']} to {cidrs}")
   except socket.gaierror as e:
-    logger.exception(f"failed to resolve record {event['Record']}")
+    logger.error(f"failed to resolve record {event['Record']}", exc_info=True)
+    slack.post_event(message=f"failed to resolve record {event['Record']}", state=RESOLVE_FAILED, error=e)
     return 'KO'
   
   client = boto3.client('ec2')
+
+  # describe vpn and check if subnets are associated with the vpn
+  response = client.describe_client_vpn_endpoints(
+    ClientVpnEndpointIds=[event['ClientVpnEndpointId']]
+  )
+
+  if not response['ClientVpnEndpoints']:
+    logger.error(f"endpoint not found")
+    slack.post_event(message=f"failed create routes for {event['Record']}", state=FAILED, error="endpoint not found")
+    return 'KO'
+
+  endpoint = response['ClientVpnEndpoints'][0]
+  if endpoint['Status'] == 'pending-associate':
+    logger.error(f"no subnets associated with endpoint")
+    slack.post_event(message=f"failed create routes for {event['Record']}", state=FAILED, error="vpn is in a stopped state")
+    return 'KO'
+
   routes = get_routes(client, event)
+  auth_rules = get_auth_rules(client, event)
+
+  auto_limit_increase = os.environ.get('AUTO_LIMIT_INCREASE')
+  route_limit_increase_required = False
+  auth_rules_limit_increase_required = False
 
   for cidr in cidrs:
-    route = next((route for route in routes if route['DestinationCidr'] == cidr), None)
-    
-    # if there are no existing routes for the endpoint cidr create a new route
-    if route is None:
-      try:
-        create_route(client, event, cidr)
-        if 'Groups' in event:
-          for group in event['Groups']:
-            authorize_route(client, event, cidr, group)
+    # create route if doesn't exist
+    for subnet in event['TargetSubnets']:
+      if not any(route['DestinationCidr'] == cidr and route['TargetSubnet'] == subnet for route in routes):
+        try:
+          create_route(client, event, cidr, subnet)
+        except ClientError as e:
+          if e.response['Error']['Code'] == 'ClientVpnRouteLimitExceeded':
+            route_limit_increase_required = True
+            logger.error("vpn route table has reached the route limit", exc_info=True)
+            slack.post_event(
+              message=f"unable to create route {cidr} from {event['Record']}",
+              state=ROUTE_LIMIT_EXCEEDED,
+              error="vpn route table has reached the route limit"
+            )
+          elif e.response['Error']['Code'] == 'InvalidClientVpnActiveAssociationNotFound':
+            logger.warn("no subnets are associated with the vpn", exc_info=True)
+            slack.post_event(
+              message=f"unable to create the route {cidr} from {event['Record']}", 
+              state=SUBNET_NOT_ASSOCIATED,
+              error="no subnets are associated with the vpn"
+            )
+          else:
+            logger.error("encountered a unexpected client error when creating a route", exc_info=True)
         else:
-          authorize_route(client, event, cidr)
-      except ClientError as e:
-        if e.response['Error']['Code'] == 'InvalidClientVpnDuplicateRoute':
-          logger.error(f"route for CIDR {cidr} already exists with a different endpoint")
-          continue
-        raise e
-        
-    # if the route already exists
-    else:
-      
-      logger.info(f"route for cidr {cidr} is already in place")
-      
-      # if the target subnet has changed in the payload, recreate the routes to use the new subnet
-      if route['TargetSubnet'] != event['TargetSubnet']:
-        logger.info(f"target subnet for route for {cidr} has changed, recreating the route")
+          slack.post_event(
+            message=f"created new route {cidr} ({event['Record']}) to target subnet {subnet}",
+            state=NEW_ROUTE
+          )
+    
+    # remove route if target subnet has changed
+    for route in routes:
+      if route['DestinationCidr'] == cidr and route['TargetSubnet'] not in event['TargetSubnets']:
         delete_route(client, event['ClientVpnEndpointId'], route['TargetSubnet'], cidr)
-        create_route(client, event, cidr)
-      
-      logger.info(f"checking authorization rules for the route")
-      
-      # check the rules match the payload
-      rules = get_rules(client, event['ClientVpnEndpointId'], cidr)
-      existing_groups = [rule['GroupId'] for rule in rules]
+
+    # collect all rules that matches the current cidr
+    cidr_auth_rules = [rule for rule in auth_rules if rule['DestinationCidr'] == cidr]
+
+    try:
+      # create rules for newly added groups
       if 'Groups' in event:
-        # remove expired rules not defined in the payload anymore
-        expired_rules = [rule for rule in rules if rule['GroupId'] not in event['Groups']]
-        for rule in expired_rules:
-          logger.info(f"removing expired authorization rule for group {rule['GroupId']} for route {cidr}")
-          revoke_route_auth(client, event, cidr, rule['GroupId'])
-        # add new rules defined in the payload
-        new_rules =  [group for group in event['Groups'] if group not in existing_groups]
-        for group in new_rules:
-          logger.info(f"creating new authorization rule for group {rule['GroupId']} for route {cidr}")
+        existing_groups = list(set(rule['GroupId'] for rule in cidr_auth_rules))
+        new_groups = [group for group in event['Groups'] if group not in existing_groups]
+
+        for group in new_groups:
           authorize_route(client, event, cidr, group)
-      else:
-        # if amount of rules for the cidr is greater than 1 when no groups are specified in the payload 
-        # we'll assume that all groups have been removed from the payload so we'll remove all existing rules and add a rule for allow all 
-        if len(rules) > 1:
-          logger.info(f"creating an allow all rule for route {cidr}")
-          revoke_route_auth(client, event, cidr)
-          authorize_route(client, event, cidr)
-          
-      
 
-  
-  # clean up any expired routes when the ips for an endpoint change
-  expired_routes = [route for route in routes if route['DestinationCidr'] not in cidrs]
-  for route in expired_routes:
-    logger.info(f"removing expired route {route['DestinationCidr']} for endpoint {event['Record']}")
-    
-    try:
-      revoke_route_auth(client, event, route['DestinationCidr'])
-    except ClientError as e:
-      if e.response['Error']['Code'] == 'InvalidClientVpnEndpointAuthorizationRuleNotFound':
-        pass
-      else:
-        raise e
-                          
-    try:
-      delete_route(client, event['ClientVpnEndpointId'], route['TargetSubnet'], route['DestinationCidr'])
+      # create an allow all rule
+      elif 'Groups' not in event and not cidr_auth_rules:
+        authorize_route(client, event, cidr)
+
     except ClientError as e:
-      if e.response['Error']['Code'] == 'InvalidClientVpnRouteNotFound':
-        pass
-      else:
-        raise e
+        if e.response['Error']['Code'] == 'ClientVpnAuthorizationRuleLimitExceeded':
+          auth_rules_limit_increase_required = True
+          logger.error("vpn has reached the authorization rule limit", exc_info=True)
+          slack.post_event(
+            message=f"unable add to authorization rule for route {cidr} from {event['Record']}",
+            state=AUTH_RULE_LIMIT_EXCEEDED,
+            error="vpn has reached the authorization rule limit"
+          )
+          continue
+        else:
+            logger.error("encountered a unexpected client error when creating an auth rule", exc_info=True)
+
+  # request route limit increase
+  if route_limit_increase_required and auto_limit_increase:
+    case_id = increase_quota(10, ROUTE_TABLE_QUOTA_CODE, event['ClientVpnEndpointId'])
+    if case_id is not None:
+      slack.post_event(message=f"requested an increase for the routes per vpn service quota", state=QUOTA_INCREASE_REQUEST, support_case=case_id)
+    else:
+      logger.info(f"routes per vpn service quota increase request pending")
+
+  # request auth rule limit increase
+  if auth_rules_limit_increase_required and auto_limit_increase:
+    case_id = increase_quota(20, AUTH_RULE_TABLE_QUOTA_CODE, event['ClientVpnEndpointId'])
+    if case_id is not None:
+      slack.post_event(message=f"requested an increase for the authorization rules per vpn service quota", state=QUOTA_INCREASE_REQUEST, support_case=case_id)
+    else:
+      logger.info(f"authorization rules per vpn service quota increase request pending")
+
+  # remove expired auth rules
+  for rule in expired_auth_rules(auth_rules, cidrs, event.get('Groups', [])):
+    logger.info(f"removing expired auth rule {rule['DestinationCidr']} for endpoint {event['Record']}")
+    revoke_route_auth(client, event, route['DestinationCidr'])
+
+  # remove expired routes
+  for route in expired_routes(routes, cidrs):
+    logger.info(f"removing expired route {route['DestinationCidr']} for endpoint {event['Record']}")
+    delete_route(client, event['ClientVpnEndpointId'], route['TargetSubnet'], route['DestinationCidr'])
+    slack.post_event(message=f"removed expired route {route['DestinationCidr']} for endpoint {event['Record']}", state=EXPIRED_ROUTE)
   
   return 'OK'

data/lib/cfnvpn/templates/lambdas/auto_route_populator/quotas.py

@@ -0,0 +1,37 @@
+import boto3
+
+ROUTE_TABLE_QUOTA_CODE = 'L-401D78F7'
+AUTH_RULE_TABLE_QUOTA_CODE = 'L-9A1BC94B'
+EC2_SERVICE_CODE = 'ec2'
+IN_PROGRESS = ['PENDING', 'CASE_OPENED']
+
+def get_route_count(endpoint) -> int:
+  client = boto3.client('ec2')
+  response = client.describe_client_vpn_routes(
+    ClientVpnEndpointId=endpoint,
+  )
+  return len(response['Routes'])
+
+def quota_request_open(quota_code) -> bool:
+  client = boto3.client('service-quotas')
+  response = client.list_requested_service_quota_change_history_by_quota(
+    ServiceCode=EC2_SERVICE_CODE,
+    QuotaCode=quota_code
+  )
+  # Status='PENDING'|'CASE_OPENED'|'APPROVED'|'DENIED'|'CASE_CLOSED'
+  return any(req['status'] in IN_PROGRESS for req in response['RequestedQuotas'])
+
+def increase_quota(increase_value, quota_code, endpoint) -> str:
+  if quota_request_open(quota_code):
+    return None
+
+  current_route_count = get_route_count(endpoint)
+  desired_value = current_route_count + increase_value
+
+  client = boto3.client('service-quotas')
+  response = client.request_service_quota_increase(
+    ServiceCode=EC2_SERVICE_CODE,
+    QuotaCode=quota_code,
+    DesiredValue=desired_value
+  )
+  return response['CaseId']

data/lib/cfnvpn/templates/lambdas/auto_route_populator/states.py

@@ -0,0 +1,21 @@
+"""
+states:
+
+FAILED: general failure
+NEW_ROUTE: new route added to route table
+EXPIRED_ROUTE: cidr is no longer associated with DNS entry and is removed from the route table
+ROUTE_LIMIT_EXCEEDED: no new routes can be added to the route table due to aws route table limit
+AUTH_RULE_LIMIT_EXCEEDED: no new authorization rules can be added to the rule list due to aws auth rule limit
+RESOLVE_FAILED: failed to resolve the provided dns entry
+SUBNET_NOT_ASSOCIATED: no subnets are associated with the client vpn
+QUOTA_INCREASE_REQUEST: automatic quota increase made
+"""
+
+FAILED = 'FAILED'
+NEW_ROUTE = 'NEW_ROUTE'
+EXPIRED_ROUTE = 'EXPIRED_ROUTE'
+ROUTE_LIMIT_EXCEEDED = 'ROUTE_LIMIT_EXCEEDED'
+AUTH_RULE_LIMIT_EXCEEDED = 'AUTH_RULE_LIMIT_EXCEEDED'
+RESOLVE_FAILED = 'RESOLVE_FAILED'
+SUBNET_NOT_ASSOCIATED = 'SUBNET_NOT_ASSOCIATED'
+QUOTA_INCREASE_REQUEST = 'QUOTA_INCREASE_REQUEST'

data/lib/cfnvpn/templates/lambdas/lib/slack.py

@@ -0,0 +1,66 @@
+import os
+import json
+import logging
+import urllib
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+
+class Slack:
+
+    def __init__(self, username):
+        self.username = username
+        self.slack_url = os.environ.get('SLACK_URL')
+
+    def post_event(self, message, state, error=None, support_case=None):
+        """Posts event to slack using an incoming webhook
+        Parameters
+        ----------
+        message: str
+            message to post to slack
+        state: str
+            the state of the event
+        error: str
+            error message to add to the message
+        support_case: str
+            displays a aws console link to the support case in the message
+        """
+
+        if not self.slack_url.startswith('https://hooks.slack.com'):
+            return
+
+        if 'FAILED' in state or 'LIMIT_EXCEEDED' in state:
+            colour = '#ad0614'
+        elif 'NOT_ASSOCIATED' in state:
+            colour = '#d4b126'
+        else: 
+            colour = '#3ead3e'
+
+        text = f'Message: {message}\nState: {state}'
+        
+        if error:
+            text += f'\nError: {error}'
+        
+        if support_case:
+            text += f'\nSupport Case: <https://console.aws.amazon.com/support/cases#/{support_case}|{support_case}>'
+
+        payload = {
+            'username': self.username,
+            'attachments': [
+                {
+                    'color': colour,
+                    'text': text,
+                    'mrkdwn_in': ['text','pretext']
+                }
+            ]
+        }
+
+        try:
+            urllib.request.urlopen(urllib.request.Request(
+                self.slack_url,
+                headers={'Content-Type': 'application/json'},
+                data=json.dumps(payload).encode('utf-8')
+            ))
+        except urllib.error.HTTPError as e:
+            logger.error(f"failed to post slack notification. REASON: {e.reason} CODE: {e.code}", exc_info=True)
+

data/lib/cfnvpn/templates/lambdas/scheduler/app.py

@@ -1,36 +1,54 @@
 import boto3
 import logging
+from lib.slack import Slack
+from states import *
 
 logger = logging.getLogger()
 logger.setLevel(logging.INFO)
 
+SLACK_USERNAME = 'CfnVpn Scheduler'
+
 def handler(event, context):
 
   logger.info(f"updating cfn-vpn stack {event['StackName']} parameter AssociateSubnets with value {event['AssociateSubnets']}")
+  slack = Slack(username=SLACK_USERNAME)
+
+  try:
+    if event['AssociateSubnets'] == 'false':
+      logger.info(f"terminating current vpn sessions to {event['ClientVpnEndpointId']}")
+      ec2 = boto3.client('ec2')
+      resp = ec2.describe_client_vpn_connections(ClientVpnEndpointId=event['ClientVpnEndpointId'])
+      for conn in resp['Connections']:
+        if conn['Status']['Code'] == 'active':
+          ec2.terminate_client_vpn_connections(
+            ClientVpnEndpointId=event['ClientVpnEndpointId'],
+            ConnectionId=conn['ConnectionId']
+          )
+          logger.info(f"terminated session {conn['ConnectionId']}")
+
+    client = boto3.client('cloudformation')
+    logger.info(client.update_stack(
+      StackName=event['StackName'],
+      UsePreviousTemplate=True,
+      Capabilities=['CAPABILITY_IAM'],
+      Parameters=[
+        {
+          'ParameterKey': 'AssociateSubnets',
+          'ParameterValue': event['AssociateSubnets']
+        }
+      ]
+    ))
+  except Exception as ex:
+    logger.error(f"failed to start/stop client vpn", exc_info=True)
+    if event['AssociateSubnets'] == 'true':
+      slack.post_event(message=f"failed to associate subnets with the client vpn", state=START_FAILED, error=ex)
+    else:
+      slack.post_event(message=f"failed to disassociate subnets with the client vpn", state=STOP_FAILED, error=ex)
+    return 'KO'
 
-  if event['AssociateSubnets'] == 'false':
-    logger.info(f"terminating current vpn sessions to {event['ClientVpnEndpointId']}")
-    ec2 = boto3.client('ec2')
-    resp = ec2.describe_client_vpn_connections(ClientVpnEndpointId=event['ClientVpnEndpointId'])
-    for conn in resp['Connections']:
-      if conn['Status']['Code'] == 'active':
-        ec2.terminate_client_vpn_connections(
-          ClientVpnEndpointId=event['ClientVpnEndpointId'],
-          ConnectionId=conn['ConnectionId']
-        )
-        logger.info(f"terminated session {conn['ConnectionId']}")
-
-  client = boto3.client('cloudformation')
-  logger.info(client.update_stack(
-    StackName=event['StackName'],
-    UsePreviousTemplate=True,
-    Capabilities=['CAPABILITY_IAM'],
-    Parameters=[
-      {
-        'ParameterKey': 'AssociateSubnets',
-        'ParameterValue': event['AssociateSubnets']
-      }
-    ]
-  ))
+  if event['AssociateSubnets'] == 'true':
+    slack.post_event(message=f"successfully associated subnets with the client vpn", state=START_IN_PROGRESS)
+  else:
+    slack.post_event(message=f"successfully disassociated subnets with the client vpn", state=STOP_IN_PROGRESS)
 
   return 'OK'

data/lib/cfnvpn/templates/lambdas/scheduler/states.py

@@ -0,0 +1,13 @@
+"""
+states:
+
+START_IN_PROGRESS: associating subnets with the Client VPN
+STOP_IN_PROGRESS: disassociating subnets with the Client VPN
+START_FAILED: failed to associated subnets with the Client VPN
+STOP_FAILED: failed to disassociated subnets with the Client VPN
+"""
+
+START_IN_PROGRESS = 'START_IN_PROGRESS'
+STOP_IN_PROGRESS = 'STOP_IN_PROGRESS'
+START_FAILED = 'START_FAILED'
+STOP_FAILED = 'STOP_FAILED'

data/lib/cfnvpn/templates/lambdas.rb

@@ -17,7 +17,16 @@ module CfnVpn
         FileUtils.mkdir_p(zipfile_path)
         Zip::File.open("#{zipfile_path}/#{zipfile_name}", Zip::File::CREATE) do |zipfile|
           files.each do |file|
-            zipfile.add(file, File.join("#{lambdas_dir}/#{func}", file))
+            file_path = lambdas_dir
+            
+            # this is to allow for shared library methods in a different lambda directory
+            # so the files in the function lambda is in the root of the zip
+            if file.include? func
+              file_path = "#{file_path}/#{func}"
+              file.gsub!("#{func}/", "")
+            end
+
+            zipfile.add(file, File.join(file_path, file))
           end
         end