@opensearch-project/agent-health 0.3.0 → 0.5.2

package/deployment/cloudformation/agent-health-observability.yaml

@@ -0,0 +1,762 @@
+# Copyright OpenSearch Contributors
+# SPDX-License-Identifier: Apache-2.0
+#
+# Agent Health — CloudFormation template for managed observability infrastructure.
+# Deploys an Amazon OpenSearch Service domain and OpenSearch Ingestion (OSIS) pipeline
+# for collecting OpenTelemetry traces from AI agents.
+#
+# Launch Stack URL pattern (replace REGION):
+#   https://console.aws.amazon.com/cloudformation/home?region=REGION#/stacks/create/template?stackName=AgentHealthObservability&templateURL=https://agent-health-cfn-REGION.s3.REGION.amazonaws.com/agent-health-observability.yaml
+
+AWSTemplateFormatVersion: '2010-09-09'
+Description: >-
+  Agent Health Observability Stack — Amazon OpenSearch Service domain with
+  OpenSearch Ingestion (OSIS) pipeline for OTLP trace collection from AI agents.
+
+# =============================================================================
+# Parameters
+# =============================================================================
+Parameters:
+  DomainName:
+    Type: String
+    Default: ah-traces
+    Description: >-
+      Name for the OpenSearch Service domain. Keep ≤21 chars because OSIS
+      pipeline names are "${DomainName}-traces" with a 28-char max.
+    AllowedPattern: '[a-z][a-z0-9\-]+'
+    MinLength: 3
+    MaxLength: 21
+
+  InstanceType:
+    Type: String
+    Default: r8g.large.search
+    Description: OpenSearch instance type
+    AllowedValues:
+      - t3.small.search
+      - t3.medium.search
+      - r8g.large.search
+      - r8g.xlarge.search
+      - r8g.2xlarge.search
+
+  VolumeSize:
+    Type: Number
+    Default: 100
+    Description: EBS volume size in GB per data node
+    MinValue: 10
+    MaxValue: 1000
+
+  PipelineMinUnits:
+    Type: Number
+    Default: 1
+    Description: Minimum OSIS pipeline capacity units
+    MinValue: 1
+    MaxValue: 96
+
+  PipelineMaxUnits:
+    Type: Number
+    Default: 4
+    Description: Maximum OSIS pipeline capacity units
+    MinValue: 1
+    MaxValue: 96
+
+  AdminRoleARN:
+    Type: String
+    Description: >-
+      Optional IAM role ARN to grant OpenSearch all_access (e.g., your Admin role).
+      The domain master user is always the internal RoleMappingFunctionRole;
+      this parameter adds an additional role with full access via backend role mapping.
+    Default: ''
+
+# =============================================================================
+# Conditions
+# =============================================================================
+Conditions:
+  HasAdminRoleARN: !Not [!Equals [!Ref AdminRoleARN, '']]
+  # Note: Domain MasterUserARN is always RoleMappingFunctionRole (not conditional)
+
+# =============================================================================
+# Resources
+# =============================================================================
+Resources:
+
+  # ---------------------------------------------------------------------------
+  # IAM Role — FGAC Role Mapping Custom Resource (also used as MasterUserARN)
+  # Defined first because the OpenSearch domain references it as master user.
+  # ---------------------------------------------------------------------------
+  RoleMappingFunctionRole:
+    Type: AWS::IAM::Role
+    Properties:
+      RoleName: !Sub '${DomainName}-${AWS::Region}-role-mapping-cr-role'
+      AssumeRolePolicyDocument:
+        Version: '2012-10-17'
+        Statement:
+          - Effect: Allow
+            Principal:
+              Service: lambda.amazonaws.com
+            Action: 'sts:AssumeRole'
+      Policies:
+        - PolicyName: OpenSearchAccess
+          PolicyDocument:
+            Version: '2012-10-17'
+            Statement:
+              - Effect: Allow
+                Action:
+                  - 'es:ESHttp*'
+                Resource:
+                  - !Sub 'arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/${DomainName}/*'
+        - PolicyName: CloudWatchLogs
+          PolicyDocument:
+            Version: '2012-10-17'
+            Statement:
+              - Effect: Allow
+                Action:
+                  - 'logs:CreateLogGroup'
+                  - 'logs:CreateLogStream'
+                  - 'logs:PutLogEvents'
+                Resource: !Sub 'arn:aws:logs:${AWS::Region}:${AWS::AccountId}:*'
+      Tags:
+        - Key: agent-health
+          Value: observability
+        - Key: ManagedBy
+          Value: AgentHealthCFN
+
+  # ---------------------------------------------------------------------------
+  # OpenSearch Service Domain
+  # ---------------------------------------------------------------------------
+  OpenSearchDomain:
+    Type: AWS::OpenSearchService::Domain
+    DeletionPolicy: Retain
+    UpdateReplacePolicy: Retain
+    Properties:
+      DomainName: !Ref DomainName
+      EngineVersion: OpenSearch_3.5
+      ClusterConfig:
+        InstanceType: !Ref InstanceType
+        InstanceCount: 3
+        DedicatedMasterEnabled: true
+        DedicatedMasterType: !Ref InstanceType
+        DedicatedMasterCount: 3
+        ZoneAwarenessEnabled: true
+        ZoneAwarenessConfig:
+          AvailabilityZoneCount: 3
+      EBSOptions:
+        EBSEnabled: true
+        VolumeType: gp3
+        VolumeSize: !Ref VolumeSize
+      EncryptionAtRestOptions:
+        Enabled: true
+      NodeToNodeEncryptionOptions:
+        Enabled: true
+      DomainEndpointOptions:
+        EnforceHTTPS: true
+        TLSSecurityPolicy: Policy-Min-TLS-1-2-PFS-2023-10
+      AdvancedSecurityOptions:
+        Enabled: true
+        InternalUserDatabaseEnabled: false
+        MasterUserOptions:
+          MasterUserARN: !GetAtt RoleMappingFunctionRole.Arn
+      # "Only use fine-grained access control" — open domain policy,
+      # all authorization is handled by the OpenSearch Security plugin (FGAC).
+      AccessPolicies:
+        Version: '2012-10-17'
+        Statement:
+          - Effect: Allow
+            Principal:
+              AWS: '*'
+            Action: 'es:*'
+            Resource: !Sub 'arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/${DomainName}/*'
+      Tags:
+        - Key: agent-health
+          Value: observability
+        - Key: ManagedBy
+          Value: AgentHealthCFN
+
+  # ---------------------------------------------------------------------------
+  # IAM Role — OSIS Pipeline Execution
+  # ---------------------------------------------------------------------------
+  OSISPipelineRole:
+    Type: AWS::IAM::Role
+    Properties:
+      RoleName: !Sub '${DomainName}-${AWS::Region}-osis-pipeline-role'
+      AssumeRolePolicyDocument:
+        Version: '2012-10-17'
+        Statement:
+          - Effect: Allow
+            Principal:
+              Service: osis-pipelines.amazonaws.com
+            Action: 'sts:AssumeRole'
+      Policies:
+        - PolicyName: OSISToOpenSearch
+          PolicyDocument:
+            Version: '2012-10-17'
+            Statement:
+              - Effect: Allow
+                Action:
+                  - 'es:DescribeDomain'
+                  - 'es:ESHttp*'
+                Resource:
+                  - !Sub 'arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/${DomainName}'
+                  - !Sub 'arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/${DomainName}/*'
+      Tags:
+        - Key: agent-health
+          Value: observability
+        - Key: ManagedBy
+          Value: AgentHealthCFN
+
+  # ---------------------------------------------------------------------------
+  # IAM Role — Ingestion (for agents pushing telemetry via SigV4)
+  # ---------------------------------------------------------------------------
+  IngestionRole:
+    Type: AWS::IAM::Role
+    Properties:
+      RoleName: !Sub '${DomainName}-${AWS::Region}-ingestion-role'
+      AssumeRolePolicyDocument:
+        Version: '2012-10-17'
+        Statement:
+          - Effect: Allow
+            Principal:
+              AWS: !Sub 'arn:aws:iam::${AWS::AccountId}:root'
+            Action: 'sts:AssumeRole'
+      Policies:
+        - PolicyName: OSISIngest
+          PolicyDocument:
+            Version: '2012-10-17'
+            Statement:
+              - Effect: Allow
+                Action:
+                  - 'osis:Ingest'
+                Resource:
+                  - !Sub 'arn:aws:osis:${AWS::Region}:${AWS::AccountId}:pipeline/${DomainName}-traces'
+                  - !Sub 'arn:aws:osis:${AWS::Region}:${AWS::AccountId}:pipeline/${DomainName}-logs'
+      Tags:
+        - Key: agent-health
+          Value: observability
+        - Key: ManagedBy
+          Value: AgentHealthCFN
+
+  # ---------------------------------------------------------------------------
+  # OpenSearch Ingestion (OSIS) Pipeline — OTLP Traces → OpenSearch
+  # ---------------------------------------------------------------------------
+  OSISTracePipeline:
+    Type: AWS::OSIS::Pipeline
+    DependsOn: OpenSearchDomain
+    Properties:
+      PipelineName: !Sub '${DomainName}-traces'
+      MinUnits: !Ref PipelineMinUnits
+      MaxUnits: !Ref PipelineMaxUnits
+      PipelineConfigurationBody: !Sub |
+        version: "2"
+        otel-traces-entry:
+          source:
+            otel_trace_source:
+              path: "/${DomainName}-traces/v1/traces"
+          sink:
+            - pipeline:
+                name: "otel-traces-raw-pipeline"
+            - pipeline:
+                name: "otel-service-map-pipeline"
+        otel-traces-raw-pipeline:
+          source:
+            pipeline:
+              name: "otel-traces-entry"
+          processor:
+            - otel_traces:
+          sink:
+            - opensearch:
+                hosts:
+                  - "https://${OpenSearchDomain.DomainEndpoint}"
+                index_type: trace-analytics-plain-raw
+                aws:
+                  sts_role_arn: "${OSISPipelineRole.Arn}"
+                  region: "${AWS::Region}"
+        otel-service-map-pipeline:
+          source:
+            pipeline:
+              name: "otel-traces-entry"
+          processor:
+            - service_map:
+                window_duration: 180
+          sink:
+            - opensearch:
+                hosts:
+                  - "https://${OpenSearchDomain.DomainEndpoint}"
+                index_type: trace-analytics-service-map
+                aws:
+                  sts_role_arn: "${OSISPipelineRole.Arn}"
+                  region: "${AWS::Region}"
+      Tags:
+        - Key: agent-health
+          Value: observability
+        - Key: ManagedBy
+          Value: AgentHealthCFN
+
+  # ---------------------------------------------------------------------------
+  # OpenSearch Ingestion (OSIS) Pipeline — OTLP Logs → OpenSearch
+  # ---------------------------------------------------------------------------
+  OSISLogsPipeline:
+    Type: AWS::OSIS::Pipeline
+    DependsOn: OpenSearchDomain
+    Properties:
+      PipelineName: !Sub '${DomainName}-logs'
+      MinUnits: !Ref PipelineMinUnits
+      MaxUnits: !Ref PipelineMaxUnits
+      PipelineConfigurationBody: !Sub |
+        version: "2"
+        otel-logs-pipeline:
+          source:
+            otel_logs_source:
+              path: "/${DomainName}-logs/v1/logs"
+          sink:
+            - opensearch:
+                hosts:
+                  - "https://${OpenSearchDomain.DomainEndpoint}"
+                index: "otel-logs-%{yyyy.MM.dd}"
+                aws:
+                  sts_role_arn: "${OSISPipelineRole.Arn}"
+                  region: "${AWS::Region}"
+      Tags:
+        - Key: agent-health
+          Value: observability
+        - Key: ManagedBy
+          Value: AgentHealthCFN
+
+  # ---------------------------------------------------------------------------
+  # OTLP Ingestion — API Gateway + Lambda → OpenSearch (direct)
+  # Provides a public HTTPS endpoint so apps can send OTLP without SigV4 auth.
+  # Converts OTLP JSON to trace-analytics format and bulk-indexes into OpenSearch.
+  # Also forwards to OSIS pipelines when available for service-map generation.
+  # ---------------------------------------------------------------------------
+  OTLPIngestLambdaRole:
+    Type: AWS::IAM::Role
+    Properties:
+      RoleName: !Sub '${DomainName}-${AWS::Region}-otlp-ingest-lambda-role'
+      AssumeRolePolicyDocument:
+        Version: '2012-10-17'
+        Statement:
+          - Effect: Allow
+            Principal:
+              Service: lambda.amazonaws.com
+            Action: 'sts:AssumeRole'
+      Policies:
+        - PolicyName: OpenSearchAccess
+          PolicyDocument:
+            Version: '2012-10-17'
+            Statement:
+              - Effect: Allow
+                Action:
+                  - 'es:ESHttp*'
+                Resource:
+                  - !Sub 'arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/${DomainName}/*'
+        - PolicyName: OSISIngest
+          PolicyDocument:
+            Version: '2012-10-17'
+            Statement:
+              - Effect: Allow
+                Action:
+                  - 'osis:Ingest'
+                Resource:
+                  - !Sub 'arn:aws:osis:${AWS::Region}:${AWS::AccountId}:pipeline/${DomainName}-traces'
+                  - !Sub 'arn:aws:osis:${AWS::Region}:${AWS::AccountId}:pipeline/${DomainName}-logs'
+        - PolicyName: CloudWatchLogs
+          PolicyDocument:
+            Version: '2012-10-17'
+            Statement:
+              - Effect: Allow
+                Action:
+                  - 'logs:CreateLogGroup'
+                  - 'logs:CreateLogStream'
+                  - 'logs:PutLogEvents'
+                Resource: !Sub 'arn:aws:logs:${AWS::Region}:${AWS::AccountId}:*'
+      Tags:
+        - Key: agent-health
+          Value: observability
+        - Key: ManagedBy
+          Value: AgentHealthCFN
+
+  OTLPIngestFunction:
+    Type: AWS::Lambda::Function
+    DependsOn: OpenSearchDomain
+    Properties:
+      FunctionName: !Sub '${DomainName}-otlp-ingest'
+      Runtime: python3.12
+      Handler: index.handler
+      Timeout: 30
+      MemorySize: 256
+      Role: !GetAtt OTLPIngestLambdaRole.Arn
+      Environment:
+        Variables:
+          OPENSEARCH_ENDPOINT: !Sub 'https://${OpenSearchDomain.DomainEndpoint}'
+      Code:
+        ZipFile: |
+          """OTLP Ingestion — converts OTLP JSON to OpenSearch trace-analytics format."""
+          import json
+          import os
+          import urllib.request
+          import hashlib
+          from datetime import datetime, timezone
+          from botocore.auth import SigV4Auth
+          from botocore.awsrequest import AWSRequest
+          from botocore.session import Session
+
+          OS_ENDPOINT = os.environ['OPENSEARCH_ENDPOINT']
+          REGION = os.environ.get('AWS_REGION', 'us-east-1')
+          session = Session()
+
+          STATUS_MAP = {0: 0, 1: 0, 2: 2}  # OTLP Unset/Ok->0, Error->2
+          KIND_MAP = {0: 'SPAN_KIND_UNSPECIFIED', 1: 'SPAN_KIND_INTERNAL', 2: 'SPAN_KIND_SERVER',
+                      3: 'SPAN_KIND_CLIENT', 4: 'SPAN_KIND_PRODUCER', 5: 'SPAN_KIND_CONSUMER'}
+
+          def _creds():
+              c = session.get_credentials()
+              return c.resolve_credentials() if hasattr(c, 'resolve_credentials') else c
+
+          def _attr_value(v):
+              for k in ('stringValue', 'intValue', 'boolValue', 'doubleValue'):
+                  if k in v:
+                      return v[k]
+              if 'arrayValue' in v:
+                  return [_attr_value(x) for x in v['arrayValue'].get('values', [])]
+              return str(v)
+
+          def _nano_to_iso(ns):
+              """Convert nanosecond timestamp to ISO 8601 with nanosecond precision."""
+              try:
+                  ns = int(ns)
+              except (ValueError, TypeError):
+                  return '1970-01-01T00:00:00.000000000Z'
+              secs = ns // 1_000_000_000
+              nanos = ns % 1_000_000_000
+              dt = datetime.fromtimestamp(secs, tz=timezone.utc)
+              return dt.strftime('%Y-%m-%dT%H:%M:%S') + f'.{nanos:09d}Z'
+
+          def _convert_span(span, resource_attrs, scope_name):
+              trace_id = span.get('traceId', '')
+              span_id = span.get('spanId', '')
+              parent = span.get('parentSpanId', '')
+              start_ns = span.get('startTimeUnixNano', '0')
+              end_ns = span.get('endTimeUnixNano', '0')
+              try:
+                  duration_nanos = int(end_ns) - int(start_ns)
+                  if duration_nanos < 0:
+                      duration_nanos = 0
+              except (ValueError, TypeError):
+                  duration_nanos = 0
+              status = span.get('status', {})
+              status_code = STATUS_MAP.get(status.get('code', 0), 0)
+              kind = KIND_MAP.get(span.get('kind', 0), 'SPAN_KIND_UNSPECIFIED')
+              service_name = 'unknown'
+              events = []
+              for evt in span.get('events', []):
+                  evt_attrs = {}
+                  for a in evt.get('attributes', []):
+                      evt_attrs[a['key']] = _attr_value(a['value'])
+                  events.append({
+                      'name': evt.get('name', ''),
+                      'time': _nano_to_iso(evt.get('timeUnixNano', '0')),
+                      'attributes': evt_attrs
+                  })
+              doc = {
+                  'traceId': trace_id,
+                  'spanId': span_id,
+                  'parentSpanId': parent,
+                  'traceState': span.get('traceState', ''),
+                  'name': span.get('name', ''),
+                  'kind': kind,
+                  'startTime': _nano_to_iso(start_ns),
+                  'endTime': _nano_to_iso(end_ns),
+                  'durationInNanos': duration_nanos,
+                  'serviceName': service_name,
+                  'status': {'code': status_code, 'message': status.get('message', '')},
+                  'events': events,
+                  'links': [],
+                  'instrumentationScope': {'name': scope_name},
+              }
+              if not parent:
+                  doc['traceGroup'] = span.get('name', '')
+                  doc['traceGroupFields'] = {
+                      'endTime': _nano_to_iso(end_ns),
+                      'durationInNanos': duration_nanos,
+                      'statusCode': status_code,
+                  }
+              # Resource attributes: nested resource.attributes object, literal
+              # dotted keys (Data Prepper trace-analytics-plain-raw / OTEL-faithful).
+              resource_attributes = {}
+              for a in resource_attrs:
+                  val = _attr_value(a['value'])
+                  resource_attributes[a['key']] = val
+                  if a['key'] == 'service.name':
+                      doc['serviceName'] = val
+              doc['resource'] = {'attributes': resource_attributes}
+              # Span attributes: nested attributes object, literal dotted keys.
+              span_attributes = {}
+              for a in span.get('attributes', []):
+                  span_attributes[a['key']] = _attr_value(a['value'])
+              doc['attributes'] = span_attributes
+              doc_id = hashlib.md5(f'{trace_id}{span_id}'.encode()).hexdigest()
+              return doc_id, doc
+
+          def _signed_request(method, path, body):
+              url = OS_ENDPOINT + path
+              request = AWSRequest(method=method, url=url, data=body,
+                                   headers={'Content-Type': 'application/json'})
+              SigV4Auth(_creds(), 'es', REGION).add_auth(request)
+              req = urllib.request.Request(url, data=body.encode('utf-8'), method=method,
+                                           headers={k: v for k, v in dict(request.headers).items()})
+              try:
+                  with urllib.request.urlopen(req) as resp:
+                      return resp.status, resp.read().decode('utf-8', errors='replace')
+              except urllib.error.HTTPError as e:
+                  return e.code, e.read().decode('utf-8', errors='replace')
+
+          def handler(event, context):
+              path = event.get('rawPath', '')
+              if '/v1/traces' not in path and '/v1/logs' not in path:
+                  return {'statusCode': 404, 'body': json.dumps({'error': f'Unknown path: {path}'})}
+              body = event.get('body', '')
+              is_base64 = event.get('isBase64Encoded', False)
+              if is_base64:
+                  import base64
+                  body = base64.b64decode(body).decode('utf-8', errors='replace')
+              try:
+                  otlp = json.loads(body) if isinstance(body, str) else body
+              except json.JSONDecodeError as e:
+                  return {'statusCode': 400, 'body': json.dumps({'error': f'Invalid JSON: {e}'})}
+              bulk_lines = []
+              for rs in otlp.get('resourceSpans', []):
+                  res_attrs = rs.get('resource', {}).get('attributes', [])
+                  for ss in rs.get('scopeSpans', []):
+                      scope_name = ss.get('scope', {}).get('name', '')
+                      for span in ss.get('spans', []):
+                          doc_id, doc = _convert_span(span, res_attrs, scope_name)
+                          bulk_lines.append(json.dumps({'index': {'_index': 'otel-v1-apm-span-000001', '_id': doc_id}}))
+                          bulk_lines.append(json.dumps(doc, default=str))
+              if not bulk_lines:
+                  return {'statusCode': 200, 'body': json.dumps({'message': 'No spans to index'})}
+              bulk_body = '\n'.join(bulk_lines) + '\n'
+              status, resp_body = _signed_request('POST', '/otel-v1-apm-span-000001/_bulk', bulk_body)
+              return {'statusCode': status, 'body': resp_body}
+      Tags:
+        - Key: agent-health
+          Value: observability
+        - Key: ManagedBy
+          Value: AgentHealthCFN
+
+  # ---------------------------------------------------------------------------
+  # Custom Resource — Map IAM Roles to OpenSearch FGAC Backend Roles
+  # Maps the OSIS pipeline role and Lambda ingest role to 'all_access' so they
+  # can write to OpenSearch indices. Runs once during stack create/update.
+  # ---------------------------------------------------------------------------
+  RoleMappingFunction:
+    Type: AWS::Lambda::Function
+    DependsOn: OpenSearchDomain
+    Properties:
+      FunctionName: !Sub '${DomainName}-fgac-role-mapping'
+      Runtime: python3.12
+      Handler: index.handler
+      Timeout: 60
+      MemorySize: 128
+      Role: !GetAtt RoleMappingFunctionRole.Arn
+      Environment:
+        Variables:
+          OPENSEARCH_ENDPOINT: !Sub 'https://${OpenSearchDomain.DomainEndpoint}'
+          OSIS_ROLE_ARN: !GetAtt OSISPipelineRole.Arn
+          LAMBDA_ROLE_ARN: !GetAtt OTLPIngestLambdaRole.Arn
+          ADMIN_ROLE_ARN: !If
+            - HasAdminRoleARN
+            - !Ref AdminRoleARN
+            - !Sub 'arn:aws:iam::${AWS::AccountId}:root'
+      Code:
+        ZipFile: |
+          """Custom Resource — maps IAM role ARNs to OpenSearch all_access backend role."""
+          import json
+          import os
+          import urllib.request
+          from botocore.auth import SigV4Auth
+          from botocore.awsrequest import AWSRequest
+          from botocore.session import Session
+
+          OS_ENDPOINT = os.environ['OPENSEARCH_ENDPOINT']
+          REGION = os.environ.get('AWS_REGION', 'us-east-1')
+          OSIS_ROLE_ARN = os.environ['OSIS_ROLE_ARN']
+          LAMBDA_ROLE_ARN = os.environ['LAMBDA_ROLE_ARN']
+          ADMIN_ROLE_ARN = os.environ.get('ADMIN_ROLE_ARN', '')
+          session = Session()
+
+          def _creds():
+              c = session.get_credentials()
+              return c.resolve_credentials() if hasattr(c, 'resolve_credentials') else c
+
+          def _signed_request(method, path, body=None):
+              url = OS_ENDPOINT + path
+              headers = {'Content-Type': 'application/json'}
+              request = AWSRequest(method=method, url=url, data=body or '', headers=headers)
+              SigV4Auth(_creds(), 'es', REGION).add_auth(request)
+              req = urllib.request.Request(url, data=(body or '').encode('utf-8'), method=method,
+                                           headers={k: v for k, v in dict(request.headers).items()})
+              try:
+                  with urllib.request.urlopen(req) as resp:
+                      return resp.status, json.loads(resp.read().decode('utf-8', errors='replace'))
+              except urllib.error.HTTPError as e:
+                  return e.code, json.loads(e.read().decode('utf-8', errors='replace'))
+
+          def _send_cfn_response(event, context, status, reason=''):
+              body = json.dumps({
+                  'Status': status,
+                  'Reason': reason or f'See CloudWatch Log Stream: {context.log_stream_name}',
+                  'PhysicalResourceId': context.log_stream_name,
+                  'StackId': event['StackId'],
+                  'RequestId': event['RequestId'],
+                  'LogicalResourceId': event['LogicalResourceId'],
+              })
+              req = urllib.request.Request(event['ResponseURL'], data=body.encode('utf-8'),
+                                           method='PUT', headers={'Content-Type': ''})
+              urllib.request.urlopen(req)
+
+          def handler(event, context):
+              print(f"Event: {json.dumps(event)}")
+              try:
+                  if event['RequestType'] == 'Delete':
+                      _send_cfn_response(event, context, 'SUCCESS')
+                      return
+                  # Get current mapping for all_access
+                  status, current = _signed_request('GET', '/_plugins/_security/api/rolesmapping/all_access')
+                  print(f"Current mapping (status={status}): {json.dumps(current)}")
+                  if status == 200:
+                      backend_roles = current.get('all_access', {}).get('backend_roles', [])
+                  else:
+                      backend_roles = []
+                  # Add our roles if not already present
+                  roles_to_map = [r for r in [OSIS_ROLE_ARN, LAMBDA_ROLE_ARN, ADMIN_ROLE_ARN] if r]
+                  for role in roles_to_map:
+                      if role not in backend_roles:
+                          backend_roles.append(role)
+                  # PUT the updated mapping
+                  payload = json.dumps({'backend_roles': backend_roles})
+                  status, resp = _signed_request('PUT', '/_plugins/_security/api/rolesmapping/all_access', payload)
+                  print(f"PUT mapping (status={status}): {json.dumps(resp)}")
+                  if status in (200, 201):
+                      _send_cfn_response(event, context, 'SUCCESS')
+                  else:
+                      _send_cfn_response(event, context, 'FAILED', f'OpenSearch returned {status}: {json.dumps(resp)}')
+              except Exception as e:
+                  print(f"Error: {e}")
+                  _send_cfn_response(event, context, 'FAILED', str(e))
+      Tags:
+        - Key: agent-health
+          Value: observability
+        - Key: ManagedBy
+          Value: AgentHealthCFN
+
+  RoleMappingCustomResource:
+    Type: Custom::OpenSearchRoleMapping
+    DependsOn:
+      - OpenSearchDomain
+      - OSISPipelineRole
+      - OTLPIngestLambdaRole
+      - RoleMappingFunction
+    Properties:
+      ServiceToken: !GetAtt RoleMappingFunction.Arn
+      # Include role ARNs so CFN re-triggers on changes
+      OSISRoleArn: !GetAtt OSISPipelineRole.Arn
+      LambdaRoleArn: !GetAtt OTLPIngestLambdaRole.Arn
+      AdminRoleArn: !If
+        - HasAdminRoleARN
+        - !Ref AdminRoleARN
+        - !Sub 'arn:aws:iam::${AWS::AccountId}:root'
+
+  OTLPIngestApi:
+    Type: AWS::ApiGatewayV2::Api
+    Properties:
+      Name: !Sub '${DomainName}-otlp-ingest'
+      ProtocolType: HTTP
+      CorsConfiguration:
+        AllowOrigins:
+          - '*'
+        AllowMethods:
+          - POST
+        AllowHeaders:
+          - content-type
+      Tags:
+        agent-health: observability
+        ManagedBy: AgentHealthCFN
+
+  OTLPIngestApiIntegration:
+    Type: AWS::ApiGatewayV2::Integration
+    Properties:
+      ApiId: !Ref OTLPIngestApi
+      IntegrationType: AWS_PROXY
+      IntegrationUri: !GetAtt OTLPIngestFunction.Arn
+      PayloadFormatVersion: '2.0'
+
+  OTLPIngestApiRoute:
+    Type: AWS::ApiGatewayV2::Route
+    Properties:
+      ApiId: !Ref OTLPIngestApi
+      RouteKey: 'POST /v1/{signal+}'
+      Target: !Sub 'integrations/${OTLPIngestApiIntegration}'
+
+  OTLPIngestApiStage:
+    Type: AWS::ApiGatewayV2::Stage
+    Properties:
+      ApiId: !Ref OTLPIngestApi
+      StageName: '$default'
+      AutoDeploy: true
+
+  OTLPIngestApiPermission:
+    Type: AWS::Lambda::Permission
+    Properties:
+      FunctionName: !Ref OTLPIngestFunction
+      Action: 'lambda:InvokeFunction'
+      Principal: apigateway.amazonaws.com
+      SourceArn: !Sub 'arn:aws:execute-api:${AWS::Region}:${AWS::AccountId}:${OTLPIngestApi}/*'
+
+# =============================================================================
+# Outputs
+# =============================================================================
+Outputs:
+  OpenSearchEndpoint:
+    Description: OpenSearch domain endpoint URL
+    Value: !Sub 'https://${OpenSearchDomain.DomainEndpoint}'
+
+  OSISTraceIngestEndpoint:
+    Description: OSIS trace ingest endpoint (configure as OTEL_EXPORTER_OTLP_TRACES_ENDPOINT in your agent)
+    Value: !Join
+      - ''
+      - - 'https://'
+        - !Select [0, !GetAtt OSISTracePipeline.IngestEndpointUrls]
+
+  OSISLogsIngestEndpoint:
+    Description: OSIS logs ingest endpoint (configure as OTEL_EXPORTER_OTLP_LOGS_ENDPOINT in your agent)
+    Value: !Join
+      - ''
+      - - 'https://'
+        - !Select [0, !GetAtt OSISLogsPipeline.IngestEndpointUrls]
+
+  Region:
+    Description: AWS Region where the stack was deployed
+    Value: !Ref 'AWS::Region'
+
+  IngestionRoleArn:
+    Description: IAM role ARN for agents to assume when pushing telemetry via SigV4
+    Value: !GetAtt IngestionRole.Arn
+
+  OTLPIngestEndpoint:
+    Description: >-
+      HTTPS endpoint for OTLP trace/log ingestion (no SigV4 needed on client).
+      Set OTEL_EXPORTER_OTLP_ENDPOINT to this value in your agent config.
+    Value: !Sub 'https://${OTLPIngestApi}.execute-api.${AWS::Region}.amazonaws.com'
+
+  AgentHealthConfigJSON:
+    Description: >-
+      Copy this JSON block into your agent-health.config.json file,
+      or run: npx @opensearch-project/agent-health configure --from-stack AgentHealthObservability
+    Value: !Sub |
+      {
+        "observability": {
+          "endpoint": "https://${OpenSearchDomain.DomainEndpoint}",
+          "authType": "sigv4",
+          "awsRegion": "${AWS::Region}",
+          "awsService": "es",
+          "tlsSkipVerify": false
+        }
+      }