@opensearch-project/agent-health 0.2.0 → 0.4.0

package/deployment/cloudformation/agent-health-observability.yaml

@@ -0,0 +1,571 @@
+# Copyright OpenSearch Contributors
+# SPDX-License-Identifier: Apache-2.0
+#
+# Agent Health — CloudFormation template for managed observability infrastructure.
+# Deploys an Amazon OpenSearch Service domain and OpenSearch Ingestion (OSIS) pipeline
+# for collecting OpenTelemetry traces from AI agents.
+#
+# Launch Stack URL pattern (replace REGION):
+#   https://console.aws.amazon.com/cloudformation/home?region=REGION#/stacks/create/template?stackName=AgentHealthObservability&templateURL=https://agent-health-cfn-REGION.s3.REGION.amazonaws.com/agent-health-observability.yaml
+
+AWSTemplateFormatVersion: '2010-09-09'
+Description: >-
+  Agent Health Observability Stack — Amazon OpenSearch Service domain with
+  OpenSearch Ingestion (OSIS) pipeline for OTLP trace collection from AI agents.
+
+# =============================================================================
+# Parameters
+# =============================================================================
+Parameters:
+  DomainName:
+    Type: String
+    Default: agent-health-traces
+    Description: Name for the OpenSearch Service domain
+    AllowedPattern: '[a-z][a-z0-9\-]+'
+    MinLength: 3
+    MaxLength: 28
+
+  InstanceType:
+    Type: String
+    Default: r8g.large.search
+    Description: OpenSearch instance type
+    AllowedValues:
+      - t3.small.search
+      - t3.medium.search
+      - r8g.large.search
+      - r8g.xlarge.search
+      - r8g.2xlarge.search
+
+  VolumeSize:
+    Type: Number
+    Default: 100
+    Description: EBS volume size in GB per data node
+    MinValue: 10
+    MaxValue: 1000
+
+  PipelineMinUnits:
+    Type: Number
+    Default: 1
+    Description: Minimum OSIS pipeline capacity units
+    MinValue: 1
+    MaxValue: 96
+
+  PipelineMaxUnits:
+    Type: Number
+    Default: 4
+    Description: Maximum OSIS pipeline capacity units
+    MinValue: 1
+    MaxValue: 96
+
+  MasterUserARN:
+    Type: String
+    Description: >-
+      IAM ARN for the OpenSearch FGAC master user (used for security plugin admin).
+      Defaults to the account root; override with your IAM role ARN for direct access.
+    Default: ''
+
+# =============================================================================
+# Conditions
+# =============================================================================
+Conditions:
+  HasMasterUserARN: !Not [!Equals [!Ref MasterUserARN, '']]
+
+# =============================================================================
+# Resources
+# =============================================================================
+Resources:
+
+  # ---------------------------------------------------------------------------
+  # OpenSearch Service Domain
+  # ---------------------------------------------------------------------------
+  OpenSearchDomain:
+    Type: AWS::OpenSearchService::Domain
+    Properties:
+      DomainName: !Ref DomainName
+      EngineVersion: OpenSearch_3.5
+      ClusterConfig:
+        InstanceType: !Ref InstanceType
+        InstanceCount: 3
+        DedicatedMasterEnabled: true
+        DedicatedMasterType: !Ref InstanceType
+        DedicatedMasterCount: 3
+        ZoneAwarenessEnabled: true
+        ZoneAwarenessConfig:
+          AvailabilityZoneCount: 3
+      EBSOptions:
+        EBSEnabled: true
+        VolumeType: gp3
+        VolumeSize: !Ref VolumeSize
+      EncryptionAtRestOptions:
+        Enabled: true
+      NodeToNodeEncryptionOptions:
+        Enabled: true
+      DomainEndpointOptions:
+        EnforceHTTPS: true
+        TLSSecurityPolicy: Policy-Min-TLS-1-2-PFS-2023-10
+      AdvancedSecurityOptions:
+        Enabled: true
+        InternalUserDatabaseEnabled: false
+        MasterUserOptions:
+          MasterUserARN: !If
+            - HasMasterUserARN
+            - !Ref MasterUserARN
+            - !Sub 'arn:aws:iam::${AWS::AccountId}:root'
+      # "Only use fine-grained access control" — open domain policy,
+      # all authorization is handled by the OpenSearch Security plugin (FGAC).
+      AccessPolicies:
+        Version: '2012-10-17'
+        Statement:
+          - Effect: Allow
+            Principal:
+              AWS: '*'
+            Action: 'es:*'
+            Resource: !Sub 'arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/${DomainName}/*'
+      Tags:
+        - Key: agent-health
+          Value: observability
+        - Key: ManagedBy
+          Value: AgentHealthCFN
+
+  # ---------------------------------------------------------------------------
+  # IAM Role — OSIS Pipeline Execution
+  # ---------------------------------------------------------------------------
+  OSISPipelineRole:
+    Type: AWS::IAM::Role
+    Properties:
+      RoleName: !Sub '${DomainName}-osis-pipeline-role'
+      AssumeRolePolicyDocument:
+        Version: '2012-10-17'
+        Statement:
+          - Effect: Allow
+            Principal:
+              Service: osis-pipelines.amazonaws.com
+            Action: 'sts:AssumeRole'
+      Policies:
+        - PolicyName: OSISToOpenSearch
+          PolicyDocument:
+            Version: '2012-10-17'
+            Statement:
+              - Effect: Allow
+                Action:
+                  - 'es:DescribeDomain'
+                  - 'es:ESHttp*'
+                Resource:
+                  - !Sub 'arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/${DomainName}'
+                  - !Sub 'arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/${DomainName}/*'
+
+  # ---------------------------------------------------------------------------
+  # IAM Role — Ingestion (for agents pushing telemetry via SigV4)
+  # ---------------------------------------------------------------------------
+  IngestionRole:
+    Type: AWS::IAM::Role
+    Properties:
+      RoleName: !Sub '${DomainName}-ingestion-role'
+      AssumeRolePolicyDocument:
+        Version: '2012-10-17'
+        Statement:
+          - Effect: Allow
+            Principal:
+              AWS: !Sub 'arn:aws:iam::${AWS::AccountId}:root'
+            Action: 'sts:AssumeRole'
+      Policies:
+        - PolicyName: OSISIngest
+          PolicyDocument:
+            Version: '2012-10-17'
+            Statement:
+              - Effect: Allow
+                Action:
+                  - 'osis:Ingest'
+                Resource:
+                  - !Sub 'arn:aws:osis:${AWS::Region}:${AWS::AccountId}:pipeline/${DomainName}-traces'
+                  - !Sub 'arn:aws:osis:${AWS::Region}:${AWS::AccountId}:pipeline/${DomainName}-logs'
+
+  # ---------------------------------------------------------------------------
+  # OpenSearch Ingestion (OSIS) Pipeline — OTLP Traces → OpenSearch
+  # ---------------------------------------------------------------------------
+  OSISTracePipeline:
+    Type: AWS::OSIS::Pipeline
+    DependsOn: OpenSearchDomain
+    Properties:
+      PipelineName: !Sub '${DomainName}-traces'
+      MinUnits: !Ref PipelineMinUnits
+      MaxUnits: !Ref PipelineMaxUnits
+      PipelineConfigurationBody: !Sub |
+        version: "2"
+        otel-traces-entry:
+          source:
+            otel_trace_source:
+              path: "/${DomainName}-traces/v1/traces"
+          sink:
+            - pipeline:
+                name: "otel-traces-raw-pipeline"
+            - pipeline:
+                name: "otel-service-map-pipeline"
+        otel-traces-raw-pipeline:
+          source:
+            pipeline:
+              name: "otel-traces-entry"
+          processor:
+            - otel_traces:
+          sink:
+            - opensearch:
+                hosts:
+                  - "https://${OpenSearchDomain.DomainEndpoint}"
+                index_type: trace-analytics-raw
+                aws:
+                  sts_role_arn: "${OSISPipelineRole.Arn}"
+                  region: "${AWS::Region}"
+        otel-service-map-pipeline:
+          source:
+            pipeline:
+              name: "otel-traces-entry"
+          processor:
+            - service_map:
+                window_duration: 180
+          sink:
+            - opensearch:
+                hosts:
+                  - "https://${OpenSearchDomain.DomainEndpoint}"
+                index_type: trace-analytics-service-map
+                aws:
+                  sts_role_arn: "${OSISPipelineRole.Arn}"
+                  region: "${AWS::Region}"
+      Tags:
+        - Key: agent-health
+          Value: observability
+
+  # ---------------------------------------------------------------------------
+  # OpenSearch Ingestion (OSIS) Pipeline — OTLP Logs → OpenSearch
+  # ---------------------------------------------------------------------------
+  OSISLogsPipeline:
+    Type: AWS::OSIS::Pipeline
+    DependsOn: OpenSearchDomain
+    Properties:
+      PipelineName: !Sub '${DomainName}-logs'
+      MinUnits: !Ref PipelineMinUnits
+      MaxUnits: !Ref PipelineMaxUnits
+      PipelineConfigurationBody: !Sub |
+        version: "2"
+        otel-logs-pipeline:
+          source:
+            otel_logs_source:
+              path: "/${DomainName}-logs/v1/logs"
+          sink:
+            - opensearch:
+                hosts:
+                  - "https://${OpenSearchDomain.DomainEndpoint}"
+                index: "otel-logs-%%{yyyy.MM.dd}"
+                aws:
+                  sts_role_arn: "${OSISPipelineRole.Arn}"
+                  region: "${AWS::Region}"
+      Tags:
+        - Key: agent-health
+          Value: observability
+
+  # ---------------------------------------------------------------------------
+  # OTLP Ingestion — API Gateway + Lambda → OpenSearch (direct)
+  # Provides a public HTTPS endpoint so apps can send OTLP without SigV4 auth.
+  # Converts OTLP JSON to trace-analytics format and bulk-indexes into OpenSearch.
+  # Also forwards to OSIS pipelines when available for service-map generation.
+  # ---------------------------------------------------------------------------
+  OTLPIngestLambdaRole:
+    Type: AWS::IAM::Role
+    Properties:
+      RoleName: !Sub '${DomainName}-otlp-ingest-lambda-role'
+      AssumeRolePolicyDocument:
+        Version: '2012-10-17'
+        Statement:
+          - Effect: Allow
+            Principal:
+              Service: lambda.amazonaws.com
+            Action: 'sts:AssumeRole'
+      Policies:
+        - PolicyName: OpenSearchAccess
+          PolicyDocument:
+            Version: '2012-10-17'
+            Statement:
+              - Effect: Allow
+                Action:
+                  - 'es:ESHttp*'
+                Resource:
+                  - !Sub 'arn:aws:es:${AWS::Region}:${AWS::AccountId}:domain/${DomainName}/*'
+        - PolicyName: OSISIngest
+          PolicyDocument:
+            Version: '2012-10-17'
+            Statement:
+              - Effect: Allow
+                Action:
+                  - 'osis:Ingest'
+                Resource:
+                  - !Sub 'arn:aws:osis:${AWS::Region}:${AWS::AccountId}:pipeline/${DomainName}-traces'
+                  - !Sub 'arn:aws:osis:${AWS::Region}:${AWS::AccountId}:pipeline/${DomainName}-logs'
+        - PolicyName: CloudWatchLogs
+          PolicyDocument:
+            Version: '2012-10-17'
+            Statement:
+              - Effect: Allow
+                Action:
+                  - 'logs:CreateLogGroup'
+                  - 'logs:CreateLogStream'
+                  - 'logs:PutLogEvents'
+                Resource: !Sub 'arn:aws:logs:${AWS::Region}:${AWS::AccountId}:*'
+
+  OTLPIngestFunction:
+    Type: AWS::Lambda::Function
+    DependsOn: OpenSearchDomain
+    Properties:
+      FunctionName: !Sub '${DomainName}-otlp-ingest'
+      Runtime: python3.12
+      Handler: index.handler
+      Timeout: 30
+      MemorySize: 256
+      Role: !GetAtt OTLPIngestLambdaRole.Arn
+      Environment:
+        Variables:
+          OPENSEARCH_ENDPOINT: !Sub 'https://${OpenSearchDomain.DomainEndpoint}'
+      Code:
+        ZipFile: |
+          """OTLP Ingestion — converts OTLP JSON to OpenSearch trace-analytics format."""
+          import json
+          import os
+          import urllib.request
+          import hashlib
+          from datetime import datetime, timezone
+          from botocore.auth import SigV4Auth
+          from botocore.awsrequest import AWSRequest
+          from botocore.session import Session
+
+          OS_ENDPOINT = os.environ['OPENSEARCH_ENDPOINT']
+          REGION = os.environ.get('AWS_REGION', 'us-east-1')
+          session = Session()
+
+          STATUS_MAP = {0: 0, 1: 0, 2: 2}  # OTLP Unset/Ok->0, Error->2
+          KIND_MAP = {0: 'SPAN_KIND_UNSPECIFIED', 1: 'SPAN_KIND_INTERNAL', 2: 'SPAN_KIND_SERVER',
+                      3: 'SPAN_KIND_CLIENT', 4: 'SPAN_KIND_PRODUCER', 5: 'SPAN_KIND_CONSUMER'}
+
+          def _creds():
+              c = session.get_credentials()
+              return c.resolve_credentials() if hasattr(c, 'resolve_credentials') else c
+
+          def _attr_value(v):
+              for k in ('stringValue', 'intValue', 'boolValue', 'doubleValue'):
+                  if k in v:
+                      return v[k]
+              if 'arrayValue' in v:
+                  return [_attr_value(x) for x in v['arrayValue'].get('values', [])]
+              return str(v)
+
+          def _nano_to_iso(ns):
+              """Convert nanosecond timestamp to ISO 8601 with nanosecond precision."""
+              try:
+                  ns = int(ns)
+              except (ValueError, TypeError):
+                  return '1970-01-01T00:00:00.000000000Z'
+              secs = ns // 1_000_000_000
+              nanos = ns % 1_000_000_000
+              dt = datetime.fromtimestamp(secs, tz=timezone.utc)
+              return dt.strftime('%Y-%m-%dT%H:%M:%S') + f'.{nanos:09d}Z'
+
+          def _sanitize_key(key):
+              """Replace dots with @ in attribute keys (data prepper convention)."""
+              return key.replace('.', '@')
+
+          def _convert_span(span, resource_attrs, scope_name):
+              trace_id = span.get('traceId', '')
+              span_id = span.get('spanId', '')
+              parent = span.get('parentSpanId', '')
+              start_ns = span.get('startTimeUnixNano', '0')
+              end_ns = span.get('endTimeUnixNano', '0')
+              try:
+                  duration_nanos = int(end_ns) - int(start_ns)
+                  if duration_nanos < 0:
+                      duration_nanos = 0
+              except (ValueError, TypeError):
+                  duration_nanos = 0
+              status = span.get('status', {})
+              status_code = STATUS_MAP.get(status.get('code', 0), 0)
+              kind = KIND_MAP.get(span.get('kind', 0), 'SPAN_KIND_UNSPECIFIED')
+              service_name = 'unknown'
+              events = []
+              for evt in span.get('events', []):
+                  evt_attrs = {}
+                  for a in evt.get('attributes', []):
+                      evt_attrs[_sanitize_key(a['key'])] = _attr_value(a['value'])
+                  events.append({
+                      'name': evt.get('name', ''),
+                      'time': _nano_to_iso(evt.get('timeUnixNano', '0')),
+                      'attributes': evt_attrs
+                  })
+              doc = {
+                  'traceId': trace_id,
+                  'spanId': span_id,
+                  'parentSpanId': parent,
+                  'traceState': span.get('traceState', ''),
+                  'name': span.get('name', ''),
+                  'kind': kind,
+                  'startTime': _nano_to_iso(start_ns),
+                  'endTime': _nano_to_iso(end_ns),
+                  'durationInNanos': duration_nanos,
+                  'serviceName': service_name,
+                  'status': {'code': status_code, 'message': status.get('message', '')},
+                  'events': events,
+                  'links': [],
+                  'instrumentationScope.name': scope_name,
+              }
+              if not parent:
+                  doc['traceGroup'] = span.get('name', '')
+                  doc['traceGroupFields.endTime'] = _nano_to_iso(end_ns)
+                  doc['traceGroupFields.durationInNanos'] = duration_nanos
+                  doc['traceGroupFields.statusCode'] = status_code
+              # Flatten resource attributes with @ separator for dotted keys
+              for a in resource_attrs:
+                  key = _sanitize_key(a['key'])
+                  val = _attr_value(a['value'])
+                  doc[f'resource.attributes.{key}'] = val
+                  if a['key'] == 'service.name':
+                      doc['serviceName'] = val
+              # Flatten span attributes with @ separator for dotted keys
+              for a in span.get('attributes', []):
+                  key = _sanitize_key(a['key'])
+                  val = _attr_value(a['value'])
+                  doc[f'span.attributes.{key}'] = val
+              doc_id = hashlib.md5(f'{trace_id}{span_id}'.encode()).hexdigest()
+              return doc_id, doc
+
+          def _signed_request(method, path, body):
+              url = OS_ENDPOINT + path
+              request = AWSRequest(method=method, url=url, data=body,
+                                   headers={'Content-Type': 'application/json'})
+              SigV4Auth(_creds(), 'es', REGION).add_auth(request)
+              req = urllib.request.Request(url, data=body.encode('utf-8'), method=method,
+                                           headers={k: v for k, v in dict(request.headers).items()})
+              try:
+                  with urllib.request.urlopen(req) as resp:
+                      return resp.status, resp.read().decode('utf-8', errors='replace')
+              except urllib.error.HTTPError as e:
+                  return e.code, e.read().decode('utf-8', errors='replace')
+
+          def handler(event, context):
+              path = event.get('rawPath', '')
+              if '/v1/traces' not in path and '/v1/logs' not in path:
+                  return {'statusCode': 404, 'body': json.dumps({'error': f'Unknown path: {path}'})}
+              body = event.get('body', '')
+              is_base64 = event.get('isBase64Encoded', False)
+              if is_base64:
+                  import base64
+                  body = base64.b64decode(body).decode('utf-8', errors='replace')
+              try:
+                  otlp = json.loads(body) if isinstance(body, str) else body
+              except json.JSONDecodeError as e:
+                  return {'statusCode': 400, 'body': json.dumps({'error': f'Invalid JSON: {e}'})}
+              bulk_lines = []
+              for rs in otlp.get('resourceSpans', []):
+                  res_attrs = rs.get('resource', {}).get('attributes', [])
+                  for ss in rs.get('scopeSpans', []):
+                      scope_name = ss.get('scope', {}).get('name', '')
+                      for span in ss.get('spans', []):
+                          doc_id, doc = _convert_span(span, res_attrs, scope_name)
+                          bulk_lines.append(json.dumps({'index': {'_index': 'otel-v1-apm-span-000001', '_id': doc_id}}))
+                          bulk_lines.append(json.dumps(doc, default=str))
+              if not bulk_lines:
+                  return {'statusCode': 200, 'body': json.dumps({'message': 'No spans to index'})}
+              bulk_body = '\n'.join(bulk_lines) + '\n'
+              status, resp_body = _signed_request('POST', '/otel-v1-apm-span-000001/_bulk', bulk_body)
+              return {'statusCode': status, 'body': resp_body}
+      Tags:
+        - Key: agent-health
+          Value: observability
+
+  OTLPIngestApi:
+    Type: AWS::ApiGatewayV2::Api
+    Properties:
+      Name: !Sub '${DomainName}-otlp-ingest'
+      ProtocolType: HTTP
+      CorsConfiguration:
+        AllowOrigins:
+          - '*'
+        AllowMethods:
+          - POST
+        AllowHeaders:
+          - content-type
+
+  OTLPIngestApiIntegration:
+    Type: AWS::ApiGatewayV2::Integration
+    Properties:
+      ApiId: !Ref OTLPIngestApi
+      IntegrationType: AWS_PROXY
+      IntegrationUri: !GetAtt OTLPIngestFunction.Arn
+      PayloadFormatVersion: '2.0'
+
+  OTLPIngestApiRoute:
+    Type: AWS::ApiGatewayV2::Route
+    Properties:
+      ApiId: !Ref OTLPIngestApi
+      RouteKey: 'POST /v1/{signal+}'
+      Target: !Sub 'integrations/${OTLPIngestApiIntegration}'
+
+  OTLPIngestApiStage:
+    Type: AWS::ApiGatewayV2::Stage
+    Properties:
+      ApiId: !Ref OTLPIngestApi
+      StageName: '$default'
+      AutoDeploy: true
+
+  OTLPIngestApiPermission:
+    Type: AWS::Lambda::Permission
+    Properties:
+      FunctionName: !Ref OTLPIngestFunction
+      Action: 'lambda:InvokeFunction'
+      Principal: apigateway.amazonaws.com
+      SourceArn: !Sub 'arn:aws:execute-api:${AWS::Region}:${AWS::AccountId}:${OTLPIngestApi}/*'
+
+# =============================================================================
+# Outputs
+# =============================================================================
+Outputs:
+  OpenSearchEndpoint:
+    Description: OpenSearch domain endpoint URL
+    Value: !Sub 'https://${OpenSearchDomain.DomainEndpoint}'
+
+  OSISTraceIngestEndpoint:
+    Description: OSIS trace ingest endpoint (configure as OTEL_EXPORTER_OTLP_TRACES_ENDPOINT in your agent)
+    Value: !Join
+      - ''
+      - - 'https://'
+        - !Select [0, !GetAtt OSISTracePipeline.IngestEndpointUrls]
+
+  OSISLogsIngestEndpoint:
+    Description: OSIS logs ingest endpoint (configure as OTEL_EXPORTER_OTLP_LOGS_ENDPOINT in your agent)
+    Value: !Join
+      - ''
+      - - 'https://'
+        - !Select [0, !GetAtt OSISLogsPipeline.IngestEndpointUrls]
+
+  Region:
+    Description: AWS Region where the stack was deployed
+    Value: !Ref 'AWS::Region'
+
+  IngestionRoleArn:
+    Description: IAM role ARN for agents to assume when pushing telemetry via SigV4
+    Value: !GetAtt IngestionRole.Arn
+
+  OTLPIngestEndpoint:
+    Description: >-
+      HTTPS endpoint for OTLP trace/log ingestion (no SigV4 needed on client).
+      Set OTEL_EXPORTER_OTLP_ENDPOINT to this value in your agent config.
+    Value: !Sub 'https://${OTLPIngestApi}.execute-api.${AWS::Region}.amazonaws.com'
+
+  AgentHealthConfigJSON:
+    Description: >-
+      Copy this JSON block into your agent-health.config.json file,
+      or run: npx @opensearch-project/agent-health configure --from-stack AgentHealthObservability
+    Value: !Sub |
+      {
+        "observability": {
+          "endpoint": "https://${OpenSearchDomain.DomainEndpoint}",
+          "authType": "sigv4",
+          "awsRegion": "${AWS::Region}",
+          "awsService": "es",
+          "tlsSkipVerify": false
+        }
+      }