@aws/agentcore 1.0.0-preview.8 → 1.0.0-rc.1

package/dist/assets/cdk/README.md

@@ -20,6 +20,9 @@ This CDK project is managed by the AgentCore CLI. It deploys your agent infrastr
 
 You typically don't need to interact with this directory directly. The AgentCore CLI handles synthesis and deployment:
 
+<!-- TODO: revisit these commands once the project CLI surface is final —
+     they may need a project prefix (e.g. --project / cwd) to disambiguate. -->
+
 ```bash
 agentcore deploy    # synthesizes and deploys via CDK
 agentcore status    # checks deployment status

package/dist/assets/cdk/bin/cdk.ts

@@ -1,6 +1,6 @@
 #!/usr/bin/env node
-import { AgentCoreStack } from '../lib/cdk-stack';
-import { ConfigIO, type AwsDeploymentTarget } from '@aws/agentcore-cdk';
+import { AgentCoreStack, type HarnessConfig } from '../lib/cdk-stack';
+import { ConfigIO, HarnessSpecSchema, type AwsDeploymentTarget } from '@aws/agentcore-cdk';
 import { App, type Environment } from 'aws-cdk-lib';
 import * as path from 'path';
 import * as fs from 'fs';
@@ -20,71 +20,79 @@ function toStackName(projectName: string, targetName: string): string {
   return `AgentCore-${sanitize(projectName)}-${sanitize(targetName)}`;
 }
 
-async function main() {
-  // Config root is parent of cdk/ directory. The CLI sets process.cwd() to agentcore/cdk/.
-  const configRoot = path.resolve(process.cwd(), '..');
-  const configIO = new ConfigIO({ baseDir: configRoot });
+// The vended CDK project compiles against the published @aws/agentcore-cdk schema
+// type, which may lag the CLI's own AgentCoreProjectSpec (e.g. payments, harnesses,
+// gateway fields). This alias documents each read of those not-yet-published fields.
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+type SpecWithLatestFields = any;
 
-  const spec = await configIO.readProjectSpec();
-  const targets = await configIO.readAWSDeploymentTargets();
-
-  // Extract MCP configuration from project spec.
-  // Gateway fields are stored in agentcore.json but may not yet be on the
-  // AgentCoreProjectSpec type from @aws/agentcore-cdk, so we read them
-  // dynamically and cast the resulting object.
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  const specAny = spec as any;
-  const mcpSpec = specAny.agentCoreGateways?.length
+// Extract MCP configuration from the project spec. Gateway fields are stored in
+// agentcore.json but may not yet be on the published spec type, so they are read
+// off the loosened alias.
+function resolveMcpSpec(spec: SpecWithLatestFields) {
+  return spec.agentCoreGateways?.length
     ? {
-        agentCoreGateways: specAny.agentCoreGateways,
-        mcpRuntimeTools: specAny.mcpRuntimeTools,
-        unassignedTargets: specAny.unassignedTargets,
+        agentCoreGateways: spec.agentCoreGateways,
+        mcpRuntimeTools: spec.mcpRuntimeTools,
+        unassignedTargets: spec.unassignedTargets,
       }
     : undefined;
+}
 
-  // Read deployed state for credential ARNs (populated by pre-deploy identity setup)
-  let deployedState: Record<string, unknown> | undefined;
-  try {
-    deployedState = JSON.parse(fs.readFileSync(path.join(configRoot, '.cli', 'deployed-state.json'), 'utf8'));
-  } catch {
-    // Deployed state may not exist on first deploy
-  }
-
-  if (targets.length === 0) {
-    throw new Error('No deployment targets configured. Please define targets in agentcore/aws-targets.json');
+// Read non-S3 KB connector-config files and return their parsed contents keyed by
+// the data source's connectorConfigFile path. The L3 does not read files; it
+// expects these parsed connectorParameters verbatim.
+function resolveConnectorParametersByFile(
+  spec: SpecWithLatestFields,
+  projectRoot: string
+): Record<string, Record<string, unknown>> {
+  const connectorParametersByFile: Record<string, Record<string, unknown>> = {};
+  for (const kb of spec.knowledgeBases ?? []) {
+    for (const ds of kb.dataSources ?? []) {
+      if (ds.type !== 'S3' && ds.connectorConfigFile) {
+        const abs = path.resolve(projectRoot, ds.connectorConfigFile);
+        try {
+          connectorParametersByFile[ds.connectorConfigFile] = JSON.parse(fs.readFileSync(abs, 'utf-8'));
+        } catch (err) {
+          throw new Error(
+            `Could not read connector config '${ds.connectorConfigFile}' for knowledge base '${kb.name}' at ${abs}: ${err instanceof Error ? err.message : err}`
+          );
+        }
+      }
+    }
   }
+  return connectorParametersByFile;
+}
 
-  // Read harness configs for role creation.
-  // Harness fields may not yet be on the AgentCoreProjectSpec type from @aws/agentcore-cdk,
-  // so we read them dynamically via specAny (same pattern as gateways above).
-  // Harness paths in agentcore.json are relative to the project root (parent of agentcore/).
-  const projectRoot = path.resolve(configRoot, '..');
-  const harnessConfigs: {
-    name: string;
-    executionRoleArn?: string;
-    memoryName?: string;
-    containerUri?: string;
-    hasDockerfile?: boolean;
-    dockerfile?: string;
-    codeLocation?: string;
-    tools?: { type: string; name: string }[];
-    apiKeyArn?: string;
-  }[] = [];
-  for (const entry of specAny.harnesses ?? []) {
+// Synthesize a HarnessConfig for each harness entry in the spec. The full validated
+// spec drives the AWS::BedrockAgentCore::Harness CFN resource; the role-scoped
+// fields drive the IAM role + container build.
+function resolveHarnessConfigs(spec: SpecWithLatestFields, projectRoot: string): HarnessConfig[] {
+  const harnessConfigs: HarnessConfig[] = [];
+  for (const entry of spec.harnesses ?? []) {
     const harnessDir = path.resolve(projectRoot, entry.path);
     const harnessPath = path.resolve(harnessDir, 'harness.json');
     try {
-      const harnessSpec = JSON.parse(fs.readFileSync(harnessPath, 'utf-8'));
+      const harnessSpec = HarnessSpecSchema.parse(JSON.parse(fs.readFileSync(harnessPath, 'utf-8')));
       harnessConfigs.push({
         name: entry.name,
         executionRoleArn: harnessSpec.executionRoleArn,
-        memoryName: harnessSpec.memory?.name,
+        // Only an `existing` memory ref carries a name to wire IAM against; managed memory is
+        // owned by the harness (no sibling) and disabled has none — both resolve to undefined.
+        memoryName: harnessSpec.memory?.mode === 'existing' ? harnessSpec.memory.name : undefined,
         containerUri: harnessSpec.containerUri,
         hasDockerfile: !!harnessSpec.dockerfile,
         dockerfile: harnessSpec.dockerfile,
         codeLocation: harnessSpec.dockerfile ? harnessDir : undefined,
         tools: harnessSpec.tools,
+        skills: harnessSpec.skills,
         apiKeyArn: harnessSpec.model?.apiKeyArn,
+        efsAccessPoints: harnessSpec.efsAccessPoints,
+        s3AccessPoints: harnessSpec.s3AccessPoints,
+        apiFormat: harnessSpec.model?.apiFormat,
+        // Full spec + dir drive the AWS::BedrockAgentCore::Harness CFN resource.
+        spec: harnessSpec,
+        harnessDir,
       });
     } catch (err) {
       throw new Error(
@@ -92,32 +100,74 @@ async function main() {
       );
     }
   }
+  return harnessConfigs;
+}
+
+async function main() {
+  // Config root is parent of cdk/ directory. The CLI sets process.cwd() to agentcore/cdk/.
+  const configRoot = path.resolve(process.cwd(), '..');
+  const configIO = new ConfigIO({ baseDir: configRoot });
+
+  const spec = await configIO.readProjectSpec();
+  const targets = await configIO.readAWSDeploymentTargets();
+
+  // `project build` runs before a project has anywhere to deploy, so an empty target
+  // list is not an error: it synthesizes a single environment-agnostic stack, which is
+  // enough to typecheck the app and produce a template. Only a stack synthesized for a
+  // real target is a deploy candidate; the target tag below is what marks one.
+  const stackTargets: (AwsDeploymentTarget | undefined)[] = targets.length > 0 ? targets : [undefined];
+
+  const specAny: SpecWithLatestFields = spec;
+  const projectRoot = path.resolve(configRoot, '..');
+
+  const mcpSpec = resolveMcpSpec(specAny);
+  const connectorParametersByFile = resolveConnectorParametersByFile(specAny, projectRoot);
+  const harnessConfigs = resolveHarnessConfigs(specAny, projectRoot);
+
+  // Read deployed state for credential ARNs (populated by pre-deploy identity setup).
+  // Under agentcore/.cli/ to match the released CLI's location.
+  let deployedState: Record<string, unknown> | undefined;
+  try {
+    deployedState = JSON.parse(fs.readFileSync(path.join(configRoot, '.cli', 'deployed-state.json'), 'utf8'));
+  } catch (err) {
+    // A missing file is the normal first-deploy case. A malformed one is not:
+    // surface it rather than silently synthesizing without the credential ARNs
+    // it holds (which would drop them from the stack).
+    if ((err as NodeJS.ErrnoException).code !== 'ENOENT') throw err;
+  }
 
   const app = new App();
 
-  for (const target of targets) {
-    const env = toEnvironment(target);
-    const stackName = toStackName(spec.name, target.name);
+  for (const target of stackTargets) {
+    // An environment-agnostic stack resolves its account and region from CloudFormation
+    // pseudo-parameters at deploy time instead of pinning them at synth time.
+    const env = target ? toEnvironment(target) : undefined;
+    const stackName = target ? toStackName(spec.name, target.name) : `AgentCore-${sanitize(spec.name)}`;
 
     // Extract credentials from deployed state for this target
     const targetState = (deployedState as Record<string, unknown>)?.targets as
-      | Record<string, Record<string, unknown>>
-      | undefined;
-    const targetResources = targetState?.[target.name]?.resources as Record<string, unknown> | undefined;
+      Record<string, Record<string, unknown>> | undefined;
+    const targetResources = target
+      ? (targetState?.[target.name]?.resources as Record<string, unknown> | undefined)
+      : undefined;
     const credentials = targetResources?.credentials as
-      | Record<string, { credentialProviderArn: string; clientSecretArn?: string }>
-      | undefined;
+      Record<string, { credentialProviderArn: string; clientSecretArn?: string }> | undefined;
 
     new AgentCoreStack(app, stackName, {
       spec,
       mcpSpec,
       credentials,
+      connectorParametersByFile,
       harnesses: harnessConfigs.length > 0 ? harnessConfigs : undefined,
       env,
-      description: `AgentCore stack for ${spec.name} deployed to ${target.name} (${target.region})`,
+      description: target
+        ? `AgentCore stack for ${spec.name} deployed to ${target.name} (${target.region})`
+        : `AgentCore stack for ${spec.name} (no deployment target configured)`,
+      // Only a stack synthesized for a real target carries the target tag, which is
+      // how deploy selects the stack to ship.
       tags: {
         'agentcore:project-name': spec.name,
-        'agentcore:target-name': target.name,
+        ...(target ? { 'agentcore:target-name': target.name } : {}),
       },
     });
   }
@@ -127,5 +177,5 @@ async function main() {
 
 main().catch((error: unknown) => {
   console.error('AgentCore CDK synthesis failed:', error instanceof Error ? error.message : error);
-  process.exitCode = 1;
+  process.exit(1);
 });

package/dist/assets/cdk/lib/cdk-stack.ts

@@ -1,23 +1,20 @@
 import {
   AgentCoreApplication,
   AgentCoreMcp,
+  AgentCorePayments,
   type AgentCoreProjectSpec,
   type AgentCoreMcpSpec,
+  type HarnessDeploymentConfig,
 } from '@aws/agentcore-cdk';
 import { CfnOutput, Stack, type StackProps } from 'aws-cdk-lib';
 import { Construct } from 'constructs';
 
-export interface HarnessConfig {
-  name: string;
-  executionRoleArn?: string;
-  memoryName?: string;
-  containerUri?: string;
-  hasDockerfile?: boolean;
-  dockerfile?: string;
-  codeLocation?: string;
-  tools?: { type: string; name: string }[];
-  apiKeyArn?: string;
-}
+/**
+ * Harness deployment config: role-scoped fields (for IAM role + container build)
+ * plus the full validated spec + its config directory so the L3 construct can
+ * synthesize the AWS::BedrockAgentCore::Harness resource.
+ */
+export type HarnessConfig = HarnessDeploymentConfig;
 
 export interface AgentCoreStackProps extends StackProps {
   /**
@@ -33,13 +30,14 @@ export interface AgentCoreStackProps extends StackProps {
    */
   credentials?: Record<string, { credentialProviderArn: string; clientSecretArn?: string }>;
   /**
-   * Harness role configurations. Each entry creates an IAM execution role for a harness.
-   *
-   * When `hasDockerfile` is true and `codeLocation` is provided (without an explicit
-   * `containerUri`), the L3 construct builds and pushes a container image via CodeBuild
-   * and emits its URI as a stack output for the post-CDK harness deployer.
+   * Harness role configurations.
    */
   harnesses?: HarnessConfig[];
+  /**
+   * Parsed connectorParameters for non-S3 KB data sources, keyed by
+   * connectorConfigFile path. Forwarded to AgentCoreApplication.
+   */
+  connectorParametersByFile?: Record<string, Record<string, unknown>>;
 }
 
 /**
@@ -55,12 +53,25 @@ export class AgentCoreStack extends Stack {
   constructor(scope: Construct, id: string, props: AgentCoreStackProps) {
     super(scope, id, props);
 
-    const { spec, mcpSpec, credentials, harnesses } = props;
+    const { spec, mcpSpec, credentials, harnesses, connectorParametersByFile } = props;
 
     // Create AgentCoreApplication with all agents and harness roles
-    this.application = new AgentCoreApplication(this, 'Application', {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const appProps: Record<string, unknown> = { spec };
+    if (harnesses?.length) {
+      appProps.harnesses = harnesses;
+    }
+    if (connectorParametersByFile && Object.keys(connectorParametersByFile).length > 0) {
+      appProps.connectorParametersByFile = connectorParametersByFile;
+    }
+    if (credentials) {
+      appProps.credentials = credentials;
+    }
+    this.application = new AgentCoreApplication(this, 'Application', appProps as any);
+    new AgentCorePayments(this, 'Payments', {
       spec,
-      harnesses: harnesses?.length ? harnesses : undefined,
+      credentials,
+      agentCoreApplication: this.application,
     });
 
     // Create AgentCoreMcp if there are gateways configured

package/dist/assets/cdk/package.json

@@ -14,17 +14,17 @@
     "format:check": "prettier --check ."
   },
   "devDependencies": {
-    "@types/jest": "^29.5.14",
-    "@types/node": "^24.10.1",
-    "jest": "^29.7.0",
-    "ts-jest": "^29.2.5",
-    "aws-cdk": "2.1100.1",
-    "prettier": "^3.4.2",
+    "@types/jest": "~29.5.14",
+    "@types/node": "~24.13.3",
+    "jest": "~29.7.0",
+    "ts-jest": "~29.4.11",
+    "aws-cdk": "~2.1126.0",
+    "prettier": "~3.9.5",
     "typescript": "~5.9.3"
   },
   "dependencies": {
-    "@aws/agentcore-cdk": "^0.1.0-alpha.19",
-    "aws-cdk-lib": "^2.248.0",
-    "constructs": "^10.0.0"
+    "@aws/agentcore-cdk": "0.1.0-alpha.52",
+    "aws-cdk-lib": "~2.266.0",
+    "constructs": "~10.7.0"
   }
 }

package/dist/assets/cdk/test/cdk.test.ts

@@ -1,6 +1,29 @@
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
 import * as cdk from 'aws-cdk-lib';
-import { Template } from 'aws-cdk-lib/assertions';
-import { AgentCoreStack } from '../lib/cdk-stack';
+import { Match, Template } from 'aws-cdk-lib/assertions';
+
+const originalCwd = process.cwd();
+const originalInitCwd = process.env.INIT_CWD;
+const testRoot = mkdtempSync(join(tmpdir(), 'agentcore-cdk-test-'));
+const testConfigDir = join(testRoot, 'agentcore');
+let AgentCoreStack: typeof import('../lib/cdk-stack').AgentCoreStack;
+
+beforeAll(async () => {
+  process.chdir(testRoot);
+  process.env.INIT_CWD = testRoot;
+  mkdirSync(testConfigDir, { recursive: true });
+  writeFileSync(join(testConfigDir, 'agentcore.json'), '{}');
+  ({ AgentCoreStack } = await import('../lib/cdk-stack'));
+});
+
+afterAll(() => {
+  process.chdir(originalCwd);
+  if (originalInitCwd === undefined) delete process.env.INIT_CWD;
+  else process.env.INIT_CWD = originalInitCwd;
+  rmSync(testRoot, { recursive: true, force: true });
+});
 
 test('AgentCoreStack synthesizes with empty spec', () => {
   const app = new cdk.App();
@@ -16,9 +39,12 @@ test('AgentCoreStack synthesizes with empty spec', () => {
       onlineEvalConfigs: [],
       configBundles: [],
       policyEngines: [],
+      payments: [],
       agentCoreGateways: [],
       mcpRuntimeTools: [],
       unassignedTargets: [],
+      datasets: [],
+      knowledgeBases: [],
     },
   });
   const template = Template.fromStack(stack);
@@ -26,3 +52,69 @@ test('AgentCoreStack synthesizes with empty spec', () => {
     Description: 'Name of the CloudFormation Stack',
   });
 });
+
+test('AgentCoreStack synthesizes manual and Quick Create payment connectors', () => {
+  const app = new cdk.App();
+  const stack = new AgentCoreStack(app, 'TestStack', {
+    spec: {
+      name: 'testproject',
+      version: 1,
+      managedBy: 'CDK' as const,
+      runtimes: [],
+      memories: [],
+      credentials: [
+        {
+          authorizerType: 'PaymentCredentialProvider',
+          name: 'coinbase',
+          provider: 'CoinbaseCDP',
+        },
+      ],
+      evaluators: [],
+      onlineEvalConfigs: [],
+      configBundles: [],
+      policyEngines: [],
+      payments: [
+        {
+          name: 'Payments',
+          authorizerType: 'AWS_IAM',
+          connectors: [
+            {
+              name: 'Manual',
+              provider: 'CoinbaseCDP',
+              credentialName: 'coinbase',
+            },
+            {
+              name: 'Quick',
+              provider: 'CoinbaseCDP',
+              provisionMode: 'QUICK_CREATE',
+            },
+          ],
+        },
+      ],
+      agentCoreGateways: [],
+      mcpRuntimeTools: [],
+      unassignedTargets: [],
+      datasets: [],
+      knowledgeBases: [],
+    },
+    credentials: {
+      coinbase: {
+        credentialProviderArn:
+          'arn:aws:bedrock-agentcore:us-east-1:123456789012:token-vault/default/paymentcredentialprovider/coinbase',
+      },
+    },
+  });
+  const template = Template.fromStack(stack);
+
+  template.resourceCountIs('AWS::BedrockAgentCore::PaymentConnector', 2);
+  template.hasResourceProperties('AWS::BedrockAgentCore::PaymentConnector', {
+    ConnectorName: 'Manual',
+    ProvisionMode: Match.absent(),
+  });
+  template.hasResourceProperties('AWS::BedrockAgentCore::PaymentConnector', {
+    ConnectorName: 'Quick',
+    ConnectorType: 'CoinbaseCDP',
+    ProvisionMode: 'QUICK_CREATE',
+    CredentialProviderConfigurations: [],
+  });
+});

package/dist/assets/evaluators/autoevals-lambda/README.md

@@ -0,0 +1,23 @@
+# {{ Name }}
+
+An AgentCore **code-based evaluator** backed by
+[autoevals](https://github.com/braintrustdata/autoevals) — scores each session
+with autoevals' `{{ EvaluatorClass }}` scorer.
+
+## What's here
+
+- `lambda_function.py` — wraps `{{ EvaluatorClass }}` in an `AutoEvalsAdapter`
+  behind the standard `@custom_code_based_evaluator()` handler. With a Bedrock
+  judge model set, autoevals grades via a LiteLLM client → Bedrock.
+- `pyproject.toml` — autoevals + judge dependencies, managed with
+  [uv](https://docs.astral.sh/uv/).
+- `execution-role-policy.json` — grants the Lambda `bedrock:InvokeModel` for the
+  judge model.
+
+## Customize
+
+- Change the scorer or its arguments in `lambda_function.py`.
+- Scorers like `Factuality` / `ClosedQA` / `SQL` need an expected/reference
+  output — provide it when you invoke the evaluator.
+
+`agentcore project deploy` packages this directory into the evaluator Lambda.

package/dist/assets/evaluators/autoevals-lambda/execution-role-policy.json

@@ -0,0 +1,15 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
+      "Resource": "arn:*:logs:*:*:log-group:/aws/lambda/*"
+    },
+    {
+      "Effect": "Allow",
+      "Action": ["bedrock:InvokeModel"],
+      "Resource": "*"
+    }
+  ]
+}

package/dist/assets/evaluators/autoevals-lambda/lambda_function.py

@@ -0,0 +1,37 @@
+{{#if ModelProviderBedrock}}
+import os
+
+# litellm's Bedrock provider reads AWS_REGION_NAME; Lambda only sets AWS_REGION/AWS_DEFAULT_REGION.
+os.environ.setdefault("AWS_REGION_NAME", os.environ.get("AWS_REGION", "us-west-2"))
+
+from autoevals import {{ EvaluatorClass }}, init
+from autoevals.litellm import LiteLLMClient
+
+from bedrock_agentcore.evaluation.custom_code_based_evaluators import (
+    EvaluatorInput,
+    EvaluatorOutput,
+    custom_code_based_evaluator,
+)
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.autoevals import AutoEvalsAdapter
+
+client = LiteLLMClient()
+init(client=client, default_model="bedrock/{{ Model }}")
+
+adapter = AutoEvalsAdapter(metric={{ EvaluatorClass }}(client=client, model="bedrock/{{ Model }}"){{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}})
+{{else}}
+from autoevals import {{ EvaluatorClass }}
+
+from bedrock_agentcore.evaluation.custom_code_based_evaluators import (
+    EvaluatorInput,
+    EvaluatorOutput,
+    custom_code_based_evaluator,
+)
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.autoevals import AutoEvalsAdapter
+
+adapter = AutoEvalsAdapter(metric={{ EvaluatorClass }}({{#if Model}}model="{{ Model }}"{{/if}}){{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}})
+{{/if}}
+
+
+@custom_code_based_evaluator()
+def handler(evaluator_input: EvaluatorInput, context) -> EvaluatorOutput:
+    return adapter(evaluator_input, context)

package/dist/assets/evaluators/autoevals-lambda/pyproject.toml

@@ -0,0 +1,23 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "{{ Name }}"
+version = "0.1.0"
+description = "AgentCore Code-Based Evaluator (Autoevals)"
+requires-python = ">=3.10"
+dependencies = [
+    # 1.20.0 is the first release shipping third_party.autoevals.AutoEvalsAdapter;
+    # the extra owns the autoevals version, so pinning it here only conflicts.
+    "bedrock-agentcore[autoevals]>=1.20.0,<2.0.0",
+{{#if ModelProviderBedrock}}
+    # autoevals grades via LiteLLMClient -> Bedrock (Converse); litellm replaces the openai judge
+    "litellm>=1.60,<1.85",
+{{else}}
+    "openai>=1.0.0,<2.0.0",
+{{/if}}
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["."]

package/dist/assets/evaluators/deepeval-lambda/README.md

@@ -0,0 +1,23 @@
+# {{ Name }}
+
+An AgentCore **code-based evaluator** backed by
+[DeepEval](https://docs.confident-ai.com/) — scores each session with DeepEval's
+`{{ EvaluatorClass }}` metric, judged by Amazon Bedrock.
+
+## What's here
+
+- `lambda_function.py` — wraps `{{ EvaluatorClass }}` in a `DeepEvalAdapter`
+  behind the standard `@custom_code_based_evaluator()` handler.
+- `pyproject.toml` — DeepEval + Bedrock dependencies, managed with
+  [uv](https://docs.astral.sh/uv/).
+- `execution-role-policy.json` — grants the Lambda `bedrock:InvokeModel` for the
+  judge model; add more if your metric needs it.
+
+## Customize
+
+- Swap the metric or tune its threshold in `lambda_function.py`.
+- Some DeepEval metrics need retrieval context or a reference/expected output —
+  supply those when you invoke the evaluator, or the metric returns
+  `MISSING_REQUIRED_FIELD`.
+
+`agentcore project deploy` packages this directory into the evaluator Lambda.

package/dist/assets/evaluators/deepeval-lambda/execution-role-policy.json

@@ -0,0 +1,15 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
+      "Resource": "arn:*:logs:*:*:log-group:/aws/lambda/*"
+    },
+    {
+      "Effect": "Allow",
+      "Action": ["bedrock:InvokeModel"],
+      "Resource": "*"
+    }
+  ]
+}

package/dist/assets/evaluators/deepeval-lambda/lambda_function.py

@@ -0,0 +1,29 @@
+import os
+
+os.environ.setdefault("DEEPEVAL_RESULTS_FOLDER", "/tmp/.deepeval")
+os.environ.setdefault("DEEPEVAL_TELEMETRY_OPT_OUT", "YES")
+os.chdir("/tmp")
+
+{{#if ModelProviderBedrock}}
+from deepeval.models import AmazonBedrockModel
+{{/if}}
+from deepeval.metrics import {{ EvaluatorClass }}
+
+from bedrock_agentcore.evaluation.custom_code_based_evaluators import (
+    EvaluatorInput,
+    EvaluatorOutput,
+    custom_code_based_evaluator,
+)
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.deepeval import DeepEvalAdapter
+
+{{#if ModelProviderBedrock}}
+model = AmazonBedrockModel(model="{{ Model }}", region=os.environ.get("AWS_REGION", "us-west-2"))
+adapter = DeepEvalAdapter(metric={{ EvaluatorClass }}(model=model{{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}}))
+{{else}}
+adapter = DeepEvalAdapter(metric={{ EvaluatorClass }}({{{ EvaluatorParams }}}))
+{{/if}}
+
+
+@custom_code_based_evaluator()
+def handler(evaluator_input: EvaluatorInput, context) -> EvaluatorOutput:
+    return adapter(evaluator_input, context)

package/dist/assets/evaluators/deepeval-lambda/pyproject.toml

@@ -0,0 +1,22 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "{{ Name }}"
+version = "0.1.0"
+description = "AgentCore Code-Based Evaluator (DeepEval)"
+requires-python = ">=3.10"
+dependencies = [
+    # 1.20.0 is the first release shipping third_party.deepeval.DeepEvalAdapter;
+    # the extra owns the deepeval version, so pinning it here only conflicts.
+    "bedrock-agentcore[deepeval]>=1.20.0,<2.0.0",
+{{#if ModelProviderBedrock}}
+    # deepeval's AmazonBedrockModel imports aiobotocore at runtime; only 3.x
+    # allows the botocore that bedrock-agentcore requires.
+    "aiobotocore>=3.0.0,<4.0.0",
+{{/if}}
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["."]