@aws/agentcore 1.0.0-preview.24 → 1.0.0-preview.25

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (27) hide show
  1. package/dist/assets/README.md +5 -0
  2. package/dist/assets/__tests__/__snapshots__/assets.snapshot.test.ts.snap +115 -9
  3. package/dist/assets/__tests__/input-validation.test.ts +44 -0
  4. package/dist/assets/agents/AGENTS.md +2 -0
  5. package/dist/assets/evaluators/autoevals-lambda/execution-role-policy.json +15 -0
  6. package/dist/assets/evaluators/autoevals-lambda/lambda_function.py +37 -0
  7. package/dist/assets/evaluators/autoevals-lambda/pyproject.toml +22 -0
  8. package/dist/assets/evaluators/deepeval-lambda/execution-role-policy.json +15 -0
  9. package/dist/assets/evaluators/deepeval-lambda/lambda_function.py +29 -0
  10. package/dist/assets/evaluators/deepeval-lambda/pyproject.toml +19 -0
  11. package/dist/assets/python/http/autogen/base/README.md +5 -0
  12. package/dist/assets/python/http/autogen/base/main.py +2 -0
  13. package/dist/assets/python/http/googleadk/base/README.md +5 -0
  14. package/dist/assets/python/http/googleadk/base/main.py +2 -0
  15. package/dist/assets/python/http/langchain_langgraph/base/README.md +5 -0
  16. package/dist/assets/python/http/langchain_langgraph/base/main.py +4 -0
  17. package/dist/assets/python/http/openaiagents/base/README.md +5 -0
  18. package/dist/assets/python/http/openaiagents/base/main.py +2 -0
  19. package/dist/assets/python/http/strands/base/README.md +6 -0
  20. package/dist/assets/python/http/strands/base/main.py +40 -4
  21. package/dist/assets/typescript/http/strands/base/README.md +5 -0
  22. package/dist/assets/typescript/http/strands/base/main.ts +8 -3
  23. package/dist/assets/typescript/http/vercelai/base/README.md +5 -0
  24. package/dist/assets/typescript/http/vercelai/base/main.ts +8 -2
  25. package/dist/cli/index.mjs +421 -417
  26. package/npm-shrinkwrap.json +8 -8
  27. package/package.json +1 -1
@@ -37,6 +37,11 @@ Run your agent locally:
37
37
  agentcore dev
38
38
  ```
39
39
 
40
+ ### Validate Invocation Input
41
+
42
+ Validate runtime invocation payloads before forwarding them to an agent framework. Keep user prompts typed as strings
43
+ and pass only prompt text to the agent.
44
+
40
45
  ### Deployment
41
46
 
42
47
  Deploy to AWS:
@@ -742,6 +742,12 @@ exports[`Assets Directory Snapshots > File listing > should match the expected f
742
742
  "container/typescript/dockerignore.template",
743
743
  "datasets/predefined-v1.jsonl",
744
744
  "datasets/simulated-v1.jsonl",
745
+ "evaluators/autoevals-lambda/execution-role-policy.json",
746
+ "evaluators/autoevals-lambda/lambda_function.py",
747
+ "evaluators/autoevals-lambda/pyproject.toml",
748
+ "evaluators/deepeval-lambda/execution-role-policy.json",
749
+ "evaluators/deepeval-lambda/lambda_function.py",
750
+ "evaluators/deepeval-lambda/pyproject.toml",
745
751
  "evaluators/python-lambda/execution-role-policy.json",
746
752
  "evaluators/python-lambda/lambda_function.py",
747
753
  "evaluators/python-lambda/pyproject.toml",
@@ -3329,6 +3335,11 @@ file defines a Starlette ASGI app with the AutoGen framework running within.
3329
3335
 
3330
3336
  \`model/load.py\` instantiates your chosen model provider.
3331
3337
 
3338
+ ## Input Validation
3339
+
3340
+ Validate invocation input before forwarding it to the agent framework. Keep plain prompts typed as strings and pass
3341
+ only prompt text to the agent.
3342
+
3332
3343
  ## Environment Variables
3333
3344
 
3334
3345
  | Variable | Required | Description |
@@ -3532,6 +3543,8 @@ async def invoke(payload, context):
3532
3543
 
3533
3544
  # Process the user prompt
3534
3545
  prompt = payload.get("prompt", "What can you help me with?")
3546
+ if not isinstance(prompt, str):
3547
+ raise ValueError("prompt must be a string")
3535
3548
  session_id = getattr(context, "session_id", "default-session")
3536
3549
 
3537
3550
  # Reuse the per-session agent (preserves conversation history)
@@ -3785,6 +3798,11 @@ file defines a Starlette ASGI app with the Google ADK framework running within.
3785
3798
 
3786
3799
  \`model/load.py\` instantiates your chosen model provider (Gemini).
3787
3800
 
3801
+ ## Input Validation
3802
+
3803
+ Validate invocation input before forwarding it to the agent framework. Keep plain prompts typed as strings and pass
3804
+ only prompt text to the agent.
3805
+
3788
3806
  ## Environment Variables
3789
3807
 
3790
3808
  | Variable | Required | Description |
@@ -4052,6 +4070,8 @@ async def invoke(payload, context):
4052
4070
 
4053
4071
  # Process the user prompt
4054
4072
  prompt = payload.get("prompt", "What can you help me with?")
4073
+ if not isinstance(prompt, str):
4074
+ raise ValueError("prompt must be a string")
4055
4075
  session_id = getattr(context, "session_id", "default_session")
4056
4076
  user_id = payload.get("user_id", "default_user")
4057
4077
 
@@ -4245,6 +4265,11 @@ file defines a Starlette ASGI app with the LangChain/LangGraph framework running
4245
4265
 
4246
4266
  \`model/load.py\` instantiates your chosen model provider.
4247
4267
 
4268
+ ## Input Validation
4269
+
4270
+ Validate invocation input before forwarding it to the agent framework. Keep plain prompts typed as strings and pass
4271
+ only prompt text to the agent.
4272
+
4248
4273
  ## Environment Variables
4249
4274
 
4250
4275
  | Variable | Required | Description |
@@ -4504,6 +4529,8 @@ async def invoke(payload, context):
4504
4529
 
4505
4530
  # Process the user prompt
4506
4531
  prompt = payload.get("prompt", "What can you help me with?")
4532
+ if not isinstance(prompt, str):
4533
+ raise ValueError("prompt must be a string")
4507
4534
  session_id = getattr(context, "session_id", "default-session")
4508
4535
  touch_thread(session_id)
4509
4536
  log.info(f"Agent input: {prompt}")
@@ -4523,6 +4550,8 @@ async def invoke(payload, context):
4523
4550
 
4524
4551
  # Process the user prompt
4525
4552
  prompt = payload.get("prompt", "What can you help me with?")
4553
+ if not isinstance(prompt, str):
4554
+ raise ValueError("prompt must be a string")
4526
4555
  session_id = getattr(context, "session_id", "default-session")
4527
4556
  touch_thread(session_id)
4528
4557
  log.info(f"Agent input: {prompt}")
@@ -4823,6 +4852,11 @@ file defines a Starlette ASGI app with the OpenAI Agents SDK framework running w
4823
4852
 
4824
4853
  \`model/load.py\` instantiates your chosen model provider (OpenAI).
4825
4854
 
4855
+ ## Input Validation
4856
+
4857
+ Validate invocation input before forwarding it to the agent framework. Keep plain prompts typed as strings and pass
4858
+ only prompt text to the agent.
4859
+
4826
4860
  ## Environment Variables
4827
4861
 
4828
4862
  | Variable | Required | Description |
@@ -5083,6 +5117,8 @@ async def invoke(payload, context):
5083
5117
 
5084
5118
  # Process the user prompt
5085
5119
  prompt = payload.get("prompt", "What can you help me with?")
5120
+ if not isinstance(prompt, str):
5121
+ raise ValueError("prompt must be a string")
5086
5122
  session_id = getattr(context, "session_id", "default-session")
5087
5123
  session = get_session(session_id)
5088
5124
 
@@ -5268,6 +5304,12 @@ file defines a Starlette ASGI app with the chosen Agent framework SDK running wi
5268
5304
 
5269
5305
  \`model/load.py\` instantiates your chosen model provider.
5270
5306
 
5307
+ ## Input Validation
5308
+
5309
+ Validate invocation input before forwarding it to Strands. Keep plain prompts typed as strings. If the app accepts a
5310
+ caller-supplied message history, retain \`strip_trailing_tool_use()\`, which normalizes the history tail before
5311
+ invoking the agent.
5312
+
5271
5313
  ## Environment Variables
5272
5314
 
5273
5315
  | Variable | Required | Description |
@@ -5824,17 +5866,53 @@ get_or_create_agent = agent_factory()
5824
5866
  {{/if}}
5825
5867
 
5826
5868
 
5869
+ def strip_trailing_tool_use(messages: Any) -> list[dict]:
5870
+ """Strip toolUse blocks from the tail until the last message has none."""
5871
+ if not isinstance(messages, list):
5872
+ raise ValueError("messages must be a list")
5873
+
5874
+ messages = list(messages)
5875
+ while messages:
5876
+ last = messages[-1]
5877
+ if not isinstance(last, dict):
5878
+ raise ValueError("each message must be an object")
5879
+ original_content = last.get("content", [])
5880
+ if not isinstance(original_content, list) or not all(isinstance(block, dict) for block in original_content):
5881
+ raise ValueError("each message content value must be a list of content blocks")
5882
+
5883
+ content = [block for block in original_content if "toolUse" not in block]
5884
+ if len(content) == len(original_content):
5885
+ break
5886
+ if content:
5887
+ messages[-1] = {**last, "content": content}
5888
+ break
5889
+ messages.pop()
5890
+
5891
+ return messages
5892
+
5893
+
5827
5894
  def _extract_prompt(payload: dict):
5828
- """Accept harness-style messages[], tool_results[], or plain prompt string payloads."""
5895
+ """Accept validated harness messages, tool results, or a plain prompt string."""
5896
+ if not isinstance(payload, dict):
5897
+ raise ValueError("payload must be a JSON object")
5829
5898
  if "messages" in payload:
5830
- return payload["messages"]
5899
+ return strip_trailing_tool_use(payload["messages"])
5831
5900
  if "tool_results" in payload:
5901
+ tool_results = payload["tool_results"]
5902
+ if not isinstance(tool_results, list) or not all(
5903
+ isinstance(tool_result, dict) and isinstance(tool_result.get("toolUseId"), str)
5904
+ for tool_result in tool_results
5905
+ ):
5906
+ raise ValueError("tool_results must contain objects with a toolUseId string")
5832
5907
  return [{"role": "user", "content": [{"toolResult": {
5833
5908
  "toolUseId": tr["toolUseId"],
5834
5909
  "status": tr.get("status", "success"),
5835
5910
  "content": tr.get("content", []),
5836
- }} for tr in payload["tool_results"]]}]
5837
- return payload.get("prompt", "")
5911
+ }} for tr in tool_results]}]
5912
+ prompt = payload.get("prompt", "")
5913
+ if not isinstance(prompt, str):
5914
+ raise ValueError("prompt must be a string")
5915
+ return prompt
5838
5916
 
5839
5917
 
5840
5918
  def _has_inline_function_call(messages) -> bool:
@@ -7302,6 +7380,11 @@ Run your agent locally:
7302
7380
  agentcore dev
7303
7381
  \`\`\`
7304
7382
 
7383
+ ### Validate Invocation Input
7384
+
7385
+ Validate runtime invocation payloads before forwarding them to an agent framework. Keep user prompts typed as strings
7386
+ and pass only prompt text to the agent.
7387
+
7305
7388
  ### Deployment
7306
7389
 
7307
7390
  Deploy to AWS:
@@ -7396,6 +7479,8 @@ Tags defined in \`agentcore.json\` flow through to deployed CloudFormation resou
7396
7479
  \`agentcore validate\` to check.
7397
7480
  4. **Resource Removal:** Use \`agentcore remove\` to remove resources. Run \`agentcore deploy\` after removal to tear down
7398
7481
  deployed infrastructure.
7482
+ 5. **Invocation Input:** Validate runtime payloads and require text prompts to be strings. If a Strands app accepts a
7483
+ caller-supplied message history, normalize the history tail with \`strip_trailing_tool_use()\` before invocation.
7399
7484
 
7400
7485
  ## Directory Structure
7401
7486
 
@@ -7609,6 +7694,11 @@ defines an HTTP server that streams tokens from your chosen Agent framework SDK.
7609
7694
 
7610
7695
  \`model/load.ts\` instantiates your chosen model provider.
7611
7696
 
7697
+ ## Input Validation
7698
+
7699
+ The generated Zod request schema keeps plain prompts typed as strings before forwarding them to Strands. Retain this
7700
+ validation when extending the request shape, and pass only prompt text to the agent.
7701
+
7612
7702
  ## Environment Variables
7613
7703
 
7614
7704
  | Variable | Required | Description |
@@ -7697,6 +7787,10 @@ const SYSTEM_PROMPT = \`
7697
7787
  You are a helpful assistant. Use tools when appropriate.
7698
7788
  \`;
7699
7789
 
7790
+ const requestSchema = z.object({
7791
+ prompt: z.string().default(''),
7792
+ });
7793
+
7700
7794
  {{#if hasMemory}}
7701
7795
  const agentCache = new Map<string, Agent>();
7702
7796
 
@@ -7750,7 +7844,8 @@ async function getOrCreateAgent(sessionId: string): Promise<Agent> {
7750
7844
 
7751
7845
  const app = new BedrockAgentCoreApp({
7752
7846
  invocationHandler: {
7753
- async *process(payload: any, context: any) {
7847
+ requestSchema,
7848
+ async *process(payload, context) {
7754
7849
  {{#if hasMemory}}
7755
7850
  const sessionId = context?.sessionId ?? 'default-session';
7756
7851
  const actorId = getActorId(payload, context);
@@ -7762,7 +7857,7 @@ const app = new BedrockAgentCoreApp({
7762
7857
 
7763
7858
  {{#if hasMemory}}
7764
7859
  try {
7765
- for await (const event of agent.stream(payload.prompt ?? '')) {
7860
+ for await (const event of agent.stream(payload.prompt)) {
7766
7861
  if (
7767
7862
  event.type === 'modelStreamUpdateEvent' &&
7768
7863
  event.event?.type === 'modelContentBlockDeltaEvent' &&
@@ -7786,7 +7881,7 @@ const app = new BedrockAgentCoreApp({
7786
7881
  // e.g. Anthropic). Restoring on error keeps the session reusable.
7787
7882
  const snapshot = agent.takeSnapshot({ include: ['messages'] });
7788
7883
  try {
7789
- for await (const event of agent.stream(payload.prompt ?? '')) {
7884
+ for await (const event of agent.stream(payload.prompt)) {
7790
7885
  if (
7791
7886
  event.type === 'modelStreamUpdateEvent' &&
7792
7887
  event.event?.type === 'modelContentBlockDeltaEvent' &&
@@ -8066,6 +8161,11 @@ defines an HTTP app that streams tokens using the Vercel AI SDK's \`streamText\`
8066
8161
 
8067
8162
  \`model/load.ts\` instantiates your chosen model provider.
8068
8163
 
8164
+ ## Input Validation
8165
+
8166
+ The generated Zod request schema keeps plain prompts typed as strings before forwarding them to the agent framework.
8167
+ Retain this validation when extending the request shape, and pass only prompt text to the agent.
8168
+
8069
8169
  ## Environment Variables
8070
8170
 
8071
8171
  | Variable | Required | Description |
@@ -8120,10 +8220,15 @@ Thumbs.db
8120
8220
  exports[`Assets Directory Snapshots > TypeScript assets > typescript/typescript/http/vercelai/base/main.ts should match snapshot 1`] = `
8121
8221
  "import { BedrockAgentCoreApp } from 'bedrock-agentcore/runtime';
8122
8222
  import { streamText, type ModelMessage } from 'ai';
8223
+ import { z } from 'zod';
8123
8224
  import { loadModel } from './model/load.js';
8124
8225
 
8125
8226
  const SYSTEM_PROMPT = \`You are a helpful assistant.\`;
8126
8227
 
8228
+ const requestSchema = z.object({
8229
+ prompt: z.string().default(''),
8230
+ });
8231
+
8127
8232
  const HISTORY_LIMIT = 128;
8128
8233
 
8129
8234
  // Keeps one message history per sessionId so each session remembers its own
@@ -8152,10 +8257,11 @@ function getHistory(sessionId: string): ModelMessage[] {
8152
8257
 
8153
8258
  const app = new BedrockAgentCoreApp({
8154
8259
  invocationHandler: {
8155
- async *process(payload: any, context: any) {
8260
+ requestSchema,
8261
+ async *process(payload, context) {
8156
8262
  const sessionId = context?.sessionId ?? 'default-session';
8157
8263
  const history = getHistory(sessionId);
8158
- const userMessage: ModelMessage = { role: 'user', content: payload.prompt ?? '' };
8264
+ const userMessage: ModelMessage = { role: 'user', content: payload.prompt };
8159
8265
 
8160
8266
  const model = await loadModel();
8161
8267
  const result = streamText({
@@ -0,0 +1,44 @@
1
+ import { readFileSync } from 'node:fs';
2
+ import { resolve } from 'node:path';
3
+ import { describe, expect, it } from 'vitest';
4
+
5
+ const ASSETS_DIR = resolve(__dirname, '..');
6
+
7
+ const PYTHON_HTTP_ENTRYPOINTS = [
8
+ 'python/http/autogen/base/main.py',
9
+ 'python/http/googleadk/base/main.py',
10
+ 'python/http/langchain_langgraph/base/main.py',
11
+ 'python/http/openaiagents/base/main.py',
12
+ 'python/http/strands/base/main.py',
13
+ ];
14
+
15
+ const TYPESCRIPT_HTTP_ENTRYPOINTS = [
16
+ 'typescript/http/strands/base/main.ts',
17
+ 'typescript/http/vercelai/base/main.ts',
18
+ ];
19
+
20
+ describe('HTTP agent template input validation', () => {
21
+ it.each(PYTHON_HTTP_ENTRYPOINTS)('%s rejects non-string prompts', templatePath => {
22
+ const template = readFileSync(resolve(ASSETS_DIR, templatePath), 'utf8');
23
+
24
+ expect(template).toContain('if not isinstance(prompt, str):');
25
+ expect(template).toContain('raise ValueError("prompt must be a string")');
26
+ });
27
+
28
+ it.each(TYPESCRIPT_HTTP_ENTRYPOINTS)('%s validates prompts with Zod', templatePath => {
29
+ const template = readFileSync(resolve(ASSETS_DIR, templatePath), 'utf8');
30
+
31
+ expect(template).toContain("prompt: z.string().default('')");
32
+ expect(template).toContain('requestSchema,');
33
+ });
34
+
35
+ it('strips toolUse blocks from the Python Strands message-history tail', () => {
36
+ const template = readFileSync(resolve(ASSETS_DIR, 'python/http/strands/base/main.py'), 'utf8');
37
+
38
+ expect(template).toContain('def strip_trailing_tool_use(messages: Any) -> list[dict]:');
39
+ expect(template).toContain('while messages:');
40
+ expect(template).toContain('content = [block for block in original_content if "toolUse" not in block]');
41
+ expect(template).toContain('messages.pop()');
42
+ expect(template).toContain('return strip_trailing_tool_use(payload["messages"])');
43
+ });
44
+ });
@@ -23,6 +23,8 @@ Tags defined in `agentcore.json` flow through to deployed CloudFormation resourc
23
23
  `agentcore validate` to check.
24
24
  4. **Resource Removal:** Use `agentcore remove` to remove resources. Run `agentcore deploy` after removal to tear down
25
25
  deployed infrastructure.
26
+ 5. **Invocation Input:** Validate runtime payloads and require text prompts to be strings. If a Strands app accepts a
27
+ caller-supplied message history, normalize the history tail with `strip_trailing_tool_use()` before invocation.
26
28
 
27
29
  ## Directory Structure
28
30
 
@@ -0,0 +1,15 @@
1
+ {
2
+ "Version": "2012-10-17",
3
+ "Statement": [
4
+ {
5
+ "Effect": "Allow",
6
+ "Action": ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
7
+ "Resource": "arn:*:logs:*:*:log-group:/aws/lambda/*"
8
+ },
9
+ {
10
+ "Effect": "Allow",
11
+ "Action": ["bedrock:InvokeModel"],
12
+ "Resource": "*"
13
+ }
14
+ ]
15
+ }
@@ -0,0 +1,37 @@
1
+ {{#if ModelProviderBedrock}}
2
+ import os
3
+
4
+ # litellm's Bedrock provider reads AWS_REGION_NAME; Lambda only sets AWS_REGION/AWS_DEFAULT_REGION.
5
+ os.environ.setdefault("AWS_REGION_NAME", os.environ.get("AWS_REGION", "us-west-2"))
6
+
7
+ from autoevals import {{ EvaluatorClass }}, init
8
+ from autoevals.litellm import LiteLLMClient
9
+
10
+ from bedrock_agentcore.evaluation.custom_code_based_evaluators import (
11
+ EvaluatorInput,
12
+ EvaluatorOutput,
13
+ custom_code_based_evaluator,
14
+ )
15
+ from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.autoevals import AutoEvalsAdapter
16
+
17
+ client = LiteLLMClient()
18
+ init(client=client, default_model="{{ Model }}")
19
+
20
+ adapter = AutoEvalsAdapter(metric={{ EvaluatorClass }}(client=client, model="{{ Model }}"){{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}})
21
+ {{else}}
22
+ from autoevals import {{ EvaluatorClass }}
23
+
24
+ from bedrock_agentcore.evaluation.custom_code_based_evaluators import (
25
+ EvaluatorInput,
26
+ EvaluatorOutput,
27
+ custom_code_based_evaluator,
28
+ )
29
+ from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.autoevals import AutoEvalsAdapter
30
+
31
+ adapter = AutoEvalsAdapter(metric={{ EvaluatorClass }}({{#if Model}}model="{{ Model }}"{{/if}}){{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}})
32
+ {{/if}}
33
+
34
+
35
+ @custom_code_based_evaluator()
36
+ def handler(evaluator_input: EvaluatorInput, context) -> EvaluatorOutput:
37
+ return adapter(evaluator_input, context)
@@ -0,0 +1,22 @@
1
+ [build-system]
2
+ requires = ["hatchling"]
3
+ build-backend = "hatchling.build"
4
+
5
+ [project]
6
+ name = "{{ Name }}"
7
+ version = "0.1.0"
8
+ description = "AgentCore Code-Based Evaluator (Autoevals)"
9
+ requires-python = ">=3.10"
10
+ dependencies = [
11
+ "bedrock-agentcore[autoevals]",
12
+ "autoevals>=0.0.80,<1.0.0",
13
+ {{#if ModelProviderBedrock}}
14
+ # autoevals grades via LiteLLMClient -> Bedrock (Converse); litellm replaces the openai judge
15
+ "litellm>=1.60,<1.85",
16
+ {{else}}
17
+ "openai>=1.0.0",
18
+ {{/if}}
19
+ ]
20
+
21
+ [tool.hatch.build.targets.wheel]
22
+ packages = ["."]
@@ -0,0 +1,15 @@
1
+ {
2
+ "Version": "2012-10-17",
3
+ "Statement": [
4
+ {
5
+ "Effect": "Allow",
6
+ "Action": ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
7
+ "Resource": "arn:*:logs:*:*:log-group:/aws/lambda/*"
8
+ },
9
+ {
10
+ "Effect": "Allow",
11
+ "Action": ["bedrock:InvokeModel"],
12
+ "Resource": "*"
13
+ }
14
+ ]
15
+ }
@@ -0,0 +1,29 @@
1
+ import os
2
+
3
+ os.environ.setdefault("DEEPEVAL_RESULTS_FOLDER", "/tmp/.deepeval")
4
+ os.environ.setdefault("DEEPEVAL_TELEMETRY_OPT_OUT", "YES")
5
+ os.chdir("/tmp")
6
+
7
+ {{#if ModelProviderBedrock}}
8
+ from deepeval.models import AmazonBedrockModel
9
+ {{/if}}
10
+ from deepeval.metrics import {{ EvaluatorClass }}
11
+
12
+ from bedrock_agentcore.evaluation.custom_code_based_evaluators import (
13
+ EvaluatorInput,
14
+ EvaluatorOutput,
15
+ custom_code_based_evaluator,
16
+ )
17
+ from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.deepeval import DeepEvalAdapter
18
+
19
+ {{#if ModelProviderBedrock}}
20
+ model = AmazonBedrockModel(model="{{ Model }}", region=os.environ.get("AWS_REGION", "us-west-2"))
21
+ adapter = DeepEvalAdapter(metric={{ EvaluatorClass }}(model=model{{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}}))
22
+ {{else}}
23
+ adapter = DeepEvalAdapter(metric={{ EvaluatorClass }}({{{ EvaluatorParams }}}))
24
+ {{/if}}
25
+
26
+
27
+ @custom_code_based_evaluator()
28
+ def handler(evaluator_input: EvaluatorInput, context) -> EvaluatorOutput:
29
+ return adapter(evaluator_input, context)
@@ -0,0 +1,19 @@
1
+ [build-system]
2
+ requires = ["hatchling"]
3
+ build-backend = "hatchling.build"
4
+
5
+ [project]
6
+ name = "{{ Name }}"
7
+ version = "0.1.0"
8
+ description = "AgentCore Code-Based Evaluator (DeepEval)"
9
+ requires-python = ">=3.10"
10
+ dependencies = [
11
+ "bedrock-agentcore[deepeval]",
12
+ "deepeval>=2.0.0,<3.0.0",
13
+ {{#if ModelProviderBedrock}}
14
+ "aiobotocore>=2.13.0",
15
+ {{/if}}
16
+ ]
17
+
18
+ [tool.hatch.build.targets.wheel]
19
+ packages = ["."]
@@ -13,6 +13,11 @@ file defines a Starlette ASGI app with the AutoGen framework running within.
13
13
 
14
14
  `model/load.py` instantiates your chosen model provider.
15
15
 
16
+ ## Input Validation
17
+
18
+ Validate invocation input before forwarding it to the agent framework. Keep plain prompts typed as strings and pass
19
+ only prompt text to the agent.
20
+
16
21
  ## Environment Variables
17
22
 
18
23
  | Variable | Required | Description |
@@ -127,6 +127,8 @@ async def invoke(payload, context):
127
127
 
128
128
  # Process the user prompt
129
129
  prompt = payload.get("prompt", "What can you help me with?")
130
+ if not isinstance(prompt, str):
131
+ raise ValueError("prompt must be a string")
130
132
  session_id = getattr(context, "session_id", "default-session")
131
133
 
132
134
  # Reuse the per-session agent (preserves conversation history)
@@ -13,6 +13,11 @@ file defines a Starlette ASGI app with the Google ADK framework running within.
13
13
 
14
14
  `model/load.py` instantiates your chosen model provider (Gemini).
15
15
 
16
+ ## Input Validation
17
+
18
+ Validate invocation input before forwarding it to the agent framework. Keep plain prompts typed as strings and pass
19
+ only prompt text to the agent.
20
+
16
21
  ## Environment Variables
17
22
 
18
23
  | Variable | Required | Description |
@@ -191,6 +191,8 @@ async def invoke(payload, context):
191
191
 
192
192
  # Process the user prompt
193
193
  prompt = payload.get("prompt", "What can you help me with?")
194
+ if not isinstance(prompt, str):
195
+ raise ValueError("prompt must be a string")
194
196
  session_id = getattr(context, "session_id", "default_session")
195
197
  user_id = payload.get("user_id", "default_user")
196
198
 
@@ -13,6 +13,11 @@ file defines a Starlette ASGI app with the LangChain/LangGraph framework running
13
13
 
14
14
  `model/load.py` instantiates your chosen model provider.
15
15
 
16
+ ## Input Validation
17
+
18
+ Validate invocation input before forwarding it to the agent framework. Keep plain prompts typed as strings and pass
19
+ only prompt text to the agent.
20
+
16
21
  ## Environment Variables
17
22
 
18
23
  | Variable | Required | Description |
@@ -183,6 +183,8 @@ async def invoke(payload, context):
183
183
 
184
184
  # Process the user prompt
185
185
  prompt = payload.get("prompt", "What can you help me with?")
186
+ if not isinstance(prompt, str):
187
+ raise ValueError("prompt must be a string")
186
188
  session_id = getattr(context, "session_id", "default-session")
187
189
  touch_thread(session_id)
188
190
  log.info(f"Agent input: {prompt}")
@@ -202,6 +204,8 @@ async def invoke(payload, context):
202
204
 
203
205
  # Process the user prompt
204
206
  prompt = payload.get("prompt", "What can you help me with?")
207
+ if not isinstance(prompt, str):
208
+ raise ValueError("prompt must be a string")
205
209
  session_id = getattr(context, "session_id", "default-session")
206
210
  touch_thread(session_id)
207
211
  log.info(f"Agent input: {prompt}")
@@ -13,6 +13,11 @@ file defines a Starlette ASGI app with the OpenAI Agents SDK framework running w
13
13
 
14
14
  `model/load.py` instantiates your chosen model provider (OpenAI).
15
15
 
16
+ ## Input Validation
17
+
18
+ Validate invocation input before forwarding it to the agent framework. Keep plain prompts typed as strings and pass
19
+ only prompt text to the agent.
20
+
16
21
  ## Environment Variables
17
22
 
18
23
  | Variable | Required | Description |
@@ -184,6 +184,8 @@ async def invoke(payload, context):
184
184
 
185
185
  # Process the user prompt
186
186
  prompt = payload.get("prompt", "What can you help me with?")
187
+ if not isinstance(prompt, str):
188
+ raise ValueError("prompt must be a string")
187
189
  session_id = getattr(context, "session_id", "default-session")
188
190
  session = get_session(session_id)
189
191
 
@@ -13,6 +13,12 @@ file defines a Starlette ASGI app with the chosen Agent framework SDK running wi
13
13
 
14
14
  `model/load.py` instantiates your chosen model provider.
15
15
 
16
+ ## Input Validation
17
+
18
+ Validate invocation input before forwarding it to Strands. Keep plain prompts typed as strings. If the app accepts a
19
+ caller-supplied message history, retain `strip_trailing_tool_use()`, which normalizes the history tail before
20
+ invoking the agent.
21
+
16
22
  ## Environment Variables
17
23
 
18
24
  | Variable | Required | Description |
@@ -481,17 +481,53 @@ get_or_create_agent = agent_factory()
481
481
  {{/if}}
482
482
 
483
483
 
484
+ def strip_trailing_tool_use(messages: Any) -> list[dict]:
485
+ """Strip toolUse blocks from the tail until the last message has none."""
486
+ if not isinstance(messages, list):
487
+ raise ValueError("messages must be a list")
488
+
489
+ messages = list(messages)
490
+ while messages:
491
+ last = messages[-1]
492
+ if not isinstance(last, dict):
493
+ raise ValueError("each message must be an object")
494
+ original_content = last.get("content", [])
495
+ if not isinstance(original_content, list) or not all(isinstance(block, dict) for block in original_content):
496
+ raise ValueError("each message content value must be a list of content blocks")
497
+
498
+ content = [block for block in original_content if "toolUse" not in block]
499
+ if len(content) == len(original_content):
500
+ break
501
+ if content:
502
+ messages[-1] = {**last, "content": content}
503
+ break
504
+ messages.pop()
505
+
506
+ return messages
507
+
508
+
484
509
  def _extract_prompt(payload: dict):
485
- """Accept harness-style messages[], tool_results[], or plain prompt string payloads."""
510
+ """Accept validated harness messages, tool results, or a plain prompt string."""
511
+ if not isinstance(payload, dict):
512
+ raise ValueError("payload must be a JSON object")
486
513
  if "messages" in payload:
487
- return payload["messages"]
514
+ return strip_trailing_tool_use(payload["messages"])
488
515
  if "tool_results" in payload:
516
+ tool_results = payload["tool_results"]
517
+ if not isinstance(tool_results, list) or not all(
518
+ isinstance(tool_result, dict) and isinstance(tool_result.get("toolUseId"), str)
519
+ for tool_result in tool_results
520
+ ):
521
+ raise ValueError("tool_results must contain objects with a toolUseId string")
489
522
  return [{"role": "user", "content": [{"toolResult": {
490
523
  "toolUseId": tr["toolUseId"],
491
524
  "status": tr.get("status", "success"),
492
525
  "content": tr.get("content", []),
493
- }} for tr in payload["tool_results"]]}]
494
- return payload.get("prompt", "")
526
+ }} for tr in tool_results]}]
527
+ prompt = payload.get("prompt", "")
528
+ if not isinstance(prompt, str):
529
+ raise ValueError("prompt must be a string")
530
+ return prompt
495
531
 
496
532
 
497
533
  def _has_inline_function_call(messages) -> bool: