@aws/agentcore 1.0.0-preview.24 → 1.0.0-preview.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/README.md +5 -0
- package/dist/assets/__tests__/__snapshots__/assets.snapshot.test.ts.snap +115 -9
- package/dist/assets/__tests__/input-validation.test.ts +44 -0
- package/dist/assets/agents/AGENTS.md +2 -0
- package/dist/assets/evaluators/autoevals-lambda/execution-role-policy.json +15 -0
- package/dist/assets/evaluators/autoevals-lambda/lambda_function.py +37 -0
- package/dist/assets/evaluators/autoevals-lambda/pyproject.toml +22 -0
- package/dist/assets/evaluators/deepeval-lambda/execution-role-policy.json +15 -0
- package/dist/assets/evaluators/deepeval-lambda/lambda_function.py +29 -0
- package/dist/assets/evaluators/deepeval-lambda/pyproject.toml +19 -0
- package/dist/assets/python/http/autogen/base/README.md +5 -0
- package/dist/assets/python/http/autogen/base/main.py +2 -0
- package/dist/assets/python/http/googleadk/base/README.md +5 -0
- package/dist/assets/python/http/googleadk/base/main.py +2 -0
- package/dist/assets/python/http/langchain_langgraph/base/README.md +5 -0
- package/dist/assets/python/http/langchain_langgraph/base/main.py +4 -0
- package/dist/assets/python/http/openaiagents/base/README.md +5 -0
- package/dist/assets/python/http/openaiagents/base/main.py +2 -0
- package/dist/assets/python/http/strands/base/README.md +6 -0
- package/dist/assets/python/http/strands/base/main.py +40 -4
- package/dist/assets/typescript/http/strands/base/README.md +5 -0
- package/dist/assets/typescript/http/strands/base/main.ts +8 -3
- package/dist/assets/typescript/http/vercelai/base/README.md +5 -0
- package/dist/assets/typescript/http/vercelai/base/main.ts +8 -2
- package/dist/cli/index.mjs +421 -417
- package/npm-shrinkwrap.json +8 -8
- package/package.json +1 -1
package/dist/assets/README.md
CHANGED
|
@@ -37,6 +37,11 @@ Run your agent locally:
|
|
|
37
37
|
agentcore dev
|
|
38
38
|
```
|
|
39
39
|
|
|
40
|
+
### Validate Invocation Input
|
|
41
|
+
|
|
42
|
+
Validate runtime invocation payloads before forwarding them to an agent framework. Keep user prompts typed as strings
|
|
43
|
+
and pass only prompt text to the agent.
|
|
44
|
+
|
|
40
45
|
### Deployment
|
|
41
46
|
|
|
42
47
|
Deploy to AWS:
|
|
@@ -742,6 +742,12 @@ exports[`Assets Directory Snapshots > File listing > should match the expected f
|
|
|
742
742
|
"container/typescript/dockerignore.template",
|
|
743
743
|
"datasets/predefined-v1.jsonl",
|
|
744
744
|
"datasets/simulated-v1.jsonl",
|
|
745
|
+
"evaluators/autoevals-lambda/execution-role-policy.json",
|
|
746
|
+
"evaluators/autoevals-lambda/lambda_function.py",
|
|
747
|
+
"evaluators/autoevals-lambda/pyproject.toml",
|
|
748
|
+
"evaluators/deepeval-lambda/execution-role-policy.json",
|
|
749
|
+
"evaluators/deepeval-lambda/lambda_function.py",
|
|
750
|
+
"evaluators/deepeval-lambda/pyproject.toml",
|
|
745
751
|
"evaluators/python-lambda/execution-role-policy.json",
|
|
746
752
|
"evaluators/python-lambda/lambda_function.py",
|
|
747
753
|
"evaluators/python-lambda/pyproject.toml",
|
|
@@ -3329,6 +3335,11 @@ file defines a Starlette ASGI app with the AutoGen framework running within.
|
|
|
3329
3335
|
|
|
3330
3336
|
\`model/load.py\` instantiates your chosen model provider.
|
|
3331
3337
|
|
|
3338
|
+
## Input Validation
|
|
3339
|
+
|
|
3340
|
+
Validate invocation input before forwarding it to the agent framework. Keep plain prompts typed as strings and pass
|
|
3341
|
+
only prompt text to the agent.
|
|
3342
|
+
|
|
3332
3343
|
## Environment Variables
|
|
3333
3344
|
|
|
3334
3345
|
| Variable | Required | Description |
|
|
@@ -3532,6 +3543,8 @@ async def invoke(payload, context):
|
|
|
3532
3543
|
|
|
3533
3544
|
# Process the user prompt
|
|
3534
3545
|
prompt = payload.get("prompt", "What can you help me with?")
|
|
3546
|
+
if not isinstance(prompt, str):
|
|
3547
|
+
raise ValueError("prompt must be a string")
|
|
3535
3548
|
session_id = getattr(context, "session_id", "default-session")
|
|
3536
3549
|
|
|
3537
3550
|
# Reuse the per-session agent (preserves conversation history)
|
|
@@ -3785,6 +3798,11 @@ file defines a Starlette ASGI app with the Google ADK framework running within.
|
|
|
3785
3798
|
|
|
3786
3799
|
\`model/load.py\` instantiates your chosen model provider (Gemini).
|
|
3787
3800
|
|
|
3801
|
+
## Input Validation
|
|
3802
|
+
|
|
3803
|
+
Validate invocation input before forwarding it to the agent framework. Keep plain prompts typed as strings and pass
|
|
3804
|
+
only prompt text to the agent.
|
|
3805
|
+
|
|
3788
3806
|
## Environment Variables
|
|
3789
3807
|
|
|
3790
3808
|
| Variable | Required | Description |
|
|
@@ -4052,6 +4070,8 @@ async def invoke(payload, context):
|
|
|
4052
4070
|
|
|
4053
4071
|
# Process the user prompt
|
|
4054
4072
|
prompt = payload.get("prompt", "What can you help me with?")
|
|
4073
|
+
if not isinstance(prompt, str):
|
|
4074
|
+
raise ValueError("prompt must be a string")
|
|
4055
4075
|
session_id = getattr(context, "session_id", "default_session")
|
|
4056
4076
|
user_id = payload.get("user_id", "default_user")
|
|
4057
4077
|
|
|
@@ -4245,6 +4265,11 @@ file defines a Starlette ASGI app with the LangChain/LangGraph framework running
|
|
|
4245
4265
|
|
|
4246
4266
|
\`model/load.py\` instantiates your chosen model provider.
|
|
4247
4267
|
|
|
4268
|
+
## Input Validation
|
|
4269
|
+
|
|
4270
|
+
Validate invocation input before forwarding it to the agent framework. Keep plain prompts typed as strings and pass
|
|
4271
|
+
only prompt text to the agent.
|
|
4272
|
+
|
|
4248
4273
|
## Environment Variables
|
|
4249
4274
|
|
|
4250
4275
|
| Variable | Required | Description |
|
|
@@ -4504,6 +4529,8 @@ async def invoke(payload, context):
|
|
|
4504
4529
|
|
|
4505
4530
|
# Process the user prompt
|
|
4506
4531
|
prompt = payload.get("prompt", "What can you help me with?")
|
|
4532
|
+
if not isinstance(prompt, str):
|
|
4533
|
+
raise ValueError("prompt must be a string")
|
|
4507
4534
|
session_id = getattr(context, "session_id", "default-session")
|
|
4508
4535
|
touch_thread(session_id)
|
|
4509
4536
|
log.info(f"Agent input: {prompt}")
|
|
@@ -4523,6 +4550,8 @@ async def invoke(payload, context):
|
|
|
4523
4550
|
|
|
4524
4551
|
# Process the user prompt
|
|
4525
4552
|
prompt = payload.get("prompt", "What can you help me with?")
|
|
4553
|
+
if not isinstance(prompt, str):
|
|
4554
|
+
raise ValueError("prompt must be a string")
|
|
4526
4555
|
session_id = getattr(context, "session_id", "default-session")
|
|
4527
4556
|
touch_thread(session_id)
|
|
4528
4557
|
log.info(f"Agent input: {prompt}")
|
|
@@ -4823,6 +4852,11 @@ file defines a Starlette ASGI app with the OpenAI Agents SDK framework running w
|
|
|
4823
4852
|
|
|
4824
4853
|
\`model/load.py\` instantiates your chosen model provider (OpenAI).
|
|
4825
4854
|
|
|
4855
|
+
## Input Validation
|
|
4856
|
+
|
|
4857
|
+
Validate invocation input before forwarding it to the agent framework. Keep plain prompts typed as strings and pass
|
|
4858
|
+
only prompt text to the agent.
|
|
4859
|
+
|
|
4826
4860
|
## Environment Variables
|
|
4827
4861
|
|
|
4828
4862
|
| Variable | Required | Description |
|
|
@@ -5083,6 +5117,8 @@ async def invoke(payload, context):
|
|
|
5083
5117
|
|
|
5084
5118
|
# Process the user prompt
|
|
5085
5119
|
prompt = payload.get("prompt", "What can you help me with?")
|
|
5120
|
+
if not isinstance(prompt, str):
|
|
5121
|
+
raise ValueError("prompt must be a string")
|
|
5086
5122
|
session_id = getattr(context, "session_id", "default-session")
|
|
5087
5123
|
session = get_session(session_id)
|
|
5088
5124
|
|
|
@@ -5268,6 +5304,12 @@ file defines a Starlette ASGI app with the chosen Agent framework SDK running wi
|
|
|
5268
5304
|
|
|
5269
5305
|
\`model/load.py\` instantiates your chosen model provider.
|
|
5270
5306
|
|
|
5307
|
+
## Input Validation
|
|
5308
|
+
|
|
5309
|
+
Validate invocation input before forwarding it to Strands. Keep plain prompts typed as strings. If the app accepts a
|
|
5310
|
+
caller-supplied message history, retain \`strip_trailing_tool_use()\`, which normalizes the history tail before
|
|
5311
|
+
invoking the agent.
|
|
5312
|
+
|
|
5271
5313
|
## Environment Variables
|
|
5272
5314
|
|
|
5273
5315
|
| Variable | Required | Description |
|
|
@@ -5824,17 +5866,53 @@ get_or_create_agent = agent_factory()
|
|
|
5824
5866
|
{{/if}}
|
|
5825
5867
|
|
|
5826
5868
|
|
|
5869
|
+
def strip_trailing_tool_use(messages: Any) -> list[dict]:
|
|
5870
|
+
"""Strip toolUse blocks from the tail until the last message has none."""
|
|
5871
|
+
if not isinstance(messages, list):
|
|
5872
|
+
raise ValueError("messages must be a list")
|
|
5873
|
+
|
|
5874
|
+
messages = list(messages)
|
|
5875
|
+
while messages:
|
|
5876
|
+
last = messages[-1]
|
|
5877
|
+
if not isinstance(last, dict):
|
|
5878
|
+
raise ValueError("each message must be an object")
|
|
5879
|
+
original_content = last.get("content", [])
|
|
5880
|
+
if not isinstance(original_content, list) or not all(isinstance(block, dict) for block in original_content):
|
|
5881
|
+
raise ValueError("each message content value must be a list of content blocks")
|
|
5882
|
+
|
|
5883
|
+
content = [block for block in original_content if "toolUse" not in block]
|
|
5884
|
+
if len(content) == len(original_content):
|
|
5885
|
+
break
|
|
5886
|
+
if content:
|
|
5887
|
+
messages[-1] = {**last, "content": content}
|
|
5888
|
+
break
|
|
5889
|
+
messages.pop()
|
|
5890
|
+
|
|
5891
|
+
return messages
|
|
5892
|
+
|
|
5893
|
+
|
|
5827
5894
|
def _extract_prompt(payload: dict):
|
|
5828
|
-
"""Accept harness
|
|
5895
|
+
"""Accept validated harness messages, tool results, or a plain prompt string."""
|
|
5896
|
+
if not isinstance(payload, dict):
|
|
5897
|
+
raise ValueError("payload must be a JSON object")
|
|
5829
5898
|
if "messages" in payload:
|
|
5830
|
-
return payload["messages"]
|
|
5899
|
+
return strip_trailing_tool_use(payload["messages"])
|
|
5831
5900
|
if "tool_results" in payload:
|
|
5901
|
+
tool_results = payload["tool_results"]
|
|
5902
|
+
if not isinstance(tool_results, list) or not all(
|
|
5903
|
+
isinstance(tool_result, dict) and isinstance(tool_result.get("toolUseId"), str)
|
|
5904
|
+
for tool_result in tool_results
|
|
5905
|
+
):
|
|
5906
|
+
raise ValueError("tool_results must contain objects with a toolUseId string")
|
|
5832
5907
|
return [{"role": "user", "content": [{"toolResult": {
|
|
5833
5908
|
"toolUseId": tr["toolUseId"],
|
|
5834
5909
|
"status": tr.get("status", "success"),
|
|
5835
5910
|
"content": tr.get("content", []),
|
|
5836
|
-
}} for tr in
|
|
5837
|
-
|
|
5911
|
+
}} for tr in tool_results]}]
|
|
5912
|
+
prompt = payload.get("prompt", "")
|
|
5913
|
+
if not isinstance(prompt, str):
|
|
5914
|
+
raise ValueError("prompt must be a string")
|
|
5915
|
+
return prompt
|
|
5838
5916
|
|
|
5839
5917
|
|
|
5840
5918
|
def _has_inline_function_call(messages) -> bool:
|
|
@@ -7302,6 +7380,11 @@ Run your agent locally:
|
|
|
7302
7380
|
agentcore dev
|
|
7303
7381
|
\`\`\`
|
|
7304
7382
|
|
|
7383
|
+
### Validate Invocation Input
|
|
7384
|
+
|
|
7385
|
+
Validate runtime invocation payloads before forwarding them to an agent framework. Keep user prompts typed as strings
|
|
7386
|
+
and pass only prompt text to the agent.
|
|
7387
|
+
|
|
7305
7388
|
### Deployment
|
|
7306
7389
|
|
|
7307
7390
|
Deploy to AWS:
|
|
@@ -7396,6 +7479,8 @@ Tags defined in \`agentcore.json\` flow through to deployed CloudFormation resou
|
|
|
7396
7479
|
\`agentcore validate\` to check.
|
|
7397
7480
|
4. **Resource Removal:** Use \`agentcore remove\` to remove resources. Run \`agentcore deploy\` after removal to tear down
|
|
7398
7481
|
deployed infrastructure.
|
|
7482
|
+
5. **Invocation Input:** Validate runtime payloads and require text prompts to be strings. If a Strands app accepts a
|
|
7483
|
+
caller-supplied message history, normalize the history tail with \`strip_trailing_tool_use()\` before invocation.
|
|
7399
7484
|
|
|
7400
7485
|
## Directory Structure
|
|
7401
7486
|
|
|
@@ -7609,6 +7694,11 @@ defines an HTTP server that streams tokens from your chosen Agent framework SDK.
|
|
|
7609
7694
|
|
|
7610
7695
|
\`model/load.ts\` instantiates your chosen model provider.
|
|
7611
7696
|
|
|
7697
|
+
## Input Validation
|
|
7698
|
+
|
|
7699
|
+
The generated Zod request schema keeps plain prompts typed as strings before forwarding them to Strands. Retain this
|
|
7700
|
+
validation when extending the request shape, and pass only prompt text to the agent.
|
|
7701
|
+
|
|
7612
7702
|
## Environment Variables
|
|
7613
7703
|
|
|
7614
7704
|
| Variable | Required | Description |
|
|
@@ -7697,6 +7787,10 @@ const SYSTEM_PROMPT = \`
|
|
|
7697
7787
|
You are a helpful assistant. Use tools when appropriate.
|
|
7698
7788
|
\`;
|
|
7699
7789
|
|
|
7790
|
+
const requestSchema = z.object({
|
|
7791
|
+
prompt: z.string().default(''),
|
|
7792
|
+
});
|
|
7793
|
+
|
|
7700
7794
|
{{#if hasMemory}}
|
|
7701
7795
|
const agentCache = new Map<string, Agent>();
|
|
7702
7796
|
|
|
@@ -7750,7 +7844,8 @@ async function getOrCreateAgent(sessionId: string): Promise<Agent> {
|
|
|
7750
7844
|
|
|
7751
7845
|
const app = new BedrockAgentCoreApp({
|
|
7752
7846
|
invocationHandler: {
|
|
7753
|
-
|
|
7847
|
+
requestSchema,
|
|
7848
|
+
async *process(payload, context) {
|
|
7754
7849
|
{{#if hasMemory}}
|
|
7755
7850
|
const sessionId = context?.sessionId ?? 'default-session';
|
|
7756
7851
|
const actorId = getActorId(payload, context);
|
|
@@ -7762,7 +7857,7 @@ const app = new BedrockAgentCoreApp({
|
|
|
7762
7857
|
|
|
7763
7858
|
{{#if hasMemory}}
|
|
7764
7859
|
try {
|
|
7765
|
-
for await (const event of agent.stream(payload.prompt
|
|
7860
|
+
for await (const event of agent.stream(payload.prompt)) {
|
|
7766
7861
|
if (
|
|
7767
7862
|
event.type === 'modelStreamUpdateEvent' &&
|
|
7768
7863
|
event.event?.type === 'modelContentBlockDeltaEvent' &&
|
|
@@ -7786,7 +7881,7 @@ const app = new BedrockAgentCoreApp({
|
|
|
7786
7881
|
// e.g. Anthropic). Restoring on error keeps the session reusable.
|
|
7787
7882
|
const snapshot = agent.takeSnapshot({ include: ['messages'] });
|
|
7788
7883
|
try {
|
|
7789
|
-
for await (const event of agent.stream(payload.prompt
|
|
7884
|
+
for await (const event of agent.stream(payload.prompt)) {
|
|
7790
7885
|
if (
|
|
7791
7886
|
event.type === 'modelStreamUpdateEvent' &&
|
|
7792
7887
|
event.event?.type === 'modelContentBlockDeltaEvent' &&
|
|
@@ -8066,6 +8161,11 @@ defines an HTTP app that streams tokens using the Vercel AI SDK's \`streamText\`
|
|
|
8066
8161
|
|
|
8067
8162
|
\`model/load.ts\` instantiates your chosen model provider.
|
|
8068
8163
|
|
|
8164
|
+
## Input Validation
|
|
8165
|
+
|
|
8166
|
+
The generated Zod request schema keeps plain prompts typed as strings before forwarding them to the agent framework.
|
|
8167
|
+
Retain this validation when extending the request shape, and pass only prompt text to the agent.
|
|
8168
|
+
|
|
8069
8169
|
## Environment Variables
|
|
8070
8170
|
|
|
8071
8171
|
| Variable | Required | Description |
|
|
@@ -8120,10 +8220,15 @@ Thumbs.db
|
|
|
8120
8220
|
exports[`Assets Directory Snapshots > TypeScript assets > typescript/typescript/http/vercelai/base/main.ts should match snapshot 1`] = `
|
|
8121
8221
|
"import { BedrockAgentCoreApp } from 'bedrock-agentcore/runtime';
|
|
8122
8222
|
import { streamText, type ModelMessage } from 'ai';
|
|
8223
|
+
import { z } from 'zod';
|
|
8123
8224
|
import { loadModel } from './model/load.js';
|
|
8124
8225
|
|
|
8125
8226
|
const SYSTEM_PROMPT = \`You are a helpful assistant.\`;
|
|
8126
8227
|
|
|
8228
|
+
const requestSchema = z.object({
|
|
8229
|
+
prompt: z.string().default(''),
|
|
8230
|
+
});
|
|
8231
|
+
|
|
8127
8232
|
const HISTORY_LIMIT = 128;
|
|
8128
8233
|
|
|
8129
8234
|
// Keeps one message history per sessionId so each session remembers its own
|
|
@@ -8152,10 +8257,11 @@ function getHistory(sessionId: string): ModelMessage[] {
|
|
|
8152
8257
|
|
|
8153
8258
|
const app = new BedrockAgentCoreApp({
|
|
8154
8259
|
invocationHandler: {
|
|
8155
|
-
|
|
8260
|
+
requestSchema,
|
|
8261
|
+
async *process(payload, context) {
|
|
8156
8262
|
const sessionId = context?.sessionId ?? 'default-session';
|
|
8157
8263
|
const history = getHistory(sessionId);
|
|
8158
|
-
const userMessage: ModelMessage = { role: 'user', content: payload.prompt
|
|
8264
|
+
const userMessage: ModelMessage = { role: 'user', content: payload.prompt };
|
|
8159
8265
|
|
|
8160
8266
|
const model = await loadModel();
|
|
8161
8267
|
const result = streamText({
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import { readFileSync } from 'node:fs';
|
|
2
|
+
import { resolve } from 'node:path';
|
|
3
|
+
import { describe, expect, it } from 'vitest';
|
|
4
|
+
|
|
5
|
+
const ASSETS_DIR = resolve(__dirname, '..');
|
|
6
|
+
|
|
7
|
+
const PYTHON_HTTP_ENTRYPOINTS = [
|
|
8
|
+
'python/http/autogen/base/main.py',
|
|
9
|
+
'python/http/googleadk/base/main.py',
|
|
10
|
+
'python/http/langchain_langgraph/base/main.py',
|
|
11
|
+
'python/http/openaiagents/base/main.py',
|
|
12
|
+
'python/http/strands/base/main.py',
|
|
13
|
+
];
|
|
14
|
+
|
|
15
|
+
const TYPESCRIPT_HTTP_ENTRYPOINTS = [
|
|
16
|
+
'typescript/http/strands/base/main.ts',
|
|
17
|
+
'typescript/http/vercelai/base/main.ts',
|
|
18
|
+
];
|
|
19
|
+
|
|
20
|
+
describe('HTTP agent template input validation', () => {
|
|
21
|
+
it.each(PYTHON_HTTP_ENTRYPOINTS)('%s rejects non-string prompts', templatePath => {
|
|
22
|
+
const template = readFileSync(resolve(ASSETS_DIR, templatePath), 'utf8');
|
|
23
|
+
|
|
24
|
+
expect(template).toContain('if not isinstance(prompt, str):');
|
|
25
|
+
expect(template).toContain('raise ValueError("prompt must be a string")');
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
it.each(TYPESCRIPT_HTTP_ENTRYPOINTS)('%s validates prompts with Zod', templatePath => {
|
|
29
|
+
const template = readFileSync(resolve(ASSETS_DIR, templatePath), 'utf8');
|
|
30
|
+
|
|
31
|
+
expect(template).toContain("prompt: z.string().default('')");
|
|
32
|
+
expect(template).toContain('requestSchema,');
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
it('strips toolUse blocks from the Python Strands message-history tail', () => {
|
|
36
|
+
const template = readFileSync(resolve(ASSETS_DIR, 'python/http/strands/base/main.py'), 'utf8');
|
|
37
|
+
|
|
38
|
+
expect(template).toContain('def strip_trailing_tool_use(messages: Any) -> list[dict]:');
|
|
39
|
+
expect(template).toContain('while messages:');
|
|
40
|
+
expect(template).toContain('content = [block for block in original_content if "toolUse" not in block]');
|
|
41
|
+
expect(template).toContain('messages.pop()');
|
|
42
|
+
expect(template).toContain('return strip_trailing_tool_use(payload["messages"])');
|
|
43
|
+
});
|
|
44
|
+
});
|
|
@@ -23,6 +23,8 @@ Tags defined in `agentcore.json` flow through to deployed CloudFormation resourc
|
|
|
23
23
|
`agentcore validate` to check.
|
|
24
24
|
4. **Resource Removal:** Use `agentcore remove` to remove resources. Run `agentcore deploy` after removal to tear down
|
|
25
25
|
deployed infrastructure.
|
|
26
|
+
5. **Invocation Input:** Validate runtime payloads and require text prompts to be strings. If a Strands app accepts a
|
|
27
|
+
caller-supplied message history, normalize the history tail with `strip_trailing_tool_use()` before invocation.
|
|
26
28
|
|
|
27
29
|
## Directory Structure
|
|
28
30
|
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
{
|
|
2
|
+
"Version": "2012-10-17",
|
|
3
|
+
"Statement": [
|
|
4
|
+
{
|
|
5
|
+
"Effect": "Allow",
|
|
6
|
+
"Action": ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
|
|
7
|
+
"Resource": "arn:*:logs:*:*:log-group:/aws/lambda/*"
|
|
8
|
+
},
|
|
9
|
+
{
|
|
10
|
+
"Effect": "Allow",
|
|
11
|
+
"Action": ["bedrock:InvokeModel"],
|
|
12
|
+
"Resource": "*"
|
|
13
|
+
}
|
|
14
|
+
]
|
|
15
|
+
}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
{{#if ModelProviderBedrock}}
|
|
2
|
+
import os
|
|
3
|
+
|
|
4
|
+
# litellm's Bedrock provider reads AWS_REGION_NAME; Lambda only sets AWS_REGION/AWS_DEFAULT_REGION.
|
|
5
|
+
os.environ.setdefault("AWS_REGION_NAME", os.environ.get("AWS_REGION", "us-west-2"))
|
|
6
|
+
|
|
7
|
+
from autoevals import {{ EvaluatorClass }}, init
|
|
8
|
+
from autoevals.litellm import LiteLLMClient
|
|
9
|
+
|
|
10
|
+
from bedrock_agentcore.evaluation.custom_code_based_evaluators import (
|
|
11
|
+
EvaluatorInput,
|
|
12
|
+
EvaluatorOutput,
|
|
13
|
+
custom_code_based_evaluator,
|
|
14
|
+
)
|
|
15
|
+
from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.autoevals import AutoEvalsAdapter
|
|
16
|
+
|
|
17
|
+
client = LiteLLMClient()
|
|
18
|
+
init(client=client, default_model="{{ Model }}")
|
|
19
|
+
|
|
20
|
+
adapter = AutoEvalsAdapter(metric={{ EvaluatorClass }}(client=client, model="{{ Model }}"){{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}})
|
|
21
|
+
{{else}}
|
|
22
|
+
from autoevals import {{ EvaluatorClass }}
|
|
23
|
+
|
|
24
|
+
from bedrock_agentcore.evaluation.custom_code_based_evaluators import (
|
|
25
|
+
EvaluatorInput,
|
|
26
|
+
EvaluatorOutput,
|
|
27
|
+
custom_code_based_evaluator,
|
|
28
|
+
)
|
|
29
|
+
from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.autoevals import AutoEvalsAdapter
|
|
30
|
+
|
|
31
|
+
adapter = AutoEvalsAdapter(metric={{ EvaluatorClass }}({{#if Model}}model="{{ Model }}"{{/if}}){{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}})
|
|
32
|
+
{{/if}}
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@custom_code_based_evaluator()
|
|
36
|
+
def handler(evaluator_input: EvaluatorInput, context) -> EvaluatorOutput:
|
|
37
|
+
return adapter(evaluator_input, context)
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "{{ Name }}"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "AgentCore Code-Based Evaluator (Autoevals)"
|
|
9
|
+
requires-python = ">=3.10"
|
|
10
|
+
dependencies = [
|
|
11
|
+
"bedrock-agentcore[autoevals]",
|
|
12
|
+
"autoevals>=0.0.80,<1.0.0",
|
|
13
|
+
{{#if ModelProviderBedrock}}
|
|
14
|
+
# autoevals grades via LiteLLMClient -> Bedrock (Converse); litellm replaces the openai judge
|
|
15
|
+
"litellm>=1.60,<1.85",
|
|
16
|
+
{{else}}
|
|
17
|
+
"openai>=1.0.0",
|
|
18
|
+
{{/if}}
|
|
19
|
+
]
|
|
20
|
+
|
|
21
|
+
[tool.hatch.build.targets.wheel]
|
|
22
|
+
packages = ["."]
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
{
|
|
2
|
+
"Version": "2012-10-17",
|
|
3
|
+
"Statement": [
|
|
4
|
+
{
|
|
5
|
+
"Effect": "Allow",
|
|
6
|
+
"Action": ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
|
|
7
|
+
"Resource": "arn:*:logs:*:*:log-group:/aws/lambda/*"
|
|
8
|
+
},
|
|
9
|
+
{
|
|
10
|
+
"Effect": "Allow",
|
|
11
|
+
"Action": ["bedrock:InvokeModel"],
|
|
12
|
+
"Resource": "*"
|
|
13
|
+
}
|
|
14
|
+
]
|
|
15
|
+
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import os
|
|
2
|
+
|
|
3
|
+
os.environ.setdefault("DEEPEVAL_RESULTS_FOLDER", "/tmp/.deepeval")
|
|
4
|
+
os.environ.setdefault("DEEPEVAL_TELEMETRY_OPT_OUT", "YES")
|
|
5
|
+
os.chdir("/tmp")
|
|
6
|
+
|
|
7
|
+
{{#if ModelProviderBedrock}}
|
|
8
|
+
from deepeval.models import AmazonBedrockModel
|
|
9
|
+
{{/if}}
|
|
10
|
+
from deepeval.metrics import {{ EvaluatorClass }}
|
|
11
|
+
|
|
12
|
+
from bedrock_agentcore.evaluation.custom_code_based_evaluators import (
|
|
13
|
+
EvaluatorInput,
|
|
14
|
+
EvaluatorOutput,
|
|
15
|
+
custom_code_based_evaluator,
|
|
16
|
+
)
|
|
17
|
+
from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.deepeval import DeepEvalAdapter
|
|
18
|
+
|
|
19
|
+
{{#if ModelProviderBedrock}}
|
|
20
|
+
model = AmazonBedrockModel(model="{{ Model }}", region=os.environ.get("AWS_REGION", "us-west-2"))
|
|
21
|
+
adapter = DeepEvalAdapter(metric={{ EvaluatorClass }}(model=model{{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}}))
|
|
22
|
+
{{else}}
|
|
23
|
+
adapter = DeepEvalAdapter(metric={{ EvaluatorClass }}({{{ EvaluatorParams }}}))
|
|
24
|
+
{{/if}}
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@custom_code_based_evaluator()
|
|
28
|
+
def handler(evaluator_input: EvaluatorInput, context) -> EvaluatorOutput:
|
|
29
|
+
return adapter(evaluator_input, context)
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "{{ Name }}"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "AgentCore Code-Based Evaluator (DeepEval)"
|
|
9
|
+
requires-python = ">=3.10"
|
|
10
|
+
dependencies = [
|
|
11
|
+
"bedrock-agentcore[deepeval]",
|
|
12
|
+
"deepeval>=2.0.0,<3.0.0",
|
|
13
|
+
{{#if ModelProviderBedrock}}
|
|
14
|
+
"aiobotocore>=2.13.0",
|
|
15
|
+
{{/if}}
|
|
16
|
+
]
|
|
17
|
+
|
|
18
|
+
[tool.hatch.build.targets.wheel]
|
|
19
|
+
packages = ["."]
|
|
@@ -13,6 +13,11 @@ file defines a Starlette ASGI app with the AutoGen framework running within.
|
|
|
13
13
|
|
|
14
14
|
`model/load.py` instantiates your chosen model provider.
|
|
15
15
|
|
|
16
|
+
## Input Validation
|
|
17
|
+
|
|
18
|
+
Validate invocation input before forwarding it to the agent framework. Keep plain prompts typed as strings and pass
|
|
19
|
+
only prompt text to the agent.
|
|
20
|
+
|
|
16
21
|
## Environment Variables
|
|
17
22
|
|
|
18
23
|
| Variable | Required | Description |
|
|
@@ -127,6 +127,8 @@ async def invoke(payload, context):
|
|
|
127
127
|
|
|
128
128
|
# Process the user prompt
|
|
129
129
|
prompt = payload.get("prompt", "What can you help me with?")
|
|
130
|
+
if not isinstance(prompt, str):
|
|
131
|
+
raise ValueError("prompt must be a string")
|
|
130
132
|
session_id = getattr(context, "session_id", "default-session")
|
|
131
133
|
|
|
132
134
|
# Reuse the per-session agent (preserves conversation history)
|
|
@@ -13,6 +13,11 @@ file defines a Starlette ASGI app with the Google ADK framework running within.
|
|
|
13
13
|
|
|
14
14
|
`model/load.py` instantiates your chosen model provider (Gemini).
|
|
15
15
|
|
|
16
|
+
## Input Validation
|
|
17
|
+
|
|
18
|
+
Validate invocation input before forwarding it to the agent framework. Keep plain prompts typed as strings and pass
|
|
19
|
+
only prompt text to the agent.
|
|
20
|
+
|
|
16
21
|
## Environment Variables
|
|
17
22
|
|
|
18
23
|
| Variable | Required | Description |
|
|
@@ -191,6 +191,8 @@ async def invoke(payload, context):
|
|
|
191
191
|
|
|
192
192
|
# Process the user prompt
|
|
193
193
|
prompt = payload.get("prompt", "What can you help me with?")
|
|
194
|
+
if not isinstance(prompt, str):
|
|
195
|
+
raise ValueError("prompt must be a string")
|
|
194
196
|
session_id = getattr(context, "session_id", "default_session")
|
|
195
197
|
user_id = payload.get("user_id", "default_user")
|
|
196
198
|
|
|
@@ -13,6 +13,11 @@ file defines a Starlette ASGI app with the LangChain/LangGraph framework running
|
|
|
13
13
|
|
|
14
14
|
`model/load.py` instantiates your chosen model provider.
|
|
15
15
|
|
|
16
|
+
## Input Validation
|
|
17
|
+
|
|
18
|
+
Validate invocation input before forwarding it to the agent framework. Keep plain prompts typed as strings and pass
|
|
19
|
+
only prompt text to the agent.
|
|
20
|
+
|
|
16
21
|
## Environment Variables
|
|
17
22
|
|
|
18
23
|
| Variable | Required | Description |
|
|
@@ -183,6 +183,8 @@ async def invoke(payload, context):
|
|
|
183
183
|
|
|
184
184
|
# Process the user prompt
|
|
185
185
|
prompt = payload.get("prompt", "What can you help me with?")
|
|
186
|
+
if not isinstance(prompt, str):
|
|
187
|
+
raise ValueError("prompt must be a string")
|
|
186
188
|
session_id = getattr(context, "session_id", "default-session")
|
|
187
189
|
touch_thread(session_id)
|
|
188
190
|
log.info(f"Agent input: {prompt}")
|
|
@@ -202,6 +204,8 @@ async def invoke(payload, context):
|
|
|
202
204
|
|
|
203
205
|
# Process the user prompt
|
|
204
206
|
prompt = payload.get("prompt", "What can you help me with?")
|
|
207
|
+
if not isinstance(prompt, str):
|
|
208
|
+
raise ValueError("prompt must be a string")
|
|
205
209
|
session_id = getattr(context, "session_id", "default-session")
|
|
206
210
|
touch_thread(session_id)
|
|
207
211
|
log.info(f"Agent input: {prompt}")
|
|
@@ -13,6 +13,11 @@ file defines a Starlette ASGI app with the OpenAI Agents SDK framework running w
|
|
|
13
13
|
|
|
14
14
|
`model/load.py` instantiates your chosen model provider (OpenAI).
|
|
15
15
|
|
|
16
|
+
## Input Validation
|
|
17
|
+
|
|
18
|
+
Validate invocation input before forwarding it to the agent framework. Keep plain prompts typed as strings and pass
|
|
19
|
+
only prompt text to the agent.
|
|
20
|
+
|
|
16
21
|
## Environment Variables
|
|
17
22
|
|
|
18
23
|
| Variable | Required | Description |
|
|
@@ -184,6 +184,8 @@ async def invoke(payload, context):
|
|
|
184
184
|
|
|
185
185
|
# Process the user prompt
|
|
186
186
|
prompt = payload.get("prompt", "What can you help me with?")
|
|
187
|
+
if not isinstance(prompt, str):
|
|
188
|
+
raise ValueError("prompt must be a string")
|
|
187
189
|
session_id = getattr(context, "session_id", "default-session")
|
|
188
190
|
session = get_session(session_id)
|
|
189
191
|
|
|
@@ -13,6 +13,12 @@ file defines a Starlette ASGI app with the chosen Agent framework SDK running wi
|
|
|
13
13
|
|
|
14
14
|
`model/load.py` instantiates your chosen model provider.
|
|
15
15
|
|
|
16
|
+
## Input Validation
|
|
17
|
+
|
|
18
|
+
Validate invocation input before forwarding it to Strands. Keep plain prompts typed as strings. If the app accepts a
|
|
19
|
+
caller-supplied message history, retain `strip_trailing_tool_use()`, which normalizes the history tail before
|
|
20
|
+
invoking the agent.
|
|
21
|
+
|
|
16
22
|
## Environment Variables
|
|
17
23
|
|
|
18
24
|
| Variable | Required | Description |
|
|
@@ -481,17 +481,53 @@ get_or_create_agent = agent_factory()
|
|
|
481
481
|
{{/if}}
|
|
482
482
|
|
|
483
483
|
|
|
484
|
+
def strip_trailing_tool_use(messages: Any) -> list[dict]:
|
|
485
|
+
"""Strip toolUse blocks from the tail until the last message has none."""
|
|
486
|
+
if not isinstance(messages, list):
|
|
487
|
+
raise ValueError("messages must be a list")
|
|
488
|
+
|
|
489
|
+
messages = list(messages)
|
|
490
|
+
while messages:
|
|
491
|
+
last = messages[-1]
|
|
492
|
+
if not isinstance(last, dict):
|
|
493
|
+
raise ValueError("each message must be an object")
|
|
494
|
+
original_content = last.get("content", [])
|
|
495
|
+
if not isinstance(original_content, list) or not all(isinstance(block, dict) for block in original_content):
|
|
496
|
+
raise ValueError("each message content value must be a list of content blocks")
|
|
497
|
+
|
|
498
|
+
content = [block for block in original_content if "toolUse" not in block]
|
|
499
|
+
if len(content) == len(original_content):
|
|
500
|
+
break
|
|
501
|
+
if content:
|
|
502
|
+
messages[-1] = {**last, "content": content}
|
|
503
|
+
break
|
|
504
|
+
messages.pop()
|
|
505
|
+
|
|
506
|
+
return messages
|
|
507
|
+
|
|
508
|
+
|
|
484
509
|
def _extract_prompt(payload: dict):
|
|
485
|
-
"""Accept harness
|
|
510
|
+
"""Accept validated harness messages, tool results, or a plain prompt string."""
|
|
511
|
+
if not isinstance(payload, dict):
|
|
512
|
+
raise ValueError("payload must be a JSON object")
|
|
486
513
|
if "messages" in payload:
|
|
487
|
-
return payload["messages"]
|
|
514
|
+
return strip_trailing_tool_use(payload["messages"])
|
|
488
515
|
if "tool_results" in payload:
|
|
516
|
+
tool_results = payload["tool_results"]
|
|
517
|
+
if not isinstance(tool_results, list) or not all(
|
|
518
|
+
isinstance(tool_result, dict) and isinstance(tool_result.get("toolUseId"), str)
|
|
519
|
+
for tool_result in tool_results
|
|
520
|
+
):
|
|
521
|
+
raise ValueError("tool_results must contain objects with a toolUseId string")
|
|
489
522
|
return [{"role": "user", "content": [{"toolResult": {
|
|
490
523
|
"toolUseId": tr["toolUseId"],
|
|
491
524
|
"status": tr.get("status", "success"),
|
|
492
525
|
"content": tr.get("content", []),
|
|
493
|
-
}} for tr in
|
|
494
|
-
|
|
526
|
+
}} for tr in tool_results]}]
|
|
527
|
+
prompt = payload.get("prompt", "")
|
|
528
|
+
if not isinstance(prompt, str):
|
|
529
|
+
raise ValueError("prompt must be a string")
|
|
530
|
+
return prompt
|
|
495
531
|
|
|
496
532
|
|
|
497
533
|
def _has_inline_function_call(messages) -> bool:
|