@aws/agentcore 0.3.0-preview.5.0 → 0.3.0-preview.6.0

package/dist/schema/schemas/primitives/evaluator.js

@@ -0,0 +1,58 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.EvaluatorConfigSchema = exports.LlmAsAJudgeConfigSchema = exports.BedrockModelIdSchema = exports.RatingScaleSchema = exports.CategoricalRatingSchema = exports.NumericalRatingSchema = exports.EvaluatorNameSchema = exports.EvaluationLevelSchema = void 0;
+exports.isValidBedrockModelId = isValidBedrockModelId;
+const zod_1 = require("zod");
+// ============================================================================
+// Evaluator Types
+// ============================================================================
+exports.EvaluationLevelSchema = zod_1.z.enum(['SESSION', 'TRACE', 'TOOL_CALL']);
+exports.EvaluatorNameSchema = zod_1.z
+    .string()
+    .min(1, 'Name is required')
+    .max(48)
+    .regex(/^[a-zA-Z][a-zA-Z0-9_]{0,47}$/, 'Must begin with a letter and contain only alphanumeric characters and underscores (max 48 chars)');
+// ============================================================================
+// Rating Scale
+// ============================================================================
+exports.NumericalRatingSchema = zod_1.z.object({
+    value: zod_1.z.number().int(),
+    label: zod_1.z.string().min(1),
+    definition: zod_1.z.string().min(1),
+});
+exports.CategoricalRatingSchema = zod_1.z.object({
+    label: zod_1.z.string().min(1),
+    definition: zod_1.z.string().min(1),
+});
+exports.RatingScaleSchema = zod_1.z
+    .object({
+    numerical: zod_1.z.array(exports.NumericalRatingSchema).optional(),
+    categorical: zod_1.z.array(exports.CategoricalRatingSchema).optional(),
+})
+    .refine(scale => {
+    const hasNumerical = Boolean(scale.numerical);
+    const hasCategorical = Boolean(scale.categorical);
+    return hasNumerical !== hasCategorical;
+}, { message: 'Rating scale must have either numerical or categorical, not both' });
+// ============================================================================
+// LLM-as-a-Judge Config
+// ============================================================================
+// eslint-disable-next-line security/detect-unsafe-regex -- anchored pattern, no backtracking risk
+const BEDROCK_MODEL_ID_PATTERN = /^[a-z][a-z0-9-]*\.[a-zA-Z0-9._-]+(:[0-9]+)?$/;
+const BEDROCK_ARN_PATTERN = /^arn:aws[a-z-]*:bedrock:[a-z0-9-]+:\d{12}:(inference-profile|foundation-model)\/.+$/;
+function isValidBedrockModelId(value) {
+    return BEDROCK_MODEL_ID_PATTERN.test(value) || BEDROCK_ARN_PATTERN.test(value);
+}
+exports.BedrockModelIdSchema = zod_1.z.string().min(1, 'Model ID is required');
+exports.LlmAsAJudgeConfigSchema = zod_1.z.object({
+    model: exports.BedrockModelIdSchema,
+    instructions: zod_1.z.string().min(1, 'Evaluation instructions are required'),
+    ratingScale: exports.RatingScaleSchema,
+});
+// ============================================================================
+// Evaluator Config
+// ============================================================================
+exports.EvaluatorConfigSchema = zod_1.z.object({
+    llmAsAJudge: exports.LlmAsAJudgeConfigSchema,
+});
+//# sourceMappingURL=evaluator.js.map

package/dist/schema/schemas/primitives/evaluator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"evaluator.js","sourceRoot":"","sources":["../../../../src/schema/schemas/primitives/evaluator.ts"],"names":[],"mappings":";;;AA6DA,sDAEC;AA/DD,6BAAwB;AAExB,+EAA+E;AAC/E,kBAAkB;AAClB,+EAA+E;AAElE,QAAA,qBAAqB,GAAG,OAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,OAAO,EAAE,WAAW,CAAC,CAAC,CAAC;AAGlE,QAAA,mBAAmB,GAAG,OAAC;KACjC,MAAM,EAAE;KACR,GAAG,CAAC,CAAC,EAAE,kBAAkB,CAAC;KAC1B,GAAG,CAAC,EAAE,CAAC;KACP,KAAK,CACJ,8BAA8B,EAC9B,kGAAkG,CACnG,CAAC;AAEJ,+EAA+E;AAC/E,eAAe;AACf,+EAA+E;AAElE,QAAA,qBAAqB,GAAG,OAAC,CAAC,MAAM,CAAC;IAC5C,KAAK,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE;IACvB,KAAK,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACxB,UAAU,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;CAC9B,CAAC,CAAC;AAIU,QAAA,uBAAuB,GAAG,OAAC,CAAC,MAAM,CAAC;IAC9C,KAAK,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACxB,UAAU,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;CAC9B,CAAC,CAAC;AAIU,QAAA,iBAAiB,GAAG,OAAC;KAC/B,MAAM,CAAC;IACN,SAAS,EAAE,OAAC,CAAC,KAAK,CAAC,6BAAqB,CAAC,CAAC,QAAQ,EAAE;IACpD,WAAW,EAAE,OAAC,CAAC,KAAK,CAAC,+BAAuB,CAAC,CAAC,QAAQ,EAAE;CACzD,CAAC;KACD,MAAM,CACL,KAAK,CAAC,EAAE;IACN,MAAM,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IAC9C,MAAM,cAAc,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;IAClD,OAAO,YAAY,KAAK,cAAc,CAAC;AACzC,CAAC,EACD,EAAE,OAAO,EAAE,kEAAkE,EAAE,CAChF,CAAC;AAIJ,+EAA+E;AAC/E,wBAAwB;AACxB,+EAA+E;AAE/E,kGAAkG;AAClG,MAAM,wBAAwB,GAAG,8CAA8C,CAAC;AAChF,MAAM,mBAAmB,GAAG,qFAAqF,CAAC;AAElH,SAAgB,qBAAqB,CAAC,KAAa;IACjD,OAAO,wBAAwB,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,mBAAmB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AACjF,CAAC;AAEY,QAAA,oBAAoB,GAAG,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,sBAAsB,CAAC,CAAC;AAEjE,QAAA,uBAAuB,GAAG,OAAC,CAAC,MAAM,CAAC;IAC9C,KAAK,EAAE,4BAAoB;IAC3B,YAAY,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,sCAAsC,CAAC;IACvE,WAAW,EAAE,yBAAiB;CAC/B,CAAC,CAAC;AAIH,+EAA+E;AAC/E,mBAAmB;AACnB,+EAA+E;AAElE,QAAA,qBAAqB,GAAG,OAAC,CAAC,MAAM,CAAC;IAC5C,WAAW,EAAE,+BAAuB;CACrC,CAAC,CAAC"}

package/dist/schema/schemas/primitives/index.d.ts

@@ -1,3 +1,7 @@
 export type { MemoryStrategy, MemoryStrategyType } from './memory';
 export { DEFAULT_STRATEGY_NAMESPACES, MemoryStrategyNameSchema, MemoryStrategySchema, MemoryStrategyTypeSchema, } from './memory';
+export type { EvaluationLevel, EvaluatorConfig, LlmAsAJudgeConfig, RatingScale, NumericalRating, CategoricalRating, } from './evaluator';
+export { BedrockModelIdSchema, isValidBedrockModelId, EvaluationLevelSchema, EvaluatorConfigSchema, EvaluatorNameSchema, LlmAsAJudgeConfigSchema, RatingScaleSchema, NumericalRatingSchema, CategoricalRatingSchema, } from './evaluator';
+export type { OnlineEvalConfig } from './online-eval-config';
+export { OnlineEvalConfigSchema, OnlineEvalConfigNameSchema } from './online-eval-config';
 //# sourceMappingURL=index.d.ts.map

package/dist/schema/schemas/primitives/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/schema/schemas/primitives/index.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,cAAc,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AACnE,OAAO,EACL,2BAA2B,EAC3B,wBAAwB,EACxB,oBAAoB,EACpB,wBAAwB,GACzB,MAAM,UAAU,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/schema/schemas/primitives/index.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,cAAc,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AACnE,OAAO,EACL,2BAA2B,EAC3B,wBAAwB,EACxB,oBAAoB,EACpB,wBAAwB,GACzB,MAAM,UAAU,CAAC;AAElB,YAAY,EACV,eAAe,EACf,eAAe,EACf,iBAAiB,EACjB,WAAW,EACX,eAAe,EACf,iBAAiB,GAClB,MAAM,aAAa,CAAC;AACrB,OAAO,EACL,oBAAoB,EACpB,qBAAqB,EACrB,qBAAqB,EACrB,qBAAqB,EACrB,mBAAmB,EACnB,uBAAuB,EACvB,iBAAiB,EACjB,qBAAqB,EACrB,uBAAuB,GACxB,MAAM,aAAa,CAAC;AAErB,YAAY,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAC7D,OAAO,EAAE,sBAAsB,EAAE,0BAA0B,EAAE,MAAM,sBAAsB,CAAC"}

package/dist/schema/schemas/primitives/index.js

@@ -1,9 +1,22 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.MemoryStrategyTypeSchema = exports.MemoryStrategySchema = exports.MemoryStrategyNameSchema = exports.DEFAULT_STRATEGY_NAMESPACES = void 0;
+exports.OnlineEvalConfigNameSchema = exports.OnlineEvalConfigSchema = exports.CategoricalRatingSchema = exports.NumericalRatingSchema = exports.RatingScaleSchema = exports.LlmAsAJudgeConfigSchema = exports.EvaluatorNameSchema = exports.EvaluatorConfigSchema = exports.EvaluationLevelSchema = exports.isValidBedrockModelId = exports.BedrockModelIdSchema = exports.MemoryStrategyTypeSchema = exports.MemoryStrategySchema = exports.MemoryStrategyNameSchema = exports.DEFAULT_STRATEGY_NAMESPACES = void 0;
 var memory_1 = require("./memory");
 Object.defineProperty(exports, "DEFAULT_STRATEGY_NAMESPACES", { enumerable: true, get: function () { return memory_1.DEFAULT_STRATEGY_NAMESPACES; } });
 Object.defineProperty(exports, "MemoryStrategyNameSchema", { enumerable: true, get: function () { return memory_1.MemoryStrategyNameSchema; } });
 Object.defineProperty(exports, "MemoryStrategySchema", { enumerable: true, get: function () { return memory_1.MemoryStrategySchema; } });
 Object.defineProperty(exports, "MemoryStrategyTypeSchema", { enumerable: true, get: function () { return memory_1.MemoryStrategyTypeSchema; } });
+var evaluator_1 = require("./evaluator");
+Object.defineProperty(exports, "BedrockModelIdSchema", { enumerable: true, get: function () { return evaluator_1.BedrockModelIdSchema; } });
+Object.defineProperty(exports, "isValidBedrockModelId", { enumerable: true, get: function () { return evaluator_1.isValidBedrockModelId; } });
+Object.defineProperty(exports, "EvaluationLevelSchema", { enumerable: true, get: function () { return evaluator_1.EvaluationLevelSchema; } });
+Object.defineProperty(exports, "EvaluatorConfigSchema", { enumerable: true, get: function () { return evaluator_1.EvaluatorConfigSchema; } });
+Object.defineProperty(exports, "EvaluatorNameSchema", { enumerable: true, get: function () { return evaluator_1.EvaluatorNameSchema; } });
+Object.defineProperty(exports, "LlmAsAJudgeConfigSchema", { enumerable: true, get: function () { return evaluator_1.LlmAsAJudgeConfigSchema; } });
+Object.defineProperty(exports, "RatingScaleSchema", { enumerable: true, get: function () { return evaluator_1.RatingScaleSchema; } });
+Object.defineProperty(exports, "NumericalRatingSchema", { enumerable: true, get: function () { return evaluator_1.NumericalRatingSchema; } });
+Object.defineProperty(exports, "CategoricalRatingSchema", { enumerable: true, get: function () { return evaluator_1.CategoricalRatingSchema; } });
+var online_eval_config_1 = require("./online-eval-config");
+Object.defineProperty(exports, "OnlineEvalConfigSchema", { enumerable: true, get: function () { return online_eval_config_1.OnlineEvalConfigSchema; } });
+Object.defineProperty(exports, "OnlineEvalConfigNameSchema", { enumerable: true, get: function () { return online_eval_config_1.OnlineEvalConfigNameSchema; } });
 //# sourceMappingURL=index.js.map

package/dist/schema/schemas/primitives/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/schema/schemas/primitives/index.ts"],"names":[],"mappings":";;;AACA,mCAKkB;AAJhB,qHAAA,2BAA2B,OAAA;AAC3B,kHAAA,wBAAwB,OAAA;AACxB,8GAAA,oBAAoB,OAAA;AACpB,kHAAA,wBAAwB,OAAA"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/schema/schemas/primitives/index.ts"],"names":[],"mappings":";;;AACA,mCAKkB;AAJhB,qHAAA,2BAA2B,OAAA;AAC3B,kHAAA,wBAAwB,OAAA;AACxB,8GAAA,oBAAoB,OAAA;AACpB,kHAAA,wBAAwB,OAAA;AAW1B,yCAUqB;AATnB,iHAAA,oBAAoB,OAAA;AACpB,kHAAA,qBAAqB,OAAA;AACrB,kHAAA,qBAAqB,OAAA;AACrB,kHAAA,qBAAqB,OAAA;AACrB,gHAAA,mBAAmB,OAAA;AACnB,oHAAA,uBAAuB,OAAA;AACvB,8GAAA,iBAAiB,OAAA;AACjB,kHAAA,qBAAqB,OAAA;AACrB,oHAAA,uBAAuB,OAAA;AAIzB,2DAA0F;AAAjF,4HAAA,sBAAsB,OAAA;AAAE,gIAAA,0BAA0B,OAAA"}

package/dist/schema/schemas/primitives/online-eval-config.d.ts

@@ -0,0 +1,13 @@
+import { z } from 'zod';
+export declare const OnlineEvalConfigNameSchema: z.ZodString;
+export declare const OnlineEvalConfigSchema: z.ZodObject<{
+    type: z.ZodLiteral<"OnlineEvaluationConfig">;
+    name: z.ZodString;
+    agent: z.ZodString;
+    evaluators: z.ZodArray<z.ZodString>;
+    samplingRate: z.ZodNumber;
+    description: z.ZodOptional<z.ZodString>;
+    enableOnCreate: z.ZodOptional<z.ZodBoolean>;
+}, z.core.$strip>;
+export type OnlineEvalConfig = z.infer<typeof OnlineEvalConfigSchema>;
+//# sourceMappingURL=online-eval-config.d.ts.map

package/dist/schema/schemas/primitives/online-eval-config.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"online-eval-config.d.ts","sourceRoot":"","sources":["../../../../src/schema/schemas/primitives/online-eval-config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAMxB,eAAO,MAAM,0BAA0B,aAOpC,CAAC;AAEJ,eAAO,MAAM,sBAAsB;;;;;;;;iBAajC,CAAC;AAEH,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC"}

package/dist/schema/schemas/primitives/online-eval-config.js

@@ -0,0 +1,27 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.OnlineEvalConfigSchema = exports.OnlineEvalConfigNameSchema = void 0;
+const zod_1 = require("zod");
+// ============================================================================
+// Online Eval Config Types
+// ============================================================================
+exports.OnlineEvalConfigNameSchema = zod_1.z
+    .string()
+    .min(1, 'Name is required')
+    .max(48)
+    .regex(/^[a-zA-Z][a-zA-Z0-9_]{0,47}$/, 'Must begin with a letter and contain only alphanumeric characters and underscores (max 48 chars)');
+exports.OnlineEvalConfigSchema = zod_1.z.object({
+    type: zod_1.z.literal('OnlineEvaluationConfig'),
+    name: exports.OnlineEvalConfigNameSchema,
+    /** Agent name to monitor (must match a project agent) */
+    agent: zod_1.z.string().min(1, 'Agent name is required'),
+    /** Evaluator names (custom), Builtin.* IDs, or evaluator ARNs */
+    evaluators: zod_1.z.array(zod_1.z.string().min(1)).min(1, 'At least one evaluator is required'),
+    /** Sampling rate as a percentage (0.01 to 100) */
+    samplingRate: zod_1.z.number().min(0.01).max(100),
+    /** Optional description for the online eval config */
+    description: zod_1.z.string().max(200).optional(),
+    /** Whether to enable execution on create (default: true) */
+    enableOnCreate: zod_1.z.boolean().optional(),
+});
+//# sourceMappingURL=online-eval-config.js.map

package/dist/schema/schemas/primitives/online-eval-config.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"online-eval-config.js","sourceRoot":"","sources":["../../../../src/schema/schemas/primitives/online-eval-config.ts"],"names":[],"mappings":";;;AAAA,6BAAwB;AAExB,+EAA+E;AAC/E,2BAA2B;AAC3B,+EAA+E;AAElE,QAAA,0BAA0B,GAAG,OAAC;KACxC,MAAM,EAAE;KACR,GAAG,CAAC,CAAC,EAAE,kBAAkB,CAAC;KAC1B,GAAG,CAAC,EAAE,CAAC;KACP,KAAK,CACJ,8BAA8B,EAC9B,kGAAkG,CACnG,CAAC;AAES,QAAA,sBAAsB,GAAG,OAAC,CAAC,MAAM,CAAC;IAC7C,IAAI,EAAE,OAAC,CAAC,OAAO,CAAC,wBAAwB,CAAC;IACzC,IAAI,EAAE,kCAA0B;IAChC,yDAAyD;IACzD,KAAK,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,wBAAwB,CAAC;IAClD,iEAAiE;IACjE,UAAU,EAAE,OAAC,CAAC,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,oCAAoC,CAAC;IACnF,kDAAkD;IAClD,YAAY,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC;IAC3C,sDAAsD;IACtD,WAAW,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE;IAC3C,4DAA4D;IAC5D,cAAc,EAAE,OAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;CACvC,CAAC,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aws/agentcore",
-  "version": "0.3.0-preview.5.0",
+  "version": "0.3.0-preview.6.0",
   "description": "CLI for Amazon Bedrock AgentCore",
   "license": "Apache-2.0",
   "repository": {
@@ -40,7 +40,8 @@
   },
   "files": [
     "dist",
-    "scripts"
+    "scripts",
+    "!dist/mcp-harness"
   ],
   "scripts": {
     "preinstall": "node scripts/check-old-cli.mjs",
@@ -48,6 +49,7 @@
     "build:lib": "tsc -p tsconfig.build.json",
     "build:cli": "node esbuild.config.mjs",
     "build:assets": "node scripts/copy-assets.mjs",
+    "build:harness": "BUILD_HARNESS=1 node esbuild.config.mjs",
     "cli": "npx tsx src/cli/index.ts",
     "typecheck": "tsc --noEmit",
     "lint": "eslint src/",
@@ -64,7 +66,8 @@
     "test:integ": "vitest run --project integ",
     "test:unit": "vitest run --project unit --coverage",
     "test:e2e": "vitest run --project e2e",
-    "test:update-snapshots": "vitest run --project unit --update"
+    "test:update-snapshots": "vitest run --project unit --update",
+    "test:tui": "npm run build:harness && vitest run --project tui"
   },
   "dependencies": {
     "@aws-cdk/toolkit-lib": "^1.16.0",
@@ -96,6 +99,7 @@
   },
   "devDependencies": {
     "@eslint/js": "^9.39.2",
+    "@modelcontextprotocol/sdk": "^1.0.0",
     "@secretlint/secretlint-rule-preset-recommend": "^11.3.0",
     "@trivago/prettier-plugin-sort-imports": "^6.0.2",
     "@types/node": "^25.0.3",
@@ -103,20 +107,22 @@
     "@typescript-eslint/eslint-plugin": "^8.50.0",
     "@typescript-eslint/parser": "^8.50.0",
     "@vitest/coverage-v8": "^4.0.18",
+    "@xterm/headless": "^6.0.0",
     "aws-cdk-lib": "^2.240.0",
     "constructs": "^10.4.4",
     "esbuild": "^0.27.2",
-    "eslint": "^9.39.3",
+    "eslint": "^9.39.4",
     "eslint-config-prettier": "^10.1.8",
     "eslint-import-resolver-typescript": "^4.4.4",
     "eslint-plugin-import": "^2.32.0",
     "eslint-plugin-react": "^7.37.5",
     "eslint-plugin-react-hooks": "^7.0.1",
-    "eslint-plugin-react-refresh": "^0.4.26",
+    "eslint-plugin-react-refresh": "^0.5.2",
     "eslint-plugin-security": "^4.0.0",
     "husky": "^9.1.7",
     "ink-testing-library": "^4.0.0",
     "lint-staged": "^16.2.7",
+    "node-pty": "^1.1.0",
     "prettier": "^3.7.4",
     "secretlint": "^11.3.0",
     "tsx": "^4.21.0",