@deployfoundation/foundation-deploy 0.2.49 → 0.2.51
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agent-image/Dockerfile +43 -3
- package/dist/bin/app.js +1 -1
- package/dist/bin/foundation-deploy.js +27 -0
- package/dist/{chunk-6f2hkamj.js → chunk-efqdhd2e.js} +85 -7
- package/dist/src/index.js +1 -1
- package/package.json +1 -1
- package/src/deploy/endpoint.ts +3 -1
- package/src/stacks/agent-stack.ts +13 -1
- package/src/stacks/api-stack.ts +139 -14
package/agent-image/Dockerfile
CHANGED
|
@@ -20,6 +20,8 @@
|
|
|
20
20
|
# playwright-core 1.63.0 (exact pins + frozen lockfile in
|
|
21
21
|
# packages/infra/agent-image/open-slide/), whose Chrome Headless Shell
|
|
22
22
|
# build is pinned by the playwright-core version
|
|
23
|
+
# - Node.js 24.22.1 (LTS) with npm, and pnpm through corepack, for the
|
|
24
|
+
# repositories the agent clones and tests (see the node-provider stage)
|
|
23
25
|
#
|
|
24
26
|
# Secrets bootstrap: the container receives ONE pointer env var,
|
|
25
27
|
# RUNTIME_SECRET_ID. At start, server.ts fetches that JSON secret from Secrets
|
|
@@ -40,6 +42,9 @@ ARG GH_VERSION=2.97.0
|
|
|
40
42
|
# no ECR access; a deploy pipeline can override it with a mirrored tag to stay
|
|
41
43
|
# clear of Docker Hub's anonymous rate limit.
|
|
42
44
|
ARG BUN_IMAGE=oven/bun:1.4.0-slim
|
|
45
|
+
# NODE_IMAGE: where the real Node.js comes from (node-provider stage). AWS's
|
|
46
|
+
# mirror of the Docker official image, like golang below.
|
|
47
|
+
ARG NODE_IMAGE=public.ecr.aws/docker/library/node:24.22.1-trixie-slim
|
|
43
48
|
# GH_IMAGE: where the gh BINARY comes from. The default names the `gh-source`
|
|
44
49
|
# stage below, so a local build compiles it; an override points at a prebuilt
|
|
45
50
|
# image, and under BuildKit an unreferenced stage never runs — which takes the
|
|
@@ -129,6 +134,15 @@ COPY skills/slides/export_deck.mjs skills/slides/starter.tsx ./
|
|
|
129
134
|
# below is what makes the choice load-bearing.
|
|
130
135
|
FROM ${GH_IMAGE} AS gh-provider
|
|
131
136
|
|
|
137
|
+
# --- Node.js (repositories the agent works in) -------------------------------
|
|
138
|
+
# The bun base ships only Bun's `node` shim (/usr/local/bun-node-fallback-bin),
|
|
139
|
+
# which refuses real Node invocations ("Missing script to execute"). Turns in a
|
|
140
|
+
# Node repository then began by downloading and unpacking a Node tarball before
|
|
141
|
+
# they could typecheck or test anything (Olivia, every PR turn, 2026-10). The
|
|
142
|
+
# runtime stage copies the official build's binary and its bundled npm and
|
|
143
|
+
# corepack from here; nothing else from this image is used.
|
|
144
|
+
FROM ${NODE_IMAGE} AS node-provider
|
|
145
|
+
|
|
132
146
|
# --- runtime ----------------------------------------------------------------
|
|
133
147
|
FROM ${BUN_IMAGE}
|
|
134
148
|
|
|
@@ -142,8 +156,12 @@ ARG FOUNDATION_GITHUB_APP_ID=0
|
|
|
142
156
|
# credential probe checks — without this pointer the omp Bedrock provider
|
|
143
157
|
# refuses with "No API key found". The real IMDS control is the tool policy
|
|
144
158
|
# denying 169.254.169.254, not an env var a raw curl could ignore.
|
|
159
|
+
#
|
|
160
|
+
# No NODE_ENV=production: nothing in Foundation reads it (the install below
|
|
161
|
+
# passes --production itself), and the agent's shell inherits the container
|
|
162
|
+
# env, so `pnpm install` and `npm install` in a cloned repository skipped its
|
|
163
|
+
# devDependencies and every typecheck or test then failed on missing modules.
|
|
145
164
|
ENV AWS_EC2_METADATA_SERVICE_ENDPOINT=http://169.254.169.254 \
|
|
146
|
-
NODE_ENV=production \
|
|
147
165
|
PYTHONDONTWRITEBYTECODE=1 \
|
|
148
166
|
PYTHONUNBUFFERED=1 \
|
|
149
167
|
# omp env sheet: relocate the agent base to the baked container dir
|
|
@@ -161,7 +179,12 @@ ENV AWS_EC2_METADATA_SERVICE_ENDPOINT=http://169.254.169.254 \
|
|
|
161
179
|
# loopback broker with it. poppler-utils supplies `pdftotext`, the ONLY way a
|
|
162
180
|
# PDF dropped in a thread becomes readable text (files.ts shells out to it
|
|
163
181
|
# rather than adding a PDF parser dependency); missing here, every PDF
|
|
164
|
-
# silently degrades to a path the model cannot open.
|
|
182
|
+
# silently degrades to a path the model cannot open. xz-utils (`xz`, the CLI
|
|
183
|
+
# over the liblzma5 already in the base) unpacks `.tar.xz` downloads, and
|
|
184
|
+
# `file` names what an attachment or download really is before the model
|
|
185
|
+
# picks a tool for it. Together about 12 MB installed (most of it libmagic's
|
|
186
|
+
# database); both are plain Debian CLIs that run no daemon and hold no
|
|
187
|
+
# credentials. Olivia's turns reached for both and found neither.
|
|
165
188
|
#
|
|
166
189
|
# `upgrade` (not just `install`) is what picks up Debian security patches for
|
|
167
190
|
# packages the base image baked in — without it, glibc/systemd/libcap2/liblzma5
|
|
@@ -175,7 +198,7 @@ RUN export DEBIAN_FRONTEND=noninteractive \
|
|
|
175
198
|
libc6 libc-bin libsystemd0 libudev1 libcap2 liblzma5 sed \
|
|
176
199
|
&& apt-get install -y --no-install-recommends \
|
|
177
200
|
python3 python3-venv python3-pip ca-certificates git curl \
|
|
178
|
-
poppler-utils \
|
|
201
|
+
poppler-utils xz-utils file \
|
|
179
202
|
libasound2t64 libatk-bridge2.0-0t64 libatk1.0-0t64 libatspi2.0-0t64 \
|
|
180
203
|
libcairo2 libcups2t64 libdbus-1-3 libdrm2 libgbm1 libglib2.0-0t64 \
|
|
181
204
|
libnspr4 libnss3 libpango-1.0-0 libx11-6 libxcb1 libxcomposite1 \
|
|
@@ -234,6 +257,23 @@ RUN chmod 0755 /opt/sky/bin/export_deck
|
|
|
234
257
|
ENV OPEN_SLIDE_HOME=/opt/open-slide \
|
|
235
258
|
PLAYWRIGHT_BROWSERS_PATH=/opt/open-slide/browsers
|
|
236
259
|
|
|
260
|
+
# --- Node.js, npm and pnpm (node-provider stage) -----------------------------
|
|
261
|
+
# /usr/local/bin precedes the bun base's fallback directory on PATH, so `node`
|
|
262
|
+
# is the real binary from here on; the build fails if it is not. pnpm is a
|
|
263
|
+
# corepack shim: it runs the version a repository's `packageManager` names,
|
|
264
|
+
# fetched on first use into the runtime user's cache, so no repository's pin is
|
|
265
|
+
# baked into the product. The prompt corepack shows before that fetch would
|
|
266
|
+
# stall a non-interactive shell.
|
|
267
|
+
COPY --from=node-provider /usr/local/bin/node /usr/local/bin/node
|
|
268
|
+
COPY --from=node-provider /usr/local/lib/node_modules /usr/local/lib/node_modules
|
|
269
|
+
RUN ln -s ../lib/node_modules/npm/bin/npm-cli.js /usr/local/bin/npm \
|
|
270
|
+
&& ln -s ../lib/node_modules/npm/bin/npx-cli.js /usr/local/bin/npx \
|
|
271
|
+
&& ln -s ../lib/node_modules/corepack/dist/corepack.js /usr/local/bin/corepack \
|
|
272
|
+
&& corepack enable --install-directory /usr/local/bin pnpm \
|
|
273
|
+
&& test "$(command -v node)" = /usr/local/bin/node \
|
|
274
|
+
&& node --version && npm --version && corepack --version
|
|
275
|
+
ENV COREPACK_ENABLE_DOWNLOAD_PROMPT=0
|
|
276
|
+
|
|
237
277
|
# /opt/sky/bin FIRST so `gh` and `aws` resolve to the wrappers, never the real
|
|
238
278
|
# binaries.
|
|
239
279
|
ENV PATH="/opt/sky/bin:${PATH}"
|
package/dist/bin/app.js
CHANGED
|
@@ -862,6 +862,33 @@ var UPWORK_GRAPHQL_DOCUMENTS = Object.freeze({
|
|
|
862
862
|
$filter: VendorProposalFilter!
|
|
863
863
|
$sortAttribute: VendorProposalSortAttribute!
|
|
864
864
|
$pagination: Pagination!
|
|
865
|
+
) {
|
|
866
|
+
vendorProposals(
|
|
867
|
+
filter: $filter
|
|
868
|
+
sortAttribute: $sortAttribute
|
|
869
|
+
pagination: $pagination
|
|
870
|
+
) {
|
|
871
|
+
totalCount
|
|
872
|
+
edges {
|
|
873
|
+
node {
|
|
874
|
+
id
|
|
875
|
+
status { status }
|
|
876
|
+
marketplaceJobPosting { id content { title } }
|
|
877
|
+
terms { chargeRate { displayValue } }
|
|
878
|
+
viewedByClient
|
|
879
|
+
auditDetails { createdDateTime { rawValue } }
|
|
880
|
+
}
|
|
881
|
+
}
|
|
882
|
+
pageInfo { endCursor hasNextPage }
|
|
883
|
+
}
|
|
884
|
+
}`
|
|
885
|
+
}),
|
|
886
|
+
list_proposals_basic: Object.freeze({
|
|
887
|
+
operationName: "FreelancerVendorProposalsBasic",
|
|
888
|
+
query: `query FreelancerVendorProposalsBasic(
|
|
889
|
+
$filter: VendorProposalFilter!
|
|
890
|
+
$sortAttribute: VendorProposalSortAttribute!
|
|
891
|
+
$pagination: Pagination!
|
|
865
892
|
) {
|
|
866
893
|
vendorProposals(
|
|
867
894
|
filter: $filter
|
|
@@ -670,9 +670,14 @@ class FoundationAgent extends cdk.Stack {
|
|
|
670
670
|
}));
|
|
671
671
|
role.addToPolicy(new iam.PolicyStatement({
|
|
672
672
|
sid: "ConfigList",
|
|
673
|
-
actions: ["s3:ListBucket"],
|
|
673
|
+
actions: ["s3:ListBucket", "s3:ListBucketVersions"],
|
|
674
674
|
resources: [props.bucket.bucketArn]
|
|
675
675
|
}));
|
|
676
|
+
role.addToPolicy(new iam.PolicyStatement({
|
|
677
|
+
sid: "ConfigLastGood",
|
|
678
|
+
actions: ["s3:GetObjectVersion"],
|
|
679
|
+
resources: [props.bucket.arnForObjects("config/*")]
|
|
680
|
+
}));
|
|
676
681
|
role.addToPolicy(new iam.PolicyStatement({
|
|
677
682
|
sid: "DocumentsObjects",
|
|
678
683
|
actions: ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
|
|
@@ -839,6 +844,7 @@ import * as cwActions from "aws-cdk-lib/aws-cloudwatch-actions";
|
|
|
839
844
|
import * as iam2 from "aws-cdk-lib/aws-iam";
|
|
840
845
|
import * as lambda from "aws-cdk-lib/aws-lambda";
|
|
841
846
|
import * as lambdaEventSources from "aws-cdk-lib/aws-lambda-event-sources";
|
|
847
|
+
import * as logs2 from "aws-cdk-lib/aws-logs";
|
|
842
848
|
import * as scheduler from "aws-cdk-lib/aws-scheduler";
|
|
843
849
|
import * as secretsmanager from "aws-cdk-lib/aws-secretsmanager";
|
|
844
850
|
import * as sns from "aws-cdk-lib/aws-sns";
|
|
@@ -1629,18 +1635,31 @@ class FoundationApi extends cdk2.Stack {
|
|
|
1629
1635
|
["InvokeDlqDepthAlarm", this.invokeDlq]
|
|
1630
1636
|
]) {
|
|
1631
1637
|
const alarm = new cloudwatch.Alarm(this, name, {
|
|
1632
|
-
alarmDescription: `${name}: messages
|
|
1633
|
-
metric:
|
|
1638
|
+
alarmDescription: `${name}: new messages arrived in the DLQ. Inspect them before any redrive.`,
|
|
1639
|
+
metric: new cloudwatch.MathExpression({
|
|
1640
|
+
expression: "DIFF(FILL(depth, 0))",
|
|
1641
|
+
usingMetrics: {
|
|
1642
|
+
depth: queue.metricApproximateNumberOfMessagesVisible({
|
|
1643
|
+
period: cdk2.Duration.minutes(5),
|
|
1644
|
+
statistic: "Maximum"
|
|
1645
|
+
})
|
|
1646
|
+
},
|
|
1634
1647
|
period: cdk2.Duration.minutes(5),
|
|
1635
|
-
|
|
1648
|
+
label: "Messages added to the DLQ"
|
|
1636
1649
|
}),
|
|
1637
|
-
threshold:
|
|
1638
|
-
|
|
1639
|
-
|
|
1650
|
+
threshold: 0,
|
|
1651
|
+
comparisonOperator: cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD,
|
|
1652
|
+
evaluationPeriods: 3,
|
|
1653
|
+
datapointsToAlarm: 1,
|
|
1640
1654
|
treatMissingData: cloudwatch.TreatMissingData.NOT_BREACHING
|
|
1641
1655
|
});
|
|
1642
1656
|
alarm.addAlarmAction(alarmAction);
|
|
1643
1657
|
}
|
|
1658
|
+
addConfigLoadAlarm(this, {
|
|
1659
|
+
instance: props.instance,
|
|
1660
|
+
agentRuntimeArn: props.agentRuntimeArn,
|
|
1661
|
+
alarmAction
|
|
1662
|
+
});
|
|
1644
1663
|
let dispatcherFunctionName;
|
|
1645
1664
|
if (configuredScheduledJobs.length > 0) {
|
|
1646
1665
|
const dispatchRole = new iam2.Role(this, "ScheduledJobDispatcherRole", {
|
|
@@ -1757,6 +1776,65 @@ class FoundationApi extends cdk2.Stack {
|
|
|
1757
1776
|
});
|
|
1758
1777
|
}
|
|
1759
1778
|
}
|
|
1779
|
+
var CONFIG_LOAD_FAILURE_EVENTS = ["config_fallback_active", "config_reload_failed"];
|
|
1780
|
+
function addConfigLoadAlarm(stack, args) {
|
|
1781
|
+
if (args.agentRuntimeArn.endsWith("-pending"))
|
|
1782
|
+
return;
|
|
1783
|
+
const runtimeId = cdk2.Fn.select(1, cdk2.Fn.split("/", args.agentRuntimeArn));
|
|
1784
|
+
const logGroupName = `/aws/bedrock-agentcore/runtimes/${runtimeId}-${LIVE_ENDPOINT_NAME}`;
|
|
1785
|
+
const role = new iam2.Role(stack, "AgentLiveLogGroupRole", {
|
|
1786
|
+
assumedBy: new iam2.ServicePrincipal("lambda.amazonaws.com"),
|
|
1787
|
+
description: `${args.instance.displayName} creates the runtime's live log group when missing`,
|
|
1788
|
+
managedPolicies: [
|
|
1789
|
+
iam2.ManagedPolicy.fromAwsManagedPolicyName("service-role/AWSLambdaBasicExecutionRole")
|
|
1790
|
+
]
|
|
1791
|
+
});
|
|
1792
|
+
role.addToPolicy(new iam2.PolicyStatement({
|
|
1793
|
+
sid: "CreateRuntimeLiveLogGroup",
|
|
1794
|
+
actions: ["logs:CreateLogGroup"],
|
|
1795
|
+
resources: [
|
|
1796
|
+
`arn:${stack.partition}:logs:${stack.region}:${stack.account}:log-group:${logGroupName}`
|
|
1797
|
+
]
|
|
1798
|
+
}));
|
|
1799
|
+
const ensureFunction = new lambda.Function(stack, "AgentLiveLogGroupFunction", {
|
|
1800
|
+
runtime: lambda.Runtime.NODEJS_22_X,
|
|
1801
|
+
handler: "index.runtimeLogGroupHandler",
|
|
1802
|
+
code: lambdaCode(stack, "gateway"),
|
|
1803
|
+
role,
|
|
1804
|
+
timeout: cdk2.Duration.seconds(30),
|
|
1805
|
+
memorySize: 128
|
|
1806
|
+
});
|
|
1807
|
+
const ensureLogGroup = new cdk2.CustomResource(stack, "AgentLiveLogGroup", {
|
|
1808
|
+
serviceToken: ensureFunction.functionArn,
|
|
1809
|
+
resourceType: "Custom::FoundationRuntimeLogGroup",
|
|
1810
|
+
properties: { LogGroupName: logGroupName }
|
|
1811
|
+
});
|
|
1812
|
+
ensureLogGroup.node.addDependency(role);
|
|
1813
|
+
const filter = new logs2.MetricFilter(stack, "ConfigLoadFailureMetricFilter", {
|
|
1814
|
+
logGroup: logs2.LogGroup.fromLogGroupName(stack, "AgentLiveLogs", logGroupName),
|
|
1815
|
+
filterPattern: logs2.FilterPattern.any(...CONFIG_LOAD_FAILURE_EVENTS.map((event) => logs2.FilterPattern.stringValue("$.event", "=", event))),
|
|
1816
|
+
metricNamespace: `Foundation/${args.instance.naming.prefix}`,
|
|
1817
|
+
metricName: "ConfigLoadFailures",
|
|
1818
|
+
metricValue: "1"
|
|
1819
|
+
});
|
|
1820
|
+
filter.node.addDependency(ensureLogGroup);
|
|
1821
|
+
const alarm = new cloudwatch.Alarm(stack, "ConfigLoadFailureAlarm", {
|
|
1822
|
+
alarmDescription: [
|
|
1823
|
+
`${args.instance.displayName}'s runtime is not serving the config deployed for it: the`,
|
|
1824
|
+
"config failed to load, so it runs on its last good config with company-data",
|
|
1825
|
+
"capabilities off, or on the built-in fallback. The config_fallback_active or",
|
|
1826
|
+
"config_reload_failed line in the runtime's live log group names the refused field.",
|
|
1827
|
+
"Fix config.yaml, run foundation-deploy check, then deploy or config:sync."
|
|
1828
|
+
].join(" "),
|
|
1829
|
+
metric: filter.metric({ statistic: "Sum", period: cdk2.Duration.hours(1) }),
|
|
1830
|
+
threshold: 1,
|
|
1831
|
+
evaluationPeriods: 1,
|
|
1832
|
+
comparisonOperator: cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD,
|
|
1833
|
+
treatMissingData: cloudwatch.TreatMissingData.NOT_BREACHING
|
|
1834
|
+
});
|
|
1835
|
+
alarm.addAlarmAction(args.alarmAction);
|
|
1836
|
+
return alarm;
|
|
1837
|
+
}
|
|
1760
1838
|
function crmConfigForProvisioning(configPath) {
|
|
1761
1839
|
const config = parseInstanceConfig(readFileSync(configPath, "utf8"));
|
|
1762
1840
|
const crm = config.capabilities.crm;
|
package/dist/src/index.js
CHANGED
package/package.json
CHANGED
package/src/deploy/endpoint.ts
CHANGED
|
@@ -193,7 +193,9 @@ export async function currentRuntimeVersion(runner: EndpointRunner, id: string):
|
|
|
193
193
|
* What a healthy `_smoke_test` response says. `config_source` is here because
|
|
194
194
|
* a container that booted before its runtime secret named a bucket answers
|
|
195
195
|
* happily from the fallback config — a green smoke test hiding a teammate that
|
|
196
|
-
* knows none of its own configuration.
|
|
196
|
+
* knows none of its own configuration. A runtime whose config did not parse
|
|
197
|
+
* answers `last_good` (an earlier version of its config, fail-closed) or
|
|
198
|
+
* `fallback`, and fails here too.
|
|
197
199
|
*/
|
|
198
200
|
export const SMOKE_EXPECT = ['"authenticated":true', '"config_source":"s3"'];
|
|
199
201
|
|
|
@@ -1045,13 +1045,25 @@ export class FoundationAgent extends cdk.Stack {
|
|
|
1045
1045
|
resources: [props.bucket.arnForObjects("memory/*")],
|
|
1046
1046
|
}),
|
|
1047
1047
|
);
|
|
1048
|
+
// A boot whose own config object does not parse falls back to the newest
|
|
1049
|
+
// earlier VERSION of that object that does (`loadLastGoodConfig` in the
|
|
1050
|
+
// agent's server.ts): deployment-written, read-only, and readable only
|
|
1051
|
+
// under `config/`. Listing versions reveals no more than `ListBucket`
|
|
1052
|
+
// already does (key names), so it is not scoped further.
|
|
1048
1053
|
role.addToPolicy(
|
|
1049
1054
|
new iam.PolicyStatement({
|
|
1050
1055
|
sid: "ConfigList",
|
|
1051
|
-
actions: ["s3:ListBucket"],
|
|
1056
|
+
actions: ["s3:ListBucket", "s3:ListBucketVersions"],
|
|
1052
1057
|
resources: [props.bucket.bucketArn],
|
|
1053
1058
|
}),
|
|
1054
1059
|
);
|
|
1060
|
+
role.addToPolicy(
|
|
1061
|
+
new iam.PolicyStatement({
|
|
1062
|
+
sid: "ConfigLastGood",
|
|
1063
|
+
actions: ["s3:GetObjectVersion"],
|
|
1064
|
+
resources: [props.bucket.arnForObjects("config/*")],
|
|
1065
|
+
}),
|
|
1066
|
+
);
|
|
1055
1067
|
role.addToPolicy(
|
|
1056
1068
|
new iam.PolicyStatement({
|
|
1057
1069
|
sid: "DocumentsObjects",
|
package/src/stacks/api-stack.ts
CHANGED
|
@@ -27,6 +27,7 @@ import * as iam from "aws-cdk-lib/aws-iam";
|
|
|
27
27
|
import type * as kms from "aws-cdk-lib/aws-kms";
|
|
28
28
|
import * as lambda from "aws-cdk-lib/aws-lambda";
|
|
29
29
|
import * as lambdaEventSources from "aws-cdk-lib/aws-lambda-event-sources";
|
|
30
|
+
import * as logs from "aws-cdk-lib/aws-logs";
|
|
30
31
|
import * as scheduler from "aws-cdk-lib/aws-scheduler";
|
|
31
32
|
import * as secretsmanager from "aws-cdk-lib/aws-secretsmanager";
|
|
32
33
|
import * as sns from "aws-cdk-lib/aws-sns";
|
|
@@ -100,9 +101,11 @@ export interface FoundationApiProps extends cdk.StackProps {
|
|
|
100
101
|
/**
|
|
101
102
|
* FoundationApi — the edge: API Gateway → gateway Lambda → SQS → invoker
|
|
102
103
|
* Lambda → `InvokeAgentRuntime`. The gateway must answer Slack inside 3
|
|
103
|
-
* seconds, so it only verifies, dedupes and enqueues; the invoker
|
|
104
|
-
* blocking agent turn for up to Lambda's 15 minutes
|
|
105
|
-
*
|
|
104
|
+
* seconds, so it only verifies, dedupes and enqueues; the invoker follows the
|
|
105
|
+
* blocking agent turn for up to Lambda's 15 minutes. A longer turn keeps
|
|
106
|
+
* running in its session and posts its own answer: the invoker stops waiting
|
|
107
|
+
* just before its deadline and returns, so the message is deleted rather than
|
|
108
|
+
* failed into the DLQ (`DETACH_MARGIN_MS` in gateway/src/invoker.ts).
|
|
106
109
|
*
|
|
107
110
|
* Neither Lambda is VPC-attached. Everything they call — Secrets Manager,
|
|
108
111
|
* SQS, DynamoDB, AgentCore — is a public AWS endpoint, so the VPC would buy
|
|
@@ -193,9 +196,9 @@ export class FoundationApi extends cdk.Stack {
|
|
|
193
196
|
encryption: sqs.QueueEncryption.KMS,
|
|
194
197
|
encryptionMasterKey: props.dataKey,
|
|
195
198
|
enforceSSL: true,
|
|
196
|
-
// Matches the invoker's 15-minute timeout
|
|
197
|
-
//
|
|
198
|
-
// then rolls to the DLQ.
|
|
199
|
+
// Matches the invoker's 15-minute timeout. The invoker always returns
|
|
200
|
+
// before it, even on a longer turn, so only a crashed invoker's message
|
|
201
|
+
// surfaces again, and then it rolls to the DLQ instead of re-running.
|
|
199
202
|
visibilityTimeout: cdk.Duration.seconds(900),
|
|
200
203
|
retentionPeriod: cdk.Duration.hours(6),
|
|
201
204
|
deadLetterQueue: { queue: this.invokeDlq, maxReceiveCount: 1 },
|
|
@@ -1229,25 +1232,51 @@ export class FoundationApi extends cdk.Stack {
|
|
|
1229
1232
|
});
|
|
1230
1233
|
}
|
|
1231
1234
|
const alarmAction = new cwActions.SnsAction(alarmTopic);
|
|
1232
|
-
//
|
|
1233
|
-
// the
|
|
1235
|
+
// A message ARRIVING in either DLQ means a user's message was dropped, so
|
|
1236
|
+
// the alarm fires on growth, not depth. A depth alarm (≥ 1) stays in ALARM
|
|
1237
|
+
// while anything sits in the queue and never notifies again: Olivia's
|
|
1238
|
+
// InvokeDlq alarm sat in ALARM from 2026-09-26 through every later
|
|
1239
|
+
// arrival. This one clears once the depth stops growing and fires again
|
|
1240
|
+
// on the next arrival.
|
|
1241
|
+
//
|
|
1242
|
+
// Not `NumberOfMessagesSent`: SQS does not count a message its redrive
|
|
1243
|
+
// policy moves into a DLQ, which is how every InvokeDlq message arrives.
|
|
1244
|
+
// FILL turns the first arrival into an empty, idle queue (which reports
|
|
1245
|
+
// no data at all) into a step up from 0. One breaching period in three,
|
|
1246
|
+
// because FILL also fills the newest period while its datapoint is still
|
|
1247
|
+
// on the way; the step is caught once it lands. The 5-minute Maximum
|
|
1248
|
+
// hides the brief dip of someone inspecting the queue.
|
|
1234
1249
|
for (const [name, queue] of [
|
|
1235
1250
|
["GatewayDlqDepthAlarm", this.gatewayDlq],
|
|
1236
1251
|
["InvokeDlqDepthAlarm", this.invokeDlq],
|
|
1237
1252
|
] as const) {
|
|
1238
1253
|
const alarm = new cloudwatch.Alarm(this, name, {
|
|
1239
|
-
alarmDescription: `${name}: messages
|
|
1240
|
-
metric:
|
|
1254
|
+
alarmDescription: `${name}: new messages arrived in the DLQ. Inspect them before any redrive.`,
|
|
1255
|
+
metric: new cloudwatch.MathExpression({
|
|
1256
|
+
expression: "DIFF(FILL(depth, 0))",
|
|
1257
|
+
usingMetrics: {
|
|
1258
|
+
depth: queue.metricApproximateNumberOfMessagesVisible({
|
|
1259
|
+
period: cdk.Duration.minutes(5),
|
|
1260
|
+
statistic: "Maximum",
|
|
1261
|
+
}),
|
|
1262
|
+
},
|
|
1241
1263
|
period: cdk.Duration.minutes(5),
|
|
1242
|
-
|
|
1264
|
+
label: "Messages added to the DLQ",
|
|
1243
1265
|
}),
|
|
1244
|
-
threshold:
|
|
1245
|
-
|
|
1246
|
-
|
|
1266
|
+
threshold: 0,
|
|
1267
|
+
comparisonOperator: cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD,
|
|
1268
|
+
evaluationPeriods: 3,
|
|
1269
|
+
datapointsToAlarm: 1,
|
|
1247
1270
|
treatMissingData: cloudwatch.TreatMissingData.NOT_BREACHING,
|
|
1248
1271
|
});
|
|
1249
1272
|
alarm.addAlarmAction(alarmAction);
|
|
1250
1273
|
}
|
|
1274
|
+
// The runtime serving anything but its deployed config.
|
|
1275
|
+
addConfigLoadAlarm(this, {
|
|
1276
|
+
instance: props.instance,
|
|
1277
|
+
agentRuntimeArn: props.agentRuntimeArn,
|
|
1278
|
+
alarmAction,
|
|
1279
|
+
});
|
|
1251
1280
|
|
|
1252
1281
|
// --- deployment-owned scheduled jobs --------------------------------
|
|
1253
1282
|
// Config jobs never share the agent-managed routine group or pass role.
|
|
@@ -1387,6 +1416,102 @@ export class FoundationApi extends cdk.Stack {
|
|
|
1387
1416
|
}
|
|
1388
1417
|
}
|
|
1389
1418
|
|
|
1419
|
+
/** What the runtime logs when it is not serving its deployed config. */
|
|
1420
|
+
export const CONFIG_LOAD_FAILURE_EVENTS = ["config_fallback_active", "config_reload_failed"];
|
|
1421
|
+
|
|
1422
|
+
/**
|
|
1423
|
+
* Alarm when the runtime is not serving the config deployed for it.
|
|
1424
|
+
*
|
|
1425
|
+
* A config the runtime cannot parse used to leave it on the built-in fallback
|
|
1426
|
+
* for hours with nobody told (Sky 2026-10-19, Olivia 2026-10-23). The runtime
|
|
1427
|
+
* now logs `config_fallback_active` once per boot and `config_reload_failed`
|
|
1428
|
+
* once per distinct failure in a warm container; this counts both in the
|
|
1429
|
+
* `live` endpoint's log group, where every human's turn runs, and raises the
|
|
1430
|
+
* instance's alarm topic.
|
|
1431
|
+
*
|
|
1432
|
+
* AgentCore creates that log group on the endpoint's first invocation, and
|
|
1433
|
+
* `live` itself is created by the deploy tool after CloudFormation, so a new
|
|
1434
|
+
* instance has no group for the metric filter to attach to. A custom resource
|
|
1435
|
+
* (`runtimeLogGroupHandler`, in the release-signed gateway bundle) therefore
|
|
1436
|
+
* creates it when it is missing, and leaves it alone when it exists or when
|
|
1437
|
+
* this stack is deleted. A phase-one deploy has no runtime: nothing to watch.
|
|
1438
|
+
*
|
|
1439
|
+
* The period is an hour, not five minutes: the condition lasts until someone
|
|
1440
|
+
* fixes the config, and a short period would re-notify after every quiet gap
|
|
1441
|
+
* between turns.
|
|
1442
|
+
*/
|
|
1443
|
+
export function addConfigLoadAlarm(
|
|
1444
|
+
stack: cdk.Stack,
|
|
1445
|
+
args: {
|
|
1446
|
+
instance: Instance;
|
|
1447
|
+
agentRuntimeArn: string;
|
|
1448
|
+
alarmAction: cloudwatch.IAlarmAction;
|
|
1449
|
+
},
|
|
1450
|
+
): cloudwatch.Alarm | undefined {
|
|
1451
|
+
if (args.agentRuntimeArn.endsWith("-pending")) return undefined;
|
|
1452
|
+
const runtimeId = cdk.Fn.select(1, cdk.Fn.split("/", args.agentRuntimeArn));
|
|
1453
|
+
const logGroupName = `/aws/bedrock-agentcore/runtimes/${runtimeId}-${LIVE_ENDPOINT_NAME}`;
|
|
1454
|
+
const role = new iam.Role(stack, "AgentLiveLogGroupRole", {
|
|
1455
|
+
assumedBy: new iam.ServicePrincipal("lambda.amazonaws.com"),
|
|
1456
|
+
description: `${args.instance.displayName} creates the runtime's live log group when missing`,
|
|
1457
|
+
managedPolicies: [
|
|
1458
|
+
iam.ManagedPolicy.fromAwsManagedPolicyName("service-role/AWSLambdaBasicExecutionRole"),
|
|
1459
|
+
],
|
|
1460
|
+
});
|
|
1461
|
+
role.addToPolicy(
|
|
1462
|
+
new iam.PolicyStatement({
|
|
1463
|
+
sid: "CreateRuntimeLiveLogGroup",
|
|
1464
|
+
actions: ["logs:CreateLogGroup"],
|
|
1465
|
+
resources: [
|
|
1466
|
+
`arn:${stack.partition}:logs:${stack.region}:${stack.account}:log-group:${logGroupName}`,
|
|
1467
|
+
],
|
|
1468
|
+
}),
|
|
1469
|
+
);
|
|
1470
|
+
const ensureFunction = new lambda.Function(stack, "AgentLiveLogGroupFunction", {
|
|
1471
|
+
runtime: lambda.Runtime.NODEJS_22_X,
|
|
1472
|
+
handler: "index.runtimeLogGroupHandler",
|
|
1473
|
+
code: lambdaCode(stack, "gateway"),
|
|
1474
|
+
role,
|
|
1475
|
+
timeout: cdk.Duration.seconds(30),
|
|
1476
|
+
memorySize: 128,
|
|
1477
|
+
});
|
|
1478
|
+
const ensureLogGroup = new cdk.CustomResource(stack, "AgentLiveLogGroup", {
|
|
1479
|
+
serviceToken: ensureFunction.functionArn,
|
|
1480
|
+
resourceType: "Custom::FoundationRuntimeLogGroup",
|
|
1481
|
+
properties: { LogGroupName: logGroupName },
|
|
1482
|
+
});
|
|
1483
|
+
// The function needs its permission before CloudFormation invokes it.
|
|
1484
|
+
ensureLogGroup.node.addDependency(role);
|
|
1485
|
+
const filter = new logs.MetricFilter(stack, "ConfigLoadFailureMetricFilter", {
|
|
1486
|
+
logGroup: logs.LogGroup.fromLogGroupName(stack, "AgentLiveLogs", logGroupName),
|
|
1487
|
+
filterPattern: logs.FilterPattern.any(
|
|
1488
|
+
...CONFIG_LOAD_FAILURE_EVENTS.map((event) =>
|
|
1489
|
+
logs.FilterPattern.stringValue("$.event", "=", event),
|
|
1490
|
+
),
|
|
1491
|
+
),
|
|
1492
|
+
metricNamespace: `Foundation/${args.instance.naming.prefix}`,
|
|
1493
|
+
metricName: "ConfigLoadFailures",
|
|
1494
|
+
metricValue: "1",
|
|
1495
|
+
});
|
|
1496
|
+
filter.node.addDependency(ensureLogGroup);
|
|
1497
|
+
const alarm = new cloudwatch.Alarm(stack, "ConfigLoadFailureAlarm", {
|
|
1498
|
+
alarmDescription: [
|
|
1499
|
+
`${args.instance.displayName}'s runtime is not serving the config deployed for it: the`,
|
|
1500
|
+
"config failed to load, so it runs on its last good config with company-data",
|
|
1501
|
+
"capabilities off, or on the built-in fallback. The config_fallback_active or",
|
|
1502
|
+
"config_reload_failed line in the runtime's live log group names the refused field.",
|
|
1503
|
+
"Fix config.yaml, run foundation-deploy check, then deploy or config:sync.",
|
|
1504
|
+
].join(" "),
|
|
1505
|
+
metric: filter.metric({ statistic: "Sum", period: cdk.Duration.hours(1) }),
|
|
1506
|
+
threshold: 1,
|
|
1507
|
+
evaluationPeriods: 1,
|
|
1508
|
+
comparisonOperator: cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD,
|
|
1509
|
+
treatMissingData: cloudwatch.TreatMissingData.NOT_BREACHING,
|
|
1510
|
+
});
|
|
1511
|
+
alarm.addAlarmAction(args.alarmAction);
|
|
1512
|
+
return alarm;
|
|
1513
|
+
}
|
|
1514
|
+
|
|
1390
1515
|
interface ProvisionedCrmConfig {
|
|
1391
1516
|
channels: string[];
|
|
1392
1517
|
statuses: string[];
|