checkly 7.10.0 → 7.11.0-prerelease-bb99ed9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ai-context/checkly.rules.md +77 -0
- package/dist/ai-context/context.d.ts +6 -0
- package/dist/ai-context/context.js +9 -0
- package/dist/ai-context/context.js.map +1 -1
- package/dist/ai-context/public-skills/checkly/SKILL.md +4 -1
- package/dist/ai-context/skills-command/references/configure-agentic-checks.md +73 -0
- package/dist/ai-context/skills-command/references/configure.md +3 -0
- package/dist/ai-context/skills-command/references/investigate-checks.md +20 -4
- package/dist/constants.d.ts +2 -1
- package/dist/constants.js +1 -0
- package/dist/constants.js.map +1 -1
- package/dist/constructs/agentic-check-codegen.d.ts +34 -0
- package/dist/constructs/agentic-check-codegen.js +94 -0
- package/dist/constructs/agentic-check-codegen.js.map +1 -0
- package/dist/constructs/agentic-check.d.ts +186 -0
- package/dist/constructs/agentic-check.js +136 -0
- package/dist/constructs/agentic-check.js.map +1 -0
- package/dist/constructs/alert-escalation-policy.d.ts +1 -1
- package/dist/constructs/check-codegen.d.ts +22 -1
- package/dist/constructs/check-codegen.js +12 -2
- package/dist/constructs/check-codegen.js.map +1 -1
- package/dist/constructs/index.d.ts +1 -0
- package/dist/constructs/index.js +1 -0
- package/dist/constructs/index.js.map +1 -1
- package/dist/rest/analytics.js +4 -0
- package/dist/rest/analytics.js.map +1 -1
- package/dist/services/check-parser/package-files/package-manager.d.ts +9 -15
- package/dist/services/check-parser/package-files/package-manager.js +24 -34
- package/dist/services/check-parser/package-files/package-manager.js.map +1 -1
- package/dist/testing/fixture-sandbox.js +6 -7
- package/dist/testing/fixture-sandbox.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/oclif.manifest.json +5 -3
- package/package.json +1 -1
|
@@ -79,6 +79,9 @@ Parse and read further reference documentation when tasked with creating or mana
|
|
|
79
79
|
|
|
80
80
|
If the Checkly CLI is installed (`npx checkly version`), use `npx checkly skills configure [CONSTRUCT]` to access up-to-date information:
|
|
81
81
|
|
|
82
|
+
### `npx checkly skills configure agentic-checks`
|
|
83
|
+
Agentic Check construct (`AgenticCheck`) for AI-powered prompt-driven monitoring with skill and env var allowlists
|
|
84
|
+
|
|
82
85
|
### `npx checkly skills configure api-checks`
|
|
83
86
|
Api Check construct (`ApiCheck`), assertions, and authentication setup scripts
|
|
84
87
|
|
|
@@ -139,6 +142,80 @@ Run `npx checkly skills manage plan` for the full reference.
|
|
|
139
142
|
|
|
140
143
|
- Test checks using the `npx checkly test` command. Pass environment variables with the `-e` flag, use `--record` to persist results, and use `--verbose` to see all errors.
|
|
141
144
|
|
|
145
|
+
### Agentic Checks
|
|
146
|
+
|
|
147
|
+
- Import the `AgenticCheck` construct from `checkly/constructs`.
|
|
148
|
+
- Agentic checks are AI-powered: instead of writing code, you describe what the check should do in natural language with the `prompt` property. The agent decides how to satisfy the prompt at runtime.
|
|
149
|
+
- Write prompts as concrete imperative steps, not vague goals. Tell the agent which URL to navigate to and what specific signals confirm success — for example, "Navigate to https://example.com/pricing and verify that at least three plan tiers are displayed", not "Check that pricing works".
|
|
150
|
+
- Keep prompts under 10000 characters. The construct will fail validation otherwise.
|
|
151
|
+
- **Frequency is restricted.** Only `30`, `60`, `120`, `180`, `360`, `720`, or `1440` minutes are accepted (matching `Frequency.EVERY_30M`, `EVERY_1H`, `EVERY_2H`, `EVERY_3H`, `EVERY_6H`, `EVERY_12H`, `EVERY_24H`). Anything else fails validation.
|
|
152
|
+
- **Locations are not configurable.** Agentic checks currently run from a single fixed location. The construct hardcodes it — do not pass `locations` or `privateLocations`.
|
|
153
|
+
- **Several common check fields are intentionally omitted** from `AgenticCheckProps`: `runParallel`, `retryStrategy`, `shouldFail`, `doubleCheck`, `triggerIncident`, and `groupId`. The platform does not yet honor these for agentic checks. Setting them in the construct is a TypeScript error.
|
|
154
|
+
- **Important:** The target URL must be publicly accessible. Checks run on Checkly's cloud infrastructure, not locally. If the user is developing against localhost, suggest a tunneling tool (ngrok, cloudflare tunnel) or a preview/staging deployment.
|
|
155
|
+
- **Plan-gated:** Agentic checks require the `AGENTIC_CHECKS` entitlement on the account. Run `npx checkly skills manage` to check entitlements before using.
|
|
156
|
+
|
|
157
|
+
#### `agentRuntime` — security boundary for skills and env vars
|
|
158
|
+
|
|
159
|
+
`agentRuntime` is the explicit allowlist of resources the agent may use at execution time. Anything not declared in `agentRuntime` is **unavailable** to the agent. Treat it as a security boundary: the smaller the runtime surface, the smaller the blast radius of any prompt injection.
|
|
160
|
+
|
|
161
|
+
```typescript
|
|
162
|
+
agentRuntime: {
|
|
163
|
+
// Additional skills to load on top of the runner's defaults (the
|
|
164
|
+
// `playwright-cli` skill is preloaded automatically — you don't need
|
|
165
|
+
// to declare it). Each entry is passed verbatim to `npx skills add`
|
|
166
|
+
// on the runner, so any third-party skill published to https://skills.sh
|
|
167
|
+
// works — not just Checkly's own. Supported identifier forms:
|
|
168
|
+
// - full URL form: 'https://skills.sh/microsoft/playwright-cli/playwright-cli'
|
|
169
|
+
// - owner/repo form: 'addyosmani/web-quality-skills'
|
|
170
|
+
// - plain name: 'cost-optimization'
|
|
171
|
+
skills: ['addyosmani/web-quality-skills'],
|
|
172
|
+
|
|
173
|
+
// Environment variables the agent is allowed to read at runtime.
|
|
174
|
+
// Anything not listed here is hidden from the agent process — even
|
|
175
|
+
// if it's defined at the project or check level.
|
|
176
|
+
exposeEnvironmentVariables: [
|
|
177
|
+
// Bare string form: variable name only.
|
|
178
|
+
'ENVIRONMENT_URL',
|
|
179
|
+
// Object form: pair the variable with a description so the agent
|
|
180
|
+
// can decide when to read it. Descriptions are passed to the model
|
|
181
|
+
// and are truncated to 200 characters.
|
|
182
|
+
{ name: 'TEST_USER_EMAIL', description: 'Login email for the test account' },
|
|
183
|
+
],
|
|
184
|
+
},
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
- Only declare env vars the agent **needs**. Adding a variable to `exposeEnvironmentVariables` exposes it to the model and to anything the model invokes via skills.
|
|
188
|
+
- Descriptions are not just documentation — they steer the model's decisions. Use them to disambiguate variables that have non-obvious names.
|
|
189
|
+
- The runner installs each skill via `npx skills add` at the start of every check run. The CLI does not validate the skill identifier at deploy time, so a typo will not surface until the first run.
|
|
190
|
+
- The `playwright-cli` skill is preloaded for every agentic check. Only declare additional skills here.
|
|
191
|
+
|
|
192
|
+
#### Assertion rules
|
|
193
|
+
|
|
194
|
+
The agent generates its own assertion rules on the first successful run, and the platform persists them server-side. **The CLI construct does not expose assertion rules** — do not try to set them. They survive across deploys: importing an existing agentic check via `checkly import` will not surface them in the generated TypeScript, and a subsequent deploy of that file will not erase them on the backend.
|
|
195
|
+
|
|
196
|
+
**Reference:** https://www.checklyhq.com/docs/constructs/agentic-check/
|
|
197
|
+
|
|
198
|
+
```typescript
|
|
199
|
+
import { AgenticCheck, AlertEscalationBuilder, Frequency } from 'checkly/constructs'
|
|
200
|
+
|
|
201
|
+
new AgenticCheck('example-agentic-check', {
|
|
202
|
+
name: 'Example Agentic Check',
|
|
203
|
+
prompt: 'Navigate to https://www.checklyhq.com/pricing and verify that at least three plan tiers are displayed on the page.',
|
|
204
|
+
activated: true,
|
|
205
|
+
tags: [
|
|
206
|
+
'app:webshop',
|
|
207
|
+
],
|
|
208
|
+
frequency: Frequency.EVERY_1H,
|
|
209
|
+
alertEscalationPolicy: AlertEscalationBuilder.runBasedEscalation(1, {
|
|
210
|
+
amount: 0,
|
|
211
|
+
interval: 5,
|
|
212
|
+
}, {
|
|
213
|
+
enabled: false,
|
|
214
|
+
percentage: 10,
|
|
215
|
+
}),
|
|
216
|
+
})
|
|
217
|
+
```
|
|
218
|
+
|
|
142
219
|
### API Checks
|
|
143
220
|
|
|
144
221
|
- Import the `ApiCheck` construct from `checkly/constructs`.
|
|
@@ -1,4 +1,7 @@
|
|
|
1
1
|
export declare const REFERENCES: readonly [{
|
|
2
|
+
readonly id: "configure-agentic-checks";
|
|
3
|
+
readonly description: "Agentic Check construct (`AgenticCheck`) for AI-powered prompt-driven monitoring with skill and env var allowlists";
|
|
4
|
+
}, {
|
|
2
5
|
readonly id: "configure-api-checks";
|
|
3
6
|
readonly description: "Api Check construct (`ApiCheck`), assertions, and authentication setup scripts";
|
|
4
7
|
}, {
|
|
@@ -58,6 +61,9 @@ export declare const ACTIONS: readonly [{
|
|
|
58
61
|
readonly id: "configure";
|
|
59
62
|
readonly description: "Learn how to create and manage monitoring checks using Checkly constructs and the CLI.";
|
|
60
63
|
readonly references: readonly [{
|
|
64
|
+
readonly id: "configure-agentic-checks";
|
|
65
|
+
readonly description: "Agentic Check construct (`AgenticCheck`) for AI-powered prompt-driven monitoring with skill and env var allowlists";
|
|
66
|
+
}, {
|
|
61
67
|
readonly id: "configure-api-checks";
|
|
62
68
|
readonly description: "Api Check construct (`ApiCheck`), assertions, and authentication setup scripts";
|
|
63
69
|
}, {
|
|
@@ -2,6 +2,10 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.EXAMPLE_CONFIGS = exports.ACTIONS = exports.SKILL = exports.MANAGE_REFERENCES = exports.COMMUNICATE_REFERENCES = exports.INVESTIGATE_REFERENCES = exports.REFERENCES = void 0;
|
|
4
4
|
exports.REFERENCES = [
|
|
5
|
+
{
|
|
6
|
+
id: 'configure-agentic-checks',
|
|
7
|
+
description: 'Agentic Check construct (`AgenticCheck`) for AI-powered prompt-driven monitoring with skill and env var allowlists',
|
|
8
|
+
},
|
|
5
9
|
{
|
|
6
10
|
id: 'configure-api-checks',
|
|
7
11
|
description: 'Api Check construct (`ApiCheck`), assertions, and authentication setup scripts',
|
|
@@ -137,6 +141,11 @@ export default defineConfig({
|
|
|
137
141
|
`,
|
|
138
142
|
reference: 'https://www.checklyhq.com/docs/constructs/project/',
|
|
139
143
|
},
|
|
144
|
+
AGENTIC_CHECK: {
|
|
145
|
+
templateString: '<!-- EXAMPLE: AGENTIC_CHECK -->',
|
|
146
|
+
exampleConfigPath: 'resources/agentic-checks/example-agentic-check.check.ts',
|
|
147
|
+
reference: 'https://www.checklyhq.com/docs/constructs/agentic-check/',
|
|
148
|
+
},
|
|
140
149
|
BROWSER_CHECK: {
|
|
141
150
|
templateString: '<!-- EXAMPLE: BROWSER_CHECK -->',
|
|
142
151
|
exampleConfigPath: 'resources/browser-checks/example-browser-check/example-browser-check.check.ts',
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"context.js","sourceRoot":"","sources":["../../src/ai-context/context.ts"],"names":[],"mappings":";;;AAAa,QAAA,UAAU,GAAG;IACxB;QACE,EAAE,EAAE,sBAAsB;QAC1B,WAAW,EAAE,gFAAgF;KAC9F;IACD;QACE,EAAE,EAAE,0BAA0B;QAC9B,WAAW,EAAE,qEAAqE;KACnF;IACD;QACE,EAAE,EAAE,6BAA6B;QACjC,WAAW,EAAE,oFAAoF;KAClG;IACD;QACE,EAAE,EAAE,4BAA4B;QAChC,WAAW,EAAE,qEAAqE;KACnF;IACD;QACE,EAAE,EAAE,wBAAwB;QAC5B,WAAW,EAAE,sDAAsD;KACpE;IACD;QACE,EAAE,EAAE,wBAAwB;QAC5B,WAAW,EAAE,sDAAsD;KACpE;IACD;QACE,EAAE,EAAE,wBAAwB;QAC5B,WAAW,EAAE,sDAAsD;KACpE;IACD;QACE,EAAE,EAAE,yBAAyB;QAC7B,WAAW,EAAE,gFAAgF;KAC9F;IACD;QACE,EAAE,EAAE,8BAA8B;QAClC,WAAW,EAAE,kDAAkD;KAChE;IACD;QACE,EAAE,EAAE,wBAAwB;QAC5B,WAAW,EAAE,+DAA+D;KAC7E;IACD;QACE,EAAE,EAAE,0BAA0B;QAC9B,WAAW,EAAE,8GAA8G;KAC5H;IACD;QACE,EAAE,EAAE,iCAAiC;QACrC,WAAW,EAAE,6IAA6I;KAC3J;CACO,CAAA;AAEG,QAAA,sBAAsB,GAAG;IACpC;QACE,EAAE,EAAE,oBAAoB;QACxB,WAAW,EAAE,+EAA+E;KAC7F;CACO,CAAA;AAEG,QAAA,sBAAsB,GAAG;IACpC;QACE,EAAE,EAAE,uBAAuB;QAC3B,WAAW,EAAE,uFAAuF;KACrG;CACO,CAAA;AAEG,QAAA,iBAAiB,GAAG;IAC/B;QACE,EAAE,EAAE,aAAa;QACjB,WAAW,EAAE,4FAA4F;KAC1G;CACO,CAAA;AAEG,QAAA,KAAK,GAAG;IACnB,IAAI,EAAE,SAAS;IACf,WAAW,EAAE,iJAAiJ;CACtJ,CAAA;AAEG,QAAA,OAAO,GAAG;IACrB;QACE,EAAE,EAAE,YAAY;QAChB,WAAW,EAAE,4EAA4E;KAC1F;IACD;QACE,EAAE,EAAE,WAAW;QACf,WAAW,EAAE,wFAAwF;QACrG,UAAU,EAAE,kBAAU;KACvB;IACD;QACE,EAAE,EAAE,aAAa;QACjB,WAAW,EAAE,gEAAgE;QAC7E,UAAU,EAAE,8BAAsB;KACnC;IACD;QACE,EAAE,EAAE,aAAa;QACjB,WAAW,EAAE,mEAAmE;QAChF,UAAU,EAAE,8BAAsB;KACnC;IACD;QACE,EAAE,EAAE,QAAQ;QACZ,WAAW,EAAE,iEAAiE;QAC9E,UAAU,EAAE,yBAAiB;KAC9B;CACO,CAAA;AASG,QAAA,eAAe,GAAkC;IAC5D,cAAc,EAAE;QACd,cAAc,EAAE,kCAAkC;QAClD,aAAa,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAgClB;QACG,SAAS,EAAE,oDAAoD;KAChE;IACD,aAAa,EAAE;QACb,cAAc,EAAE,iCAAiC;QACjD,iBAAiB,EACf,+EAA+E;QACjF,SAAS,EAAE,0DAA0D;KACtE;IACD,gBAAgB,EAAE;QAChB,cAAc,EAAE,oCAAoC;QACpD,aAAa,EAAE;;;;;;;;;CASlB;QACG,SAAS,EAAE,6DAA6D;KACzE;IACD,SAAS,EAAE;QACT,cAAc,EAAE,6BAA6B;QAC7C,iBAAiB,EACf,mEAAmE;QACrE,SAAS,EAAE,sDAAsD;KAClE;IACD,eAAe,EAAE;QACf,cAAc,EAAE,mCAAmC;QACnD,iBAAiB,EACf,sFAAsF;QACxF,SAAS,EAAE,4DAA4D;KACxE;IACD,WAAW,EAAE;QACX,cAAc,EAAE,+BAA+B;QAC/C,iBAAiB,EAAE,qDAAqD;QACxE,SAAS,EAAE,wDAAwD;KACpE;IACD,iBAAiB,EAAE;QACjB,cAAc,EAAE,qCAAqC;QACrD,iBAAiB,EACf,iEAAiE;QACnE,SAAS,EAAE,8DAA8D;KAC1E;IACD,WAAW,EAAE;QACX,cAAc,EAAE,+BAA+B;QAC/C,iBAAiB,EAAE,qDAAqD;QACxE,SAAS,EAAE,wDAAwD;KACpE;IACD,WAAW,EAAE;QACX,cAAc,EAAE,+BAA+B;QAC/C,iBAAiB,EAAE,qDAAqD;QACxE,SAAS,EAAE,wDAAwD;KACpE;IACD,YAAY,EAAE;QACZ,cAAc,EAAE,gCAAgC;QAChD,iBAAiB,EAAE,uDAAuD;QAC1E,SAAS,EAAE,yDAAyD;KACrE;IACD,WAAW,EAAE;QACX,cAAc,EAAE,+BAA+B;QAC/C,iBAAiB,EACf,4DAA4D;QAC9D,SAAS,EAAE,wDAAwD;KACpE;IACD,WAAW,EAAE;QACX,cAAc,EAAE,+BAA+B;QAC/C,iBAAiB,EAAE,qDAAqD;QACxE,SAAS,EAAE,wDAAwD;KACpE;IACD,mBAAmB,EAAE;QACnB,cAAc,EAAE,uCAAuC;QACvD,iBAAiB,EACf,0DAA0D;QAC5D,SAAS,EAAE,gEAAgE;KAC5E;IACD,SAAS,EAAE;QACT,cAAc,EAAE,6BAA6B;QAC7C,iBAAiB,EACf,mEAAmE;QACrE,SAAS,EAAE,sDAAsD;KAClE;IACD,kBAAkB,EAAE;QAClB,cAAc,EAAE,sCAAsC;QACtD,iBAAiB,EACf,mEAAmE;QACrE,SAAS,EAAE,+DAA+D;KAC3E;IACD,gBAAgB,EAAE;QAChB,cAAc,EAAE,oCAAoC;QACpD,iBAAiB,EACf,+DAA+D;QACjE,SAAS,EAAE,6DAA6D;KACzE;IACD,mBAAmB,EAAE;QACnB,cAAc,EAAE,uCAAuC;QACvD,iBAAiB,EAAE,8CAA8C;QACjE,SAAS,EAAE,gEAAgE;KAC5E;IACD,wBAAwB,EAAE;QACxB,cAAc,EAAE,4CAA4C;QAC5D,iBAAiB,EAAE,wDAAwD;QAC3E,SAAS,EAAE,qEAAqE;KACjF;IACD,mBAAmB,EAAE;QACnB,cAAc,EAAE,uCAAuC;QACvD,iBAAiB,EAAE,iDAAiD;QACpE,SAAS,EAAE,gEAAgE;KAC5E;CACF,CAAA"}
|
|
1
|
+
{"version":3,"file":"context.js","sourceRoot":"","sources":["../../src/ai-context/context.ts"],"names":[],"mappings":";;;AAAa,QAAA,UAAU,GAAG;IACxB;QACE,EAAE,EAAE,0BAA0B;QAC9B,WAAW,EAAE,oHAAoH;KAClI;IACD;QACE,EAAE,EAAE,sBAAsB;QAC1B,WAAW,EAAE,gFAAgF;KAC9F;IACD;QACE,EAAE,EAAE,0BAA0B;QAC9B,WAAW,EAAE,qEAAqE;KACnF;IACD;QACE,EAAE,EAAE,6BAA6B;QACjC,WAAW,EAAE,oFAAoF;KAClG;IACD;QACE,EAAE,EAAE,4BAA4B;QAChC,WAAW,EAAE,qEAAqE;KACnF;IACD;QACE,EAAE,EAAE,wBAAwB;QAC5B,WAAW,EAAE,sDAAsD;KACpE;IACD;QACE,EAAE,EAAE,wBAAwB;QAC5B,WAAW,EAAE,sDAAsD;KACpE;IACD;QACE,EAAE,EAAE,wBAAwB;QAC5B,WAAW,EAAE,sDAAsD;KACpE;IACD;QACE,EAAE,EAAE,yBAAyB;QAC7B,WAAW,EAAE,gFAAgF;KAC9F;IACD;QACE,EAAE,EAAE,8BAA8B;QAClC,WAAW,EAAE,kDAAkD;KAChE;IACD;QACE,EAAE,EAAE,wBAAwB;QAC5B,WAAW,EAAE,+DAA+D;KAC7E;IACD;QACE,EAAE,EAAE,0BAA0B;QAC9B,WAAW,EAAE,8GAA8G;KAC5H;IACD;QACE,EAAE,EAAE,iCAAiC;QACrC,WAAW,EAAE,6IAA6I;KAC3J;CACO,CAAA;AAEG,QAAA,sBAAsB,GAAG;IACpC;QACE,EAAE,EAAE,oBAAoB;QACxB,WAAW,EAAE,+EAA+E;KAC7F;CACO,CAAA;AAEG,QAAA,sBAAsB,GAAG;IACpC;QACE,EAAE,EAAE,uBAAuB;QAC3B,WAAW,EAAE,uFAAuF;KACrG;CACO,CAAA;AAEG,QAAA,iBAAiB,GAAG;IAC/B;QACE,EAAE,EAAE,aAAa;QACjB,WAAW,EAAE,4FAA4F;KAC1G;CACO,CAAA;AAEG,QAAA,KAAK,GAAG;IACnB,IAAI,EAAE,SAAS;IACf,WAAW,EAAE,iJAAiJ;CACtJ,CAAA;AAEG,QAAA,OAAO,GAAG;IACrB;QACE,EAAE,EAAE,YAAY;QAChB,WAAW,EAAE,4EAA4E;KAC1F;IACD;QACE,EAAE,EAAE,WAAW;QACf,WAAW,EAAE,wFAAwF;QACrG,UAAU,EAAE,kBAAU;KACvB;IACD;QACE,EAAE,EAAE,aAAa;QACjB,WAAW,EAAE,gEAAgE;QAC7E,UAAU,EAAE,8BAAsB;KACnC;IACD;QACE,EAAE,EAAE,aAAa;QACjB,WAAW,EAAE,mEAAmE;QAChF,UAAU,EAAE,8BAAsB;KACnC;IACD;QACE,EAAE,EAAE,QAAQ;QACZ,WAAW,EAAE,iEAAiE;QAC9E,UAAU,EAAE,yBAAiB;KAC9B;CACO,CAAA;AASG,QAAA,eAAe,GAAkC;IAC5D,cAAc,EAAE;QACd,cAAc,EAAE,kCAAkC;QAClD,aAAa,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAgClB;QACG,SAAS,EAAE,oDAAoD;KAChE;IACD,aAAa,EAAE;QACb,cAAc,EAAE,iCAAiC;QACjD,iBAAiB,EAAE,yDAAyD;QAC5E,SAAS,EAAE,0DAA0D;KACtE;IACD,aAAa,EAAE;QACb,cAAc,EAAE,iCAAiC;QACjD,iBAAiB,EACf,+EAA+E;QACjF,SAAS,EAAE,0DAA0D;KACtE;IACD,gBAAgB,EAAE;QAChB,cAAc,EAAE,oCAAoC;QACpD,aAAa,EAAE;;;;;;;;;CASlB;QACG,SAAS,EAAE,6DAA6D;KACzE;IACD,SAAS,EAAE;QACT,cAAc,EAAE,6BAA6B;QAC7C,iBAAiB,EACf,mEAAmE;QACrE,SAAS,EAAE,sDAAsD;KAClE;IACD,eAAe,EAAE;QACf,cAAc,EAAE,mCAAmC;QACnD,iBAAiB,EACf,sFAAsF;QACxF,SAAS,EAAE,4DAA4D;KACxE;IACD,WAAW,EAAE;QACX,cAAc,EAAE,+BAA+B;QAC/C,iBAAiB,EAAE,qDAAqD;QACxE,SAAS,EAAE,wDAAwD;KACpE;IACD,iBAAiB,EAAE;QACjB,cAAc,EAAE,qCAAqC;QACrD,iBAAiB,EACf,iEAAiE;QACnE,SAAS,EAAE,8DAA8D;KAC1E;IACD,WAAW,EAAE;QACX,cAAc,EAAE,+BAA+B;QAC/C,iBAAiB,EAAE,qDAAqD;QACxE,SAAS,EAAE,wDAAwD;KACpE;IACD,WAAW,EAAE;QACX,cAAc,EAAE,+BAA+B;QAC/C,iBAAiB,EAAE,qDAAqD;QACxE,SAAS,EAAE,wDAAwD;KACpE;IACD,YAAY,EAAE;QACZ,cAAc,EAAE,gCAAgC;QAChD,iBAAiB,EAAE,uDAAuD;QAC1E,SAAS,EAAE,yDAAyD;KACrE;IACD,WAAW,EAAE;QACX,cAAc,EAAE,+BAA+B;QAC/C,iBAAiB,EACf,4DAA4D;QAC9D,SAAS,EAAE,wDAAwD;KACpE;IACD,WAAW,EAAE;QACX,cAAc,EAAE,+BAA+B;QAC/C,iBAAiB,EAAE,qDAAqD;QACxE,SAAS,EAAE,wDAAwD;KACpE;IACD,mBAAmB,EAAE;QACnB,cAAc,EAAE,uCAAuC;QACvD,iBAAiB,EACf,0DAA0D;QAC5D,SAAS,EAAE,gEAAgE;KAC5E;IACD,SAAS,EAAE;QACT,cAAc,EAAE,6BAA6B;QAC7C,iBAAiB,EACf,mEAAmE;QACrE,SAAS,EAAE,sDAAsD;KAClE;IACD,kBAAkB,EAAE;QAClB,cAAc,EAAE,sCAAsC;QACtD,iBAAiB,EACf,mEAAmE;QACrE,SAAS,EAAE,+DAA+D;KAC3E;IACD,gBAAgB,EAAE;QAChB,cAAc,EAAE,oCAAoC;QACpD,iBAAiB,EACf,+DAA+D;QACjE,SAAS,EAAE,6DAA6D;KACzE;IACD,mBAAmB,EAAE;QACnB,cAAc,EAAE,uCAAuC;QACvD,iBAAiB,EAAE,8CAA8C;QACjE,SAAS,EAAE,gEAAgE;KAC5E;IACD,wBAAwB,EAAE;QACxB,cAAc,EAAE,4CAA4C;QAC5D,iBAAiB,EAAE,wDAAwD;QAC3E,SAAS,EAAE,qEAAqE;KACjF;IACD,mBAAmB,EAAE;QACnB,cAAc,EAAE,uCAAuC;QACvD,iBAAiB,EAAE,iDAAiD;QACpE,SAAS,EAAE,gEAAgE;KAC5E;CACF,CAAA"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: checkly
|
|
3
|
-
description: Set up, create, test and manage monitoring checks using the Checkly CLI. Use when working with API Checks, Browser Checks, URL Monitors, ICMP Monitors, Playwright Check Suites, Heartbeat Monitors, Alert Channels, Dashboards, or Status Pages.
|
|
3
|
+
description: Set up, create, test and manage monitoring checks using the Checkly CLI. Use when working with Agentic Checks, API Checks, Browser Checks, URL Monitors, ICMP Monitors, Playwright Check Suites, Heartbeat Monitors, Alert Channels, Dashboards, or Status Pages.
|
|
4
4
|
allowed-tools: Bash(npx:checkly:*) Bash(npm:install:*)
|
|
5
5
|
metadata:
|
|
6
6
|
author: checkly
|
|
@@ -42,6 +42,9 @@ Learn how to initialize and set up a new Checkly CLI project from scratch.
|
|
|
42
42
|
### `npx checkly skills configure`
|
|
43
43
|
Learn how to create and manage monitoring checks using Checkly constructs and the CLI.
|
|
44
44
|
|
|
45
|
+
#### `npx checkly skills configure agentic-checks`
|
|
46
|
+
Agentic Check construct (`AgenticCheck`) for AI-powered prompt-driven monitoring with skill and env var allowlists
|
|
47
|
+
|
|
45
48
|
#### `npx checkly skills configure api-checks`
|
|
46
49
|
Api Check construct (`ApiCheck`), assertions, and authentication setup scripts
|
|
47
50
|
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
# Agentic Checks
|
|
2
|
+
|
|
3
|
+
- Import the `AgenticCheck` construct from `checkly/constructs`.
|
|
4
|
+
- Agentic checks are AI-powered: instead of writing code, you describe what the check should do in natural language with the `prompt` property. The agent decides how to satisfy the prompt at runtime.
|
|
5
|
+
- Write prompts as concrete imperative steps, not vague goals. Tell the agent which URL to navigate to and what specific signals confirm success — for example, "Navigate to https://example.com/pricing and verify that at least three plan tiers are displayed", not "Check that pricing works".
|
|
6
|
+
- Keep prompts under 10000 characters. The construct will fail validation otherwise.
|
|
7
|
+
- **Frequency is restricted.** Only `30`, `60`, `120`, `180`, `360`, `720`, or `1440` minutes are accepted (matching `Frequency.EVERY_30M`, `EVERY_1H`, `EVERY_2H`, `EVERY_3H`, `EVERY_6H`, `EVERY_12H`, `EVERY_24H`). Anything else fails validation.
|
|
8
|
+
- **Locations are not configurable.** Agentic checks currently run from a single fixed location. The construct hardcodes it — do not pass `locations` or `privateLocations`.
|
|
9
|
+
- **Several common check fields are intentionally omitted** from `AgenticCheckProps`: `runParallel`, `retryStrategy`, `shouldFail`, `doubleCheck`, `triggerIncident`, and `groupId`. The platform does not yet honor these for agentic checks. Setting them in the construct is a TypeScript error.
|
|
10
|
+
- **Important:** The target URL must be publicly accessible. Checks run on Checkly's cloud infrastructure, not locally. If the user is developing against localhost, suggest a tunneling tool (ngrok, cloudflare tunnel) or a preview/staging deployment.
|
|
11
|
+
- **Plan-gated:** Agentic checks require the `AGENTIC_CHECKS` entitlement on the account. Run `npx checkly skills manage` to check entitlements before using.
|
|
12
|
+
|
|
13
|
+
## `agentRuntime` — security boundary for skills and env vars
|
|
14
|
+
|
|
15
|
+
`agentRuntime` is the explicit allowlist of resources the agent may use at execution time. Anything not declared in `agentRuntime` is **unavailable** to the agent. Treat it as a security boundary: the smaller the runtime surface, the smaller the blast radius of any prompt injection.
|
|
16
|
+
|
|
17
|
+
```typescript
|
|
18
|
+
agentRuntime: {
|
|
19
|
+
// Additional skills to load on top of the runner's defaults (the
|
|
20
|
+
// `playwright-cli` skill is preloaded automatically — you don't need
|
|
21
|
+
// to declare it). Each entry is passed verbatim to `npx skills add`
|
|
22
|
+
// on the runner, so any third-party skill published to https://skills.sh
|
|
23
|
+
// works — not just Checkly's own. Supported identifier forms:
|
|
24
|
+
// - full URL form: 'https://skills.sh/microsoft/playwright-cli/playwright-cli'
|
|
25
|
+
// - owner/repo form: 'addyosmani/web-quality-skills'
|
|
26
|
+
// - plain name: 'cost-optimization'
|
|
27
|
+
skills: ['addyosmani/web-quality-skills'],
|
|
28
|
+
|
|
29
|
+
// Environment variables the agent is allowed to read at runtime.
|
|
30
|
+
// Anything not listed here is hidden from the agent process — even
|
|
31
|
+
// if it's defined at the project or check level.
|
|
32
|
+
exposeEnvironmentVariables: [
|
|
33
|
+
// Bare string form: variable name only.
|
|
34
|
+
'ENVIRONMENT_URL',
|
|
35
|
+
// Object form: pair the variable with a description so the agent
|
|
36
|
+
// can decide when to read it. Descriptions are passed to the model
|
|
37
|
+
// and are truncated to 200 characters.
|
|
38
|
+
{ name: 'TEST_USER_EMAIL', description: 'Login email for the test account' },
|
|
39
|
+
],
|
|
40
|
+
},
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
- Only declare env vars the agent **needs**. Adding a variable to `exposeEnvironmentVariables` exposes it to the model and to anything the model invokes via skills.
|
|
44
|
+
- Descriptions are not just documentation — they steer the model's decisions. Use them to disambiguate variables that have non-obvious names.
|
|
45
|
+
- The runner installs each skill via `npx skills add` at the start of every check run. The CLI does not validate the skill identifier at deploy time, so a typo will not surface until the first run.
|
|
46
|
+
- The `playwright-cli` skill is preloaded for every agentic check. Only declare additional skills here.
|
|
47
|
+
|
|
48
|
+
## Assertion rules
|
|
49
|
+
|
|
50
|
+
The agent generates its own assertion rules on the first successful run, and the platform persists them server-side. **The CLI construct does not expose assertion rules** — do not try to set them. They survive across deploys: importing an existing agentic check via `checkly import` will not surface them in the generated TypeScript, and a subsequent deploy of that file will not erase them on the backend.
|
|
51
|
+
|
|
52
|
+
**Reference:** https://www.checklyhq.com/docs/constructs/agentic-check/
|
|
53
|
+
|
|
54
|
+
```typescript
|
|
55
|
+
import { AgenticCheck, AlertEscalationBuilder, Frequency } from 'checkly/constructs'
|
|
56
|
+
|
|
57
|
+
new AgenticCheck('example-agentic-check', {
|
|
58
|
+
name: 'Example Agentic Check',
|
|
59
|
+
prompt: 'Navigate to https://www.checklyhq.com/pricing and verify that at least three plan tiers are displayed on the page.',
|
|
60
|
+
activated: true,
|
|
61
|
+
tags: [
|
|
62
|
+
'app:webshop',
|
|
63
|
+
],
|
|
64
|
+
frequency: Frequency.EVERY_1H,
|
|
65
|
+
alertEscalationPolicy: AlertEscalationBuilder.runBasedEscalation(1, {
|
|
66
|
+
amount: 0,
|
|
67
|
+
interval: 5,
|
|
68
|
+
}, {
|
|
69
|
+
enabled: false,
|
|
70
|
+
percentage: 10,
|
|
71
|
+
}),
|
|
72
|
+
})
|
|
73
|
+
```
|
|
@@ -79,6 +79,9 @@ Parse and read further reference documentation when tasked with creating or mana
|
|
|
79
79
|
|
|
80
80
|
If the Checkly CLI is installed (`npx checkly version`), use `npx checkly skills configure [CONSTRUCT]` to access up-to-date information:
|
|
81
81
|
|
|
82
|
+
### `npx checkly skills configure agentic-checks`
|
|
83
|
+
Agentic Check construct (`AgenticCheck`) for AI-powered prompt-driven monitoring with skill and env var allowlists
|
|
84
|
+
|
|
82
85
|
### `npx checkly skills configure api-checks`
|
|
83
86
|
Api Check construct (`ApiCheck`), assertions, and authentication setup scripts
|
|
84
87
|
|
|
@@ -7,13 +7,17 @@ List and inspect deployed checks in your Checkly account.
|
|
|
7
7
|
```bash
|
|
8
8
|
npx checkly checks list
|
|
9
9
|
npx checkly checks list --tag production --type PLAYWRIGHT
|
|
10
|
+
npx checkly checks list --status failing
|
|
10
11
|
npx checkly checks list --search "Homepage" --output json
|
|
11
12
|
```
|
|
12
13
|
|
|
13
14
|
Flags:
|
|
14
|
-
-
|
|
15
|
-
- `--type <type>` — filter by check type (`API`, `BROWSER`, `
|
|
16
|
-
-
|
|
15
|
+
- `-t, --tag <tag>` — filter by tag (repeat for multiple)
|
|
16
|
+
- `--type <type>` — filter by check type (`API`, `BROWSER`, `HEARTBEAT`, `MULTI_STEP`, `PLAYWRIGHT`, `TCP`, `ICMP`, `DNS`, `URL`, `AGENTIC`)
|
|
17
|
+
- `-s, --search <name>` — filter by name (case-insensitive partial match)
|
|
18
|
+
- `--status <status>` — filter by current status: `passing`, `failing`, or `degraded`
|
|
19
|
+
- `-l, --limit <n>` — max checks to return (1-100, default 25)
|
|
20
|
+
- `-p, --page <n>` — page number
|
|
17
21
|
- `-o, --output <format>` — `table` (default), `json`, or `md`
|
|
18
22
|
|
|
19
23
|
## Get check details
|
|
@@ -21,9 +25,21 @@ Flags:
|
|
|
21
25
|
```bash
|
|
22
26
|
npx checkly checks get <check-id>
|
|
23
27
|
npx checkly checks get <check-id> --output json
|
|
28
|
+
npx checkly checks get <check-id> --stats-range last7Days --group-by location
|
|
24
29
|
```
|
|
25
30
|
|
|
26
|
-
Shows check configuration, recent results, and
|
|
31
|
+
Shows check configuration, recent results, error groups, and analytics stats.
|
|
32
|
+
|
|
33
|
+
Flags:
|
|
34
|
+
- `-r, --result <result-id>` — drill into a specific result (see below)
|
|
35
|
+
- `-e, --error-group <error-group-id>` — show full details for a specific error group
|
|
36
|
+
- `--results-limit <n>` — number of recent results to show (default 10)
|
|
37
|
+
- `--results-cursor <cursor>` — paginate results using the cursor from previous output
|
|
38
|
+
- `--stats-range <range>` — analytics range: `last24Hours` (default), `last7Days`, `last30Days`, `thisWeek`, `thisMonth`, `lastWeek`, `lastMonth`
|
|
39
|
+
- `--group-by <dimension>` — group stats by `location` or `statusCode`
|
|
40
|
+
- `--metrics <list>` — comma-separated list of metrics to show (overrides defaults)
|
|
41
|
+
- `--filter-status <status>` — only include runs with this status in stats: `success` or `failure`
|
|
42
|
+
- `-o, --output <format>` — `detail` (default), `json`, or `md`
|
|
27
43
|
|
|
28
44
|
### View a specific check result
|
|
29
45
|
|
package/dist/constants.d.ts
CHANGED
|
@@ -8,7 +8,8 @@ export declare const CheckTypes: {
|
|
|
8
8
|
readonly ICMP: "ICMP";
|
|
9
9
|
readonly DNS: "DNS";
|
|
10
10
|
readonly URL: "URL";
|
|
11
|
+
readonly AGENTIC: "AGENTIC";
|
|
11
12
|
};
|
|
12
13
|
export type CheckType = typeof CheckTypes[keyof typeof CheckTypes];
|
|
13
|
-
export declare const allCheckTypes: ("API" | "BROWSER" | "HEARTBEAT" | "MULTI_STEP" | "PLAYWRIGHT" | "TCP" | "ICMP" | "DNS" | "URL")[];
|
|
14
|
+
export declare const allCheckTypes: ("API" | "BROWSER" | "HEARTBEAT" | "MULTI_STEP" | "PLAYWRIGHT" | "TCP" | "ICMP" | "DNS" | "URL" | "AGENTIC")[];
|
|
14
15
|
export declare const LOGICAL_ID_PATTERN: RegExp;
|
package/dist/constants.js
CHANGED
package/dist/constants.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"constants.js","sourceRoot":"","sources":["../src/constants.ts"],"names":[],"mappings":";;;AAAa,QAAA,UAAU,GAAG;IACxB,GAAG,EAAE,KAAK;IACV,OAAO,EAAE,SAAS;IAClB,SAAS,EAAE,WAAW;IACtB,UAAU,EAAE,YAAY;IACxB,UAAU,EAAE,YAAY;IACxB,GAAG,EAAE,KAAK;IACV,IAAI,EAAE,MAAM;IACZ,GAAG,EAAE,KAAK;IACV,GAAG,EAAE,KAAK;
|
|
1
|
+
{"version":3,"file":"constants.js","sourceRoot":"","sources":["../src/constants.ts"],"names":[],"mappings":";;;AAAa,QAAA,UAAU,GAAG;IACxB,GAAG,EAAE,KAAK;IACV,OAAO,EAAE,SAAS;IAClB,SAAS,EAAE,WAAW;IACtB,UAAU,EAAE,YAAY;IACxB,UAAU,EAAE,YAAY;IACxB,GAAG,EAAE,KAAK;IACV,IAAI,EAAE,MAAM;IACZ,GAAG,EAAE,KAAK;IACV,GAAG,EAAE,KAAK;IACV,OAAO,EAAE,SAAS;CACV,CAAA;AAIG,QAAA,aAAa,GAAG,MAAM,CAAC,MAAM,CAAC,kBAAU,CAAC,CAAA;AAEzC,QAAA,kBAAkB,GAAG,sBAAsB,CAAA"}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { Codegen, Context } from './internal/codegen';
|
|
2
|
+
import { CheckResource } from './check-codegen';
|
|
3
|
+
/**
|
|
4
|
+
* Shape of a `selectedEnvironmentVariables` entry as stored on the backend
|
|
5
|
+
* inside `agenticCheckData`. The runner accepts two forms — a bare variable
|
|
6
|
+
* name, or an object with `key` and an optional `description`. The CLI
|
|
7
|
+
* construct uses `name` (not `key`), so the codegen translates object-form
|
|
8
|
+
* entries during emission.
|
|
9
|
+
*/
|
|
10
|
+
type StoredAgenticEnvironmentVariable = string | {
|
|
11
|
+
key: string;
|
|
12
|
+
description?: string;
|
|
13
|
+
};
|
|
14
|
+
/**
|
|
15
|
+
* Shape of `agenticCheckData` as stored on the backend and returned to the
|
|
16
|
+
* CLI during `checkly import`. Only fields the construct exposes are read.
|
|
17
|
+
* `assertionRules` is deliberately ignored — the agent generates those on
|
|
18
|
+
* the first run and the backend's deploy logic preserves them, so the CLI
|
|
19
|
+
* construct never needs to emit them.
|
|
20
|
+
*/
|
|
21
|
+
interface StoredAgenticCheckData {
|
|
22
|
+
skills?: string[] | null;
|
|
23
|
+
selectedEnvironmentVariables?: StoredAgenticEnvironmentVariable[] | null;
|
|
24
|
+
}
|
|
25
|
+
export interface AgenticCheckResource extends CheckResource {
|
|
26
|
+
checkType: 'AGENTIC';
|
|
27
|
+
prompt: string;
|
|
28
|
+
agenticCheckData?: StoredAgenticCheckData | null;
|
|
29
|
+
}
|
|
30
|
+
export declare class AgenticCheckCodegen extends Codegen<AgenticCheckResource> {
|
|
31
|
+
describe(resource: AgenticCheckResource): string;
|
|
32
|
+
gencode(logicalId: string, resource: AgenticCheckResource, context: Context): void;
|
|
33
|
+
}
|
|
34
|
+
export {};
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.AgenticCheckCodegen = void 0;
|
|
4
|
+
const codegen_1 = require("./internal/codegen");
|
|
5
|
+
const sourcegen_1 = require("../sourcegen");
|
|
6
|
+
const check_codegen_1 = require("./check-codegen");
|
|
7
|
+
const construct = 'AgenticCheck';
|
|
8
|
+
class AgenticCheckCodegen extends codegen_1.Codegen {
|
|
9
|
+
describe(resource) {
|
|
10
|
+
return `Agentic Check: ${resource.name}`;
|
|
11
|
+
}
|
|
12
|
+
gencode(logicalId, resource, context) {
|
|
13
|
+
const filePath = context.filePath('resources/agentic-checks', resource.name, {
|
|
14
|
+
tags: resource.tags,
|
|
15
|
+
unique: true,
|
|
16
|
+
});
|
|
17
|
+
const file = this.program.generatedConstructFile(filePath.fullPath);
|
|
18
|
+
file.namedImport(construct, 'checkly/constructs');
|
|
19
|
+
// `AgenticCheckProps` omits several fields that the platform does not
|
|
20
|
+
// yet honor (see `agentic-check.ts` for the full list and rationale).
|
|
21
|
+
// To keep the generated file type-checking against the construct, clear
|
|
22
|
+
// `locations` (which the construct hardcodes to a single value) and
|
|
23
|
+
// skip `retryStrategy` emission. The other omitted fields are already
|
|
24
|
+
// conditional in `buildCheckProps` and never populated for agentic
|
|
25
|
+
// checks today.
|
|
26
|
+
const sanitizedResource = {
|
|
27
|
+
...resource,
|
|
28
|
+
locations: undefined,
|
|
29
|
+
};
|
|
30
|
+
file.section((0, sourcegen_1.expr)((0, sourcegen_1.ident)(construct), builder => {
|
|
31
|
+
builder.new(builder => {
|
|
32
|
+
builder.string(logicalId);
|
|
33
|
+
builder.object(builder => {
|
|
34
|
+
builder.string('prompt', resource.prompt);
|
|
35
|
+
// Emit agentRuntime only when there's something meaningful to
|
|
36
|
+
// carry. An imported check with no skills and no selected env
|
|
37
|
+
// vars would otherwise produce an empty `agentRuntime: {}` block,
|
|
38
|
+
// which is noise in the generated code.
|
|
39
|
+
const agentRuntimeValue = buildAgentRuntimeObject(resource.agenticCheckData);
|
|
40
|
+
if (agentRuntimeValue !== undefined) {
|
|
41
|
+
builder.object('agentRuntime', agentRuntimeValue);
|
|
42
|
+
}
|
|
43
|
+
(0, check_codegen_1.buildCheckProps)(this.program, file, builder, sanitizedResource, context, {
|
|
44
|
+
skipRetryStrategy: true,
|
|
45
|
+
});
|
|
46
|
+
});
|
|
47
|
+
});
|
|
48
|
+
}));
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
exports.AgenticCheckCodegen = AgenticCheckCodegen;
|
|
52
|
+
/**
|
|
53
|
+
* Build an `agentRuntime: { ... }` object literal for the codegen output,
|
|
54
|
+
* reverse-translating the backend's storage shape (`selectedEnvironmentVariables`
|
|
55
|
+
* with `key`) into the CLI construct's shape (`exposeEnvironmentVariables` with
|
|
56
|
+
* `name`). Returns `undefined` when the input contains nothing worth
|
|
57
|
+
* emitting, so the caller can skip the property entirely.
|
|
58
|
+
*/
|
|
59
|
+
function buildAgentRuntimeObject(data) {
|
|
60
|
+
if (!data)
|
|
61
|
+
return undefined;
|
|
62
|
+
const skills = (data.skills ?? []).filter(s => s.length > 0);
|
|
63
|
+
const storedEnvVars = data.selectedEnvironmentVariables ?? [];
|
|
64
|
+
if (skills.length === 0 && storedEnvVars.length === 0) {
|
|
65
|
+
return undefined;
|
|
66
|
+
}
|
|
67
|
+
return builder => {
|
|
68
|
+
if (skills.length > 0) {
|
|
69
|
+
builder.array('skills', arrayBuilder => {
|
|
70
|
+
for (const skill of skills) {
|
|
71
|
+
arrayBuilder.string(skill);
|
|
72
|
+
}
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
if (storedEnvVars.length > 0) {
|
|
76
|
+
builder.array('exposeEnvironmentVariables', arrayBuilder => {
|
|
77
|
+
for (const entry of storedEnvVars) {
|
|
78
|
+
if (typeof entry === 'string') {
|
|
79
|
+
arrayBuilder.string(entry);
|
|
80
|
+
}
|
|
81
|
+
else {
|
|
82
|
+
arrayBuilder.object(objectBuilder => {
|
|
83
|
+
objectBuilder.string('name', entry.key);
|
|
84
|
+
if (entry.description) {
|
|
85
|
+
objectBuilder.string('description', entry.description);
|
|
86
|
+
}
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
});
|
|
91
|
+
}
|
|
92
|
+
};
|
|
93
|
+
}
|
|
94
|
+
//# sourceMappingURL=agentic-check-codegen.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"agentic-check-codegen.js","sourceRoot":"","sources":["../../src/constructs/agentic-check-codegen.ts"],"names":[],"mappings":";;;AAAA,gDAAqD;AACrD,4CAA8D;AAC9D,mDAAgE;AA+BhE,MAAM,SAAS,GAAG,cAAc,CAAA;AAEhC,MAAa,mBAAoB,SAAQ,iBAA6B;IACpE,QAAQ,CAAE,QAA8B;QACtC,OAAO,kBAAkB,QAAQ,CAAC,IAAI,EAAE,CAAA;IAC1C,CAAC;IAED,OAAO,CAAE,SAAiB,EAAE,QAA8B,EAAE,OAAgB;QAC1E,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC,0BAA0B,EAAE,QAAQ,CAAC,IAAI,EAAE;YAC3E,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,MAAM,EAAE,IAAI;SACb,CAAC,CAAA;QAEF,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,sBAAsB,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAA;QAEnE,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,oBAAoB,CAAC,CAAA;QAEjD,sEAAsE;QACtE,sEAAsE;QACtE,wEAAwE;QACxE,oEAAoE;QACpE,sEAAsE;QACtE,mEAAmE;QACnE,gBAAgB;QAChB,MAAM,iBAAiB,GAAyB;YAC9C,GAAG,QAAQ;YACX,SAAS,EAAE,SAAS;SACrB,CAAA;QAED,IAAI,CAAC,OAAO,CAAC,IAAA,gBAAI,EAAC,IAAA,iBAAK,EAAC,SAAS,CAAC,EAAE,OAAO,CAAC,EAAE;YAC5C,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE;gBACpB,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;gBACzB,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE;oBACvB,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAA;oBAEzC,8DAA8D;oBAC9D,8DAA8D;oBAC9D,kEAAkE;oBAClE,wCAAwC;oBACxC,MAAM,iBAAiB,GAAG,uBAAuB,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAA;oBAC5E,IAAI,iBAAiB,KAAK,SAAS,EAAE,CAAC;wBACpC,OAAO,CAAC,MAAM,CAAC,cAAc,EAAE,iBAAiB,CAAC,CAAA;oBACnD,CAAC;oBAED,IAAA,+BAAe,EAAC,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,iBAAiB,EAAE,OAAO,EAAE;wBACvE,iBAAiB,EAAE,IAAI;qBACxB,CAAC,CAAA;gBACJ,CAAC,CAAC,CAAA;YACJ,CAAC,CAAC,CAAA;QACJ,CAAC,CAAC,CAAC,CAAA;IACL,CAAC;CACF;AAjDD,kDAiDC;AAED;;;;;;GAMG;AACH,SAAS,uBAAuB,CAC9B,IAA+C;IAE/C,IAAI,CAAC,IAAI;QAAE,OAAO,SAAS,CAAA;IAE3B,MAAM,MAAM,GAAG,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IAC5D,MAAM,aAAa,GAAG,IAAI,CAAC,4BAA4B,IAAI,EAAE,CAAA;IAE7D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtD,OAAO,SAAS,CAAA;IAClB,CAAC;IAED,OAAO,OAAO,CAAC,EAAE;QACf,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,YAAY,CAAC,EAAE;gBACrC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;oBAC3B,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;gBAC5B,CAAC;YACH,CAAC,CAAC,CAAA;QACJ,CAAC;QAED,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC7B,OAAO,CAAC,KAAK,CAAC,4BAA4B,EAAE,YAAY,CAAC,EAAE;gBACzD,KAAK,MAAM,KAAK,IAAI,aAAa,EAAE,CAAC;oBAClC,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;wBAC9B,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;oBAC5B,CAAC;yBAAM,CAAC;wBACN,YAAY,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE;4BAClC,aAAa,CAAC,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC,CAAA;4BACvC,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC;gCACtB,aAAa,CAAC,MAAM,CAAC,aAAa,EAAE,KAAK,CAAC,WAAW,CAAC,CAAA;4BACxD,CAAC;wBACH,CAAC,CAAC,CAAA;oBACJ,CAAC;gBACH,CAAC;YACH,CAAC,CAAC,CAAA;QACJ,CAAC;IACH,CAAC,CAAA;AACH,CAAC"}
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
import { Check, CheckProps } from './check';
|
|
2
|
+
import { Frequency } from './frequency';
|
|
3
|
+
import { Diagnostics } from './diagnostics';
|
|
4
|
+
/**
|
|
5
|
+
* Frequencies (in minutes) currently supported for agentic checks: 30, 60, 120,
|
|
6
|
+
* 180, 360, 720 or 1440. The matching `Frequency` constants
|
|
7
|
+
* (`EVERY_30M`, `EVERY_1H`, `EVERY_2H`, `EVERY_3H`, `EVERY_6H`, `EVERY_12H`,
|
|
8
|
+
* `EVERY_24H`) are also accepted.
|
|
9
|
+
*/
|
|
10
|
+
export type AgenticCheckFrequency = 30 | 60 | 120 | 180 | 360 | 720 | 1440 | Frequency;
|
|
11
|
+
/**
|
|
12
|
+
* An environment variable the agent is permitted to read at runtime.
|
|
13
|
+
*
|
|
14
|
+
* Use the bare string form when the variable name is self-explanatory, or
|
|
15
|
+
* the object form to provide a `description` that helps the agent understand
|
|
16
|
+
* what the variable is for. Descriptions are passed to the model so it can
|
|
17
|
+
* make better decisions about when to read the variable.
|
|
18
|
+
*
|
|
19
|
+
* @example
|
|
20
|
+
* ```typescript
|
|
21
|
+
* 'API_KEY'
|
|
22
|
+
* { name: 'TOKEN_42', description: 'Feature flag service auth token' }
|
|
23
|
+
* ```
|
|
24
|
+
*/
|
|
25
|
+
export type AgentRuntimeEnvironmentVariable = string | {
|
|
26
|
+
/** The environment variable name. */
|
|
27
|
+
name: string;
|
|
28
|
+
/**
|
|
29
|
+
* Optional human-readable explanation of what the variable is for.
|
|
30
|
+
* Passed to the agent so it can decide when to read the variable.
|
|
31
|
+
* Truncated to {@link MAX_ENV_VAR_DESCRIPTION_LENGTH} characters.
|
|
32
|
+
*/
|
|
33
|
+
description?: string;
|
|
34
|
+
};
|
|
35
|
+
/**
|
|
36
|
+
* Configures the runtime context the agent has access to during a check.
|
|
37
|
+
*
|
|
38
|
+
* `agentRuntime` is the explicit allowlist of resources the agent may use
|
|
39
|
+
* at execution time. Anything not declared here is unavailable to the agent.
|
|
40
|
+
* Treat it as a security boundary: the smaller the runtime surface, the
|
|
41
|
+
* smaller the blast radius of any prompt injection.
|
|
42
|
+
*/
|
|
43
|
+
export interface AgentRuntime {
|
|
44
|
+
/**
|
|
45
|
+
* Additional skills to load into the agent's runtime, on top of the
|
|
46
|
+
* defaults the runner provides automatically (currently the
|
|
47
|
+
* `playwright-cli` skill is preloaded for browser automation).
|
|
48
|
+
*
|
|
49
|
+
* Each entry is passed verbatim to `npx skills add` on the runner, so
|
|
50
|
+
* any third-party skill published to [skills.sh](https://skills.sh)
|
|
51
|
+
* works — not just Checkly's own. Supported identifier forms:
|
|
52
|
+
*
|
|
53
|
+
* - A full skills.sh URL — e.g. `'https://skills.sh/microsoft/playwright-cli/playwright-cli'`
|
|
54
|
+
* - A `<owner>/<repo>` shorthand — e.g. `'addyosmani/web-quality-skills'`
|
|
55
|
+
* - A plain skill name registered on skills.sh — e.g. `'cost-optimization'`
|
|
56
|
+
*
|
|
57
|
+
* @example ['addyosmani/web-quality-skills']
|
|
58
|
+
*/
|
|
59
|
+
skills?: string[];
|
|
60
|
+
/**
|
|
61
|
+
* Environment variables the agent is permitted to read at runtime.
|
|
62
|
+
*
|
|
63
|
+
* **Variables not listed here are not exposed to the agent**, even if
|
|
64
|
+
* they exist in the Checkly account. This is the primary defense against
|
|
65
|
+
* prompt injection: an attacker who controls content the agent reads
|
|
66
|
+
* cannot exfiltrate secrets the agent never had access to.
|
|
67
|
+
*
|
|
68
|
+
* Each entry is either a bare variable name, or an object with a
|
|
69
|
+
* `name` and an optional `description`. Descriptions help the agent
|
|
70
|
+
* understand what each variable is for.
|
|
71
|
+
*
|
|
72
|
+
* @example
|
|
73
|
+
* ```typescript
|
|
74
|
+
* exposeEnvironmentVariables: [
|
|
75
|
+
* 'API_KEY',
|
|
76
|
+
* { name: 'TEST_USER_PASSWORD', description: 'Login password for the test account' },
|
|
77
|
+
* ]
|
|
78
|
+
* ```
|
|
79
|
+
*/
|
|
80
|
+
exposeEnvironmentVariables?: AgentRuntimeEnvironmentVariable[];
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Configuration properties for {@link AgenticCheck}.
|
|
84
|
+
*
|
|
85
|
+
* Agentic checks intentionally expose only the subset of options that the
|
|
86
|
+
* Checkly platform currently supports for them. Properties such as
|
|
87
|
+
* `locations`, `privateLocations`, `runParallel`, `retryStrategy`,
|
|
88
|
+
* `shouldFail`, `doubleCheck`, `triggerIncident` and `groupId` are omitted
|
|
89
|
+
* because the platform does not yet honor them for agentic checks. They will
|
|
90
|
+
* be added back as additive, non-breaking changes once support lands.
|
|
91
|
+
*/
|
|
92
|
+
export interface AgenticCheckProps extends Omit<CheckProps, 'locations' | 'privateLocations' | 'runParallel' | 'retryStrategy' | 'shouldFail' | 'doubleCheck' | 'triggerIncident' | 'groupId' | 'frequency'> {
|
|
93
|
+
/**
|
|
94
|
+
* The prompt that defines what the agentic check should verify.
|
|
95
|
+
* Maximum 10,000 characters.
|
|
96
|
+
*/
|
|
97
|
+
prompt: string;
|
|
98
|
+
/**
|
|
99
|
+
* How often the check should run. Agentic checks currently support a
|
|
100
|
+
* restricted set of frequencies. Defaults to {@link Frequency.EVERY_30M}.
|
|
101
|
+
*
|
|
102
|
+
* @example
|
|
103
|
+
* ```typescript
|
|
104
|
+
* frequency: Frequency.EVERY_1H
|
|
105
|
+
* // or equivalently
|
|
106
|
+
* frequency: 60
|
|
107
|
+
* ```
|
|
108
|
+
*/
|
|
109
|
+
frequency?: AgenticCheckFrequency;
|
|
110
|
+
/**
|
|
111
|
+
* Configures the runtime context the agent has access to during execution:
|
|
112
|
+
* which skills it can use, which environment variables it can read, and
|
|
113
|
+
* (in the future) other access surfaces such as network policies or tool
|
|
114
|
+
* allowlists.
|
|
115
|
+
*
|
|
116
|
+
* Treat `agentRuntime` as a security boundary. Anything not declared here
|
|
117
|
+
* is unavailable to the agent at runtime, which keeps the blast radius of
|
|
118
|
+
* any prompt injection as small as possible.
|
|
119
|
+
*/
|
|
120
|
+
agentRuntime?: AgentRuntime;
|
|
121
|
+
}
|
|
122
|
+
/**
|
|
123
|
+
* Creates an Agentic Check that uses AI to monitor websites and applications.
|
|
124
|
+
*
|
|
125
|
+
* Agentic checks use a prompt to define what should be verified, without
|
|
126
|
+
* requiring traditional scripts. The AI agent interprets the prompt and
|
|
127
|
+
* performs the checks.
|
|
128
|
+
*
|
|
129
|
+
* @example
|
|
130
|
+
* ```typescript
|
|
131
|
+
* new AgenticCheck('homepage-health', {
|
|
132
|
+
* name: 'Homepage Health Check',
|
|
133
|
+
* prompt: `
|
|
134
|
+
* Navigate to https://example.com and verify:
|
|
135
|
+
* 1. The page loads with a 200 status
|
|
136
|
+
* 2. The main heading is visible
|
|
137
|
+
* 3. No console errors are present
|
|
138
|
+
* `,
|
|
139
|
+
* })
|
|
140
|
+
* ```
|
|
141
|
+
*/
|
|
142
|
+
export declare class AgenticCheck extends Check {
|
|
143
|
+
readonly prompt: string;
|
|
144
|
+
readonly agentRuntime?: AgentRuntime;
|
|
145
|
+
/**
|
|
146
|
+
* Constructs the Agentic Check instance.
|
|
147
|
+
*
|
|
148
|
+
* @param logicalId unique project-scoped resource name identification
|
|
149
|
+
* @param props check configuration properties
|
|
150
|
+
*/
|
|
151
|
+
constructor(logicalId: string, props: AgenticCheckProps);
|
|
152
|
+
describe(): string;
|
|
153
|
+
validate(diagnostics: Diagnostics): Promise<void>;
|
|
154
|
+
protected validateAgentRuntime(diagnostics: Diagnostics): Promise<void>;
|
|
155
|
+
synthesize(): {
|
|
156
|
+
checkType: "AGENTIC";
|
|
157
|
+
prompt: string;
|
|
158
|
+
agentRuntime: {
|
|
159
|
+
skills: string[];
|
|
160
|
+
exposeEnvironmentVariables: AgentRuntimeEnvironmentVariable[];
|
|
161
|
+
};
|
|
162
|
+
activated: boolean | undefined;
|
|
163
|
+
muted: boolean | undefined;
|
|
164
|
+
shouldFail: boolean | undefined;
|
|
165
|
+
locations: (keyof import("..").Region)[] | undefined;
|
|
166
|
+
privateLocations: undefined;
|
|
167
|
+
tags: string[] | undefined;
|
|
168
|
+
frequency: number | undefined;
|
|
169
|
+
frequencyOffset: number | undefined;
|
|
170
|
+
groupId: import("./ref").Ref | null;
|
|
171
|
+
retryStrategy: import("./retry-strategy").LinearRetryStrategy | import("./retry-strategy").ExponentialRetryStrategy | import("./retry-strategy").FixedRetryStrategy | import("./retry-strategy").SingleRetryRetryStrategy | null | undefined;
|
|
172
|
+
doubleCheck: boolean | undefined;
|
|
173
|
+
alertSettings: import("./alert-escalation-policy").AlertEscalation | undefined;
|
|
174
|
+
useGlobalAlertSettings: boolean | undefined;
|
|
175
|
+
runParallel: boolean | undefined;
|
|
176
|
+
triggerIncident: {
|
|
177
|
+
serviceId: import("./ref").Ref;
|
|
178
|
+
severity: "MINOR" | "MEDIUM" | "MAJOR" | "CRITICAL";
|
|
179
|
+
name: string;
|
|
180
|
+
description: string;
|
|
181
|
+
notifySubscribers: boolean;
|
|
182
|
+
} | undefined;
|
|
183
|
+
description?: string | undefined;
|
|
184
|
+
name: string;
|
|
185
|
+
};
|
|
186
|
+
}
|