@checkstack/healthcheck-backend 1.5.0 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +223 -0
- package/drizzle/0018_abnormal_preak.sql +10 -0
- package/drizzle/meta/0018_snapshot.json +600 -0
- package/drizzle/meta/_journal.json +7 -0
- package/package.json +26 -21
- package/src/ai/assertion-validation.test.ts +117 -0
- package/src/ai/assertion-validation.ts +147 -0
- package/src/ai/healthcheck-capabilities.test.ts +158 -0
- package/src/ai/healthcheck-capabilities.ts +217 -0
- package/src/ai/healthcheck-delete.test.ts +81 -0
- package/src/ai/healthcheck-delete.ts +81 -0
- package/src/ai/healthcheck-projection.test.ts +36 -0
- package/src/ai/healthcheck-propose.test.ts +268 -0
- package/src/ai/healthcheck-propose.ts +290 -0
- package/src/ai/healthcheck-script-tools.test.ts +93 -0
- package/src/ai/healthcheck-script-tools.ts +179 -0
- package/src/ai/healthcheck-update.test.ts +123 -0
- package/src/ai/healthcheck-update.ts +123 -0
- package/src/ai/notify-subscribers.test.ts +109 -0
- package/src/ai/notify-subscribers.ts +176 -0
- package/src/ai/register-ai-tools.test.ts +41 -0
- package/src/ai/register-ai-tools.ts +53 -0
- package/src/ai/shell-env-table.test.ts +47 -0
- package/src/automations.test.ts +2 -1
- package/src/automations.ts +9 -1
- package/src/collector-script-test.test.ts +53 -1
- package/src/collector-script-test.ts +59 -7
- package/src/effective-environments.test.ts +93 -0
- package/src/effective-environments.ts +64 -0
- package/src/health-entity-id.ts +57 -0
- package/src/health-entity.test.ts +384 -6
- package/src/health-entity.ts +93 -35
- package/src/health-state.ts +41 -4
- package/src/healthcheck-gitops-kinds.test.ts +95 -0
- package/src/healthcheck-gitops-kinds.ts +56 -13
- package/src/index.ts +30 -0
- package/src/migration-chain-contract.test.ts +57 -0
- package/src/queue-executor.test.ts +801 -0
- package/src/queue-executor.ts +336 -52
- package/src/realtime-aggregation.test.ts +30 -0
- package/src/realtime-aggregation.ts +16 -0
- package/src/retention-job.ts +167 -93
- package/src/retention-rollup.test.ts +118 -0
- package/src/router.test.ts +120 -1
- package/src/router.ts +20 -0
- package/src/schema.ts +44 -6
- package/src/service.ts +199 -43
- package/src/state-transitions.test.ts +104 -0
- package/src/state-transitions.ts +39 -1
- package/src/validate-configuration.test.ts +205 -0
- package/src/validate-configuration.ts +159 -0
- package/tsconfig.json +9 -0
|
@@ -0,0 +1,290 @@
|
|
|
1
|
+
import { stringify as toYaml } from "yaml";
|
|
2
|
+
import { qualifyAccessRuleId } from "@checkstack/common";
|
|
3
|
+
import type { RpcClient, AuthUser } from "@checkstack/backend-api";
|
|
4
|
+
import {
|
|
5
|
+
HealthCheckApi,
|
|
6
|
+
CreateHealthCheckConfigurationSchema,
|
|
7
|
+
healthCheckAccess,
|
|
8
|
+
pluginMetadata as healthcheckPluginMetadata,
|
|
9
|
+
type HealthCheckConfiguration,
|
|
10
|
+
type HealthCheckStrategyDto,
|
|
11
|
+
type CollectorDto,
|
|
12
|
+
} from "@checkstack/healthcheck-common";
|
|
13
|
+
import { z } from "zod";
|
|
14
|
+
import type { AiProposalPreview } from "@checkstack/ai-common";
|
|
15
|
+
import type { RegisteredAiTool } from "@checkstack/ai-backend";
|
|
16
|
+
import { validateCollectorAssertions } from "./assertion-validation";
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Input for the `healthcheck.propose` composite tool (plan OQ-6, Phase 5) - the
|
|
20
|
+
* mirror of the flagship `automation.propose`. The model authors a structured
|
|
21
|
+
* draft health-check configuration (the hard part of "NL -> health check");
|
|
22
|
+
* this tool VALIDATES that draft against the live strategy/collector registries
|
|
23
|
+
* via the published `validateConfiguration` RPC (the SAME deep migrate-then-
|
|
24
|
+
* validate-strict path the create / gitops-apply path uses) and returns it for
|
|
25
|
+
* a human to apply via the propose/apply gate. The shape reuses
|
|
26
|
+
* `CreateHealthCheckConfigurationSchema` so the model is constrained to a valid
|
|
27
|
+
* create skeleton (name, strategyId, config, intervalSeconds, collectors).
|
|
28
|
+
*/
|
|
29
|
+
export const HealthcheckProposeInputSchema =
|
|
30
|
+
CreateHealthCheckConfigurationSchema;
|
|
31
|
+
|
|
32
|
+
export type HealthcheckProposeInput = z.infer<
|
|
33
|
+
typeof HealthcheckProposeInputSchema
|
|
34
|
+
>;
|
|
35
|
+
|
|
36
|
+
/** Output returned once a human applies the proposal (the created config). */
|
|
37
|
+
export interface HealthcheckProposeApplyResult {
|
|
38
|
+
configuration: HealthCheckConfiguration;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
class HealthcheckProposeValidationError extends Error {
|
|
42
|
+
constructor(
|
|
43
|
+
message: string,
|
|
44
|
+
public readonly issues: Array<{ path: Array<string | number>; message: string }>,
|
|
45
|
+
) {
|
|
46
|
+
super(message);
|
|
47
|
+
this.name = "HealthcheckProposeValidationError";
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Flatten structured validation issues into a single, model-actionable string.
|
|
53
|
+
* The dry-run error surfaces to the model as plain text (a tool error), so the
|
|
54
|
+
* detail must live IN the message - not just the `issues` array - for the model
|
|
55
|
+
* to self-correct.
|
|
56
|
+
*/
|
|
57
|
+
function formatIssues(
|
|
58
|
+
issues: Array<{ path: Array<string | number>; message: string }>,
|
|
59
|
+
): string {
|
|
60
|
+
return issues
|
|
61
|
+
.map((issue) =>
|
|
62
|
+
issue.path.length > 0
|
|
63
|
+
? `${issue.path.join(".")}: ${issue.message}`
|
|
64
|
+
: issue.message,
|
|
65
|
+
)
|
|
66
|
+
.join("; ");
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* Appended to every health-check propose summary + the tool description: a newly
|
|
71
|
+
* created health check does NOT execute until it is assigned to a system, which
|
|
72
|
+
* the model must tell the operator (it cannot assign automatically yet).
|
|
73
|
+
*/
|
|
74
|
+
const SYSTEM_ASSIGNMENT_HINT =
|
|
75
|
+
"A new health check does not run until it is assigned to a system - after it is applied, tell the operator they must assign it to a system (Health Checks -> the check -> assign to a system) for it to start running.";
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Validate a drafted health-check configuration via the health-check plugin's
|
|
79
|
+
* `validateConfiguration` RPC - the SAME deep migrate-then-validate-strict path
|
|
80
|
+
* the create / gitops-apply path uses, so propose-time errors are identical to
|
|
81
|
+
* apply-time errors (a wrong config type or unknown key now surfaces at propose
|
|
82
|
+
* time, not just at apply). Throws a {@link HealthcheckProposeValidationError}
|
|
83
|
+
* carrying every structured issue when the draft is invalid; on success
|
|
84
|
+
* resolves the strategy + collector DTOs (via the published introspection RPCs)
|
|
85
|
+
* so the caller can render a precise confirm card with human-readable names.
|
|
86
|
+
*/
|
|
87
|
+
export async function validateHealthcheckDraft({
|
|
88
|
+
input,
|
|
89
|
+
rpcClient,
|
|
90
|
+
}: {
|
|
91
|
+
input: HealthcheckProposeInput;
|
|
92
|
+
rpcClient: RpcClient;
|
|
93
|
+
}): Promise<{ strategy: HealthCheckStrategyDto; collectors: CollectorDto[] }> {
|
|
94
|
+
const healthcheckClient = rpcClient.forPlugin(HealthCheckApi);
|
|
95
|
+
|
|
96
|
+
// Deep validation authority: identical to apply-time / gitops-apply.
|
|
97
|
+
const validation = await healthcheckClient.validateConfiguration({
|
|
98
|
+
name: input.name,
|
|
99
|
+
strategyId: input.strategyId,
|
|
100
|
+
config: input.config,
|
|
101
|
+
intervalSeconds: input.intervalSeconds,
|
|
102
|
+
collectors: input.collectors,
|
|
103
|
+
});
|
|
104
|
+
if (!validation.valid) {
|
|
105
|
+
throw new HealthcheckProposeValidationError(
|
|
106
|
+
`The drafted health check is invalid: ${formatIssues(validation.errors)}`,
|
|
107
|
+
validation.errors,
|
|
108
|
+
);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
// Valid: resolve human-readable DTOs for the confirm card. The strategy is
|
|
112
|
+
// guaranteed to exist (validation passed), so a missing DTO would be a
|
|
113
|
+
// registry/introspection mismatch - fall back to the raw id rather than
|
|
114
|
+
// failing the (already-valid) proposal.
|
|
115
|
+
const strategies = await healthcheckClient.getStrategies();
|
|
116
|
+
const strategy: HealthCheckStrategyDto = strategies.find(
|
|
117
|
+
(s) => s.id === input.strategyId,
|
|
118
|
+
) ?? {
|
|
119
|
+
id: input.strategyId,
|
|
120
|
+
displayName: input.strategyId,
|
|
121
|
+
category: "other",
|
|
122
|
+
configSchema: {},
|
|
123
|
+
};
|
|
124
|
+
|
|
125
|
+
const availableCollectors = input.collectors?.length
|
|
126
|
+
? await healthcheckClient.getCollectors({ strategyId: input.strategyId })
|
|
127
|
+
: [];
|
|
128
|
+
const resolvedCollectors: CollectorDto[] = [];
|
|
129
|
+
for (const entry of input.collectors ?? []) {
|
|
130
|
+
const collector = availableCollectors.find(
|
|
131
|
+
(c) => c.id === entry.collectorId,
|
|
132
|
+
);
|
|
133
|
+
if (collector) resolvedCollectors.push(collector);
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
// Assertion field/operator are free-form strings that `validateConfiguration`
|
|
137
|
+
// does not check, so an assertion with a bogus field/operator would save and
|
|
138
|
+
// then render as empty dropdowns in the editor. Validate them against each
|
|
139
|
+
// collector's RESULT schema + the canonical operator vocabulary so the model
|
|
140
|
+
// gets a precise, self-correcting error instead.
|
|
141
|
+
const resultSchemasById = new Map<string, Record<string, unknown>>();
|
|
142
|
+
for (const collector of availableCollectors) {
|
|
143
|
+
resultSchemasById.set(collector.id, collector.resultSchema);
|
|
144
|
+
}
|
|
145
|
+
const assertionIssues = validateCollectorAssertions({
|
|
146
|
+
collectors: input.collectors,
|
|
147
|
+
resultSchemasById,
|
|
148
|
+
});
|
|
149
|
+
if (assertionIssues.length > 0) {
|
|
150
|
+
throw new HealthcheckProposeValidationError(
|
|
151
|
+
`The drafted health check has invalid assertions: ${formatIssues(assertionIssues)}`,
|
|
152
|
+
assertionIssues,
|
|
153
|
+
);
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
return { strategy, collectors: resolvedCollectors };
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/**
|
|
160
|
+
* Pull the script source out of a collector config entry, if any. Inline-script
|
|
161
|
+
* (TS) and shell `script` collectors carry their source under `script` or
|
|
162
|
+
* `source`; we surface it on the confirm card so the human reviewing the
|
|
163
|
+
* proposal sees exactly what code would run. Returns undefined for non-script
|
|
164
|
+
* collectors.
|
|
165
|
+
*/
|
|
166
|
+
export function extractCollectorScriptSource(
|
|
167
|
+
config: Record<string, unknown>,
|
|
168
|
+
): string | undefined {
|
|
169
|
+
const candidate = config.script ?? config.source;
|
|
170
|
+
return typeof candidate === "string" ? candidate : undefined;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
/**
|
|
174
|
+
* `healthcheck.propose` - the mirror of `automation.propose` (plan OQ-6,
|
|
175
|
+
* Phase 5): natural language -> validated draft health check -> human applies.
|
|
176
|
+
* The AI NEVER silently creates a health check. `dryRun` validates the draft
|
|
177
|
+
* against the live registries WITHOUT mutating; the actual `createConfiguration`
|
|
178
|
+
* happens only at `apply`, behind the propose/apply token gate.
|
|
179
|
+
*
|
|
180
|
+
* `effect: "mutate"` - creating a health-check configuration is a
|
|
181
|
+
* non-destructive create, so it auto-applies in AUTO mode and is confirm-gated
|
|
182
|
+
* in APPROVE mode via the Phase 4 permission machinery, exactly like
|
|
183
|
+
* `automation.propose`. It is NOT `destructive`.
|
|
184
|
+
*
|
|
185
|
+
* Authorization: a SINGLE `requiredAccessRules` of `healthcheck.healthcheck.manage`
|
|
186
|
+
* (one rule, so the framework's AND-gate is correct - the same privilege the UI
|
|
187
|
+
* create form requires), and the propose/apply service re-checks `isAllowed` at
|
|
188
|
+
* BOTH propose and apply time. The underlying RPC calls use the USER-SCOPED
|
|
189
|
+
* client passed at call time, so handler-side authorization (access rules AND
|
|
190
|
+
* per-resource/team scoping) is enforced exactly as a direct UI/RPC call; the
|
|
191
|
+
* resolver gate + the propose/apply re-check are the additional authorization
|
|
192
|
+
* authority for this composite tool, identical to `automation.propose`.
|
|
193
|
+
*/
|
|
194
|
+
export function createHealthcheckProposeTool(): RegisteredAiTool<
|
|
195
|
+
HealthcheckProposeInput,
|
|
196
|
+
HealthcheckProposeApplyResult
|
|
197
|
+
> {
|
|
198
|
+
const dryRun = async ({
|
|
199
|
+
input,
|
|
200
|
+
rpcClient,
|
|
201
|
+
}: {
|
|
202
|
+
input: HealthcheckProposeInput;
|
|
203
|
+
principal: AuthUser;
|
|
204
|
+
rpcClient: RpcClient;
|
|
205
|
+
}): Promise<AiProposalPreview<HealthcheckProposeInput>> => {
|
|
206
|
+
// Validate the draft against the live strategy/collector registries WITHOUT
|
|
207
|
+
// creating anything (the same registries the UI pickers read).
|
|
208
|
+
const { strategy, collectors } = await validateHealthcheckDraft({
|
|
209
|
+
input,
|
|
210
|
+
rpcClient,
|
|
211
|
+
});
|
|
212
|
+
|
|
213
|
+
const scriptCollectors = (input.collectors ?? []).filter((entry) =>
|
|
214
|
+
extractCollectorScriptSource(entry.config),
|
|
215
|
+
);
|
|
216
|
+
|
|
217
|
+
// Render the full draft for human review: the configuration fields plus the
|
|
218
|
+
// resolved strategy/collector names and any script source.
|
|
219
|
+
const yaml = toYaml({
|
|
220
|
+
healthCheck: {
|
|
221
|
+
name: input.name,
|
|
222
|
+
strategy: strategy.displayName,
|
|
223
|
+
strategyId: input.strategyId,
|
|
224
|
+
intervalSeconds: input.intervalSeconds,
|
|
225
|
+
config: input.config,
|
|
226
|
+
...(input.collectors?.length
|
|
227
|
+
? {
|
|
228
|
+
collectors: input.collectors.map((entry) => {
|
|
229
|
+
const match = collectors.find(
|
|
230
|
+
(c) => c.id === entry.collectorId,
|
|
231
|
+
);
|
|
232
|
+
return {
|
|
233
|
+
collector: match?.displayName ?? entry.collectorId,
|
|
234
|
+
collectorId: entry.collectorId,
|
|
235
|
+
config: entry.config,
|
|
236
|
+
...(entry.assertions?.length
|
|
237
|
+
? { assertions: entry.assertions }
|
|
238
|
+
: {}),
|
|
239
|
+
};
|
|
240
|
+
}),
|
|
241
|
+
}
|
|
242
|
+
: {}),
|
|
243
|
+
},
|
|
244
|
+
});
|
|
245
|
+
|
|
246
|
+
const collectorCount = input.collectors?.length ?? 0;
|
|
247
|
+
const scriptNote =
|
|
248
|
+
scriptCollectors.length > 0
|
|
249
|
+
? ` (includes ${scriptCollectors.length} script collector${scriptCollectors.length === 1 ? "" : "s"})`
|
|
250
|
+
: "";
|
|
251
|
+
const summary = `Create health check "${input.name}" using strategy "${strategy.displayName}" with ${collectorCount} collector(s), running every ${input.intervalSeconds}s${scriptNote}. ${SYSTEM_ASSIGNMENT_HINT}`;
|
|
252
|
+
|
|
253
|
+
return {
|
|
254
|
+
summary,
|
|
255
|
+
// The validated, ready-to-apply payload captured at propose time. The
|
|
256
|
+
// chat confirm card / editor seeds from this; the YAML is for display.
|
|
257
|
+
payload: { ...input, yaml } as HealthcheckProposeInput & { yaml: string },
|
|
258
|
+
};
|
|
259
|
+
};
|
|
260
|
+
|
|
261
|
+
return {
|
|
262
|
+
name: "healthcheck.propose",
|
|
263
|
+
description:
|
|
264
|
+
"Validate a drafted health check (strategy, collectors, interval, and any inline script source) and return it for a human to review and apply. Never creates a health check directly - a person must approve the proposal. Use this to turn a natural-language health-check request (including a script health check) into a concrete, validated draft after testing the script with testScript. If you do not know what an endpoint returns, call probeUrl first to inspect its status code and body, then assert on the real response. Use getCapabilitySchema to get exact collector config fields AND the assertable result fields + valid operators before drafting assertions (assertion field must be a result-schema field like statusCode, operator must be a full word like equals/greaterThan, never an abbreviation). Note: a newly created health check does not run until the operator assigns it to a system.",
|
|
265
|
+
effect: "mutate",
|
|
266
|
+
input: HealthcheckProposeInputSchema,
|
|
267
|
+
requiredAccessRules: [
|
|
268
|
+
qualifyAccessRuleId(
|
|
269
|
+
healthcheckPluginMetadata,
|
|
270
|
+
healthCheckAccess.configuration.manage,
|
|
271
|
+
),
|
|
272
|
+
],
|
|
273
|
+
dryRun,
|
|
274
|
+
async execute({ input, rpcClient }) {
|
|
275
|
+
// Only reached via `apply` (the propose/apply token gate). The create
|
|
276
|
+
// handler runs its own zod + registry validation; this re-validates the
|
|
277
|
+
// server-stored payload against the input schema is already done by the
|
|
278
|
+
// propose/apply service before we get here.
|
|
279
|
+
const healthcheckClient = rpcClient.forPlugin(HealthCheckApi);
|
|
280
|
+
const configuration = await healthcheckClient.createConfiguration({
|
|
281
|
+
name: input.name,
|
|
282
|
+
strategyId: input.strategyId,
|
|
283
|
+
config: input.config,
|
|
284
|
+
intervalSeconds: input.intervalSeconds,
|
|
285
|
+
collectors: input.collectors,
|
|
286
|
+
});
|
|
287
|
+
return { configuration };
|
|
288
|
+
},
|
|
289
|
+
};
|
|
290
|
+
}
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import { describe, expect, test, mock } from "bun:test";
|
|
2
|
+
import type { AuthUser, RpcClient } from "@checkstack/backend-api";
|
|
3
|
+
import {
|
|
4
|
+
GetScriptContextOutputSchema,
|
|
5
|
+
TestScriptOutputSchema,
|
|
6
|
+
} from "@checkstack/ai-common";
|
|
7
|
+
import {
|
|
8
|
+
createHealthcheckGetScriptContextTool,
|
|
9
|
+
createHealthcheckTestScriptTool,
|
|
10
|
+
} from "./healthcheck-script-tools";
|
|
11
|
+
|
|
12
|
+
const principal: AuthUser = {
|
|
13
|
+
type: "user",
|
|
14
|
+
id: "u1",
|
|
15
|
+
accessRules: ["healthcheck.healthcheck.manage"],
|
|
16
|
+
};
|
|
17
|
+
|
|
18
|
+
/** A canned collector test result the stub RPC returns; the tool must map it through. */
|
|
19
|
+
const CANNED_RESULT = {
|
|
20
|
+
result: { statusCode: 200 },
|
|
21
|
+
stdout: "probe ok\n",
|
|
22
|
+
stderr: "",
|
|
23
|
+
exitCode: 0,
|
|
24
|
+
durationMs: 42,
|
|
25
|
+
timedOut: false,
|
|
26
|
+
error: undefined,
|
|
27
|
+
};
|
|
28
|
+
|
|
29
|
+
function fakeHealthcheckRpcClient(): RpcClient {
|
|
30
|
+
return {
|
|
31
|
+
forPlugin: () => ({
|
|
32
|
+
testCollectorScript: mock(() => Promise.resolve(CANNED_RESULT)),
|
|
33
|
+
}),
|
|
34
|
+
} as unknown as RpcClient;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
describe("healthcheck.getScriptContext tool", () => {
|
|
38
|
+
test("declares read effect + healthcheck manage gate, no dryRun", () => {
|
|
39
|
+
const tool = createHealthcheckGetScriptContextTool();
|
|
40
|
+
expect(tool.name).toBe("healthcheck.getScriptContext");
|
|
41
|
+
expect(tool.effect).toBe("read");
|
|
42
|
+
expect(tool.requiredAccessRules).toEqual([
|
|
43
|
+
"healthcheck.healthcheck.manage",
|
|
44
|
+
]);
|
|
45
|
+
expect(tool.dryRun).toBeUndefined();
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
test("resolves a healthcheck-script context from the real SDK bundle", async () => {
|
|
49
|
+
const tool = createHealthcheckGetScriptContextTool();
|
|
50
|
+
const out = await tool.execute({
|
|
51
|
+
input: { context: "healthcheck-script" },
|
|
52
|
+
principal,
|
|
53
|
+
rpcClient: fakeHealthcheckRpcClient(),
|
|
54
|
+
});
|
|
55
|
+
expect(GetScriptContextOutputSchema.safeParse(out).success).toBe(true);
|
|
56
|
+
expect(out.context).toBe("healthcheck-script");
|
|
57
|
+
expect(out.declarations.length).toBeGreaterThan(0);
|
|
58
|
+
});
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
describe("healthcheck.testScript tool", () => {
|
|
62
|
+
test("declares read effect + healthcheck manage gate, no dryRun", () => {
|
|
63
|
+
const tool = createHealthcheckTestScriptTool();
|
|
64
|
+
expect(tool.name).toBe("healthcheck.testScript");
|
|
65
|
+
expect(tool.effect).toBe("read");
|
|
66
|
+
expect(tool.requiredAccessRules).toEqual([
|
|
67
|
+
"healthcheck.healthcheck.manage",
|
|
68
|
+
]);
|
|
69
|
+
expect(tool.dryRun).toBeUndefined();
|
|
70
|
+
});
|
|
71
|
+
|
|
72
|
+
test("maps the RPC result fields through to the tool output", async () => {
|
|
73
|
+
const tool = createHealthcheckTestScriptTool();
|
|
74
|
+
const out = await tool.execute({
|
|
75
|
+
input: {
|
|
76
|
+
context: "healthcheck-script",
|
|
77
|
+
source: "export default async () => ({ statusCode: 200 });",
|
|
78
|
+
timeoutMs: 10_000,
|
|
79
|
+
},
|
|
80
|
+
principal,
|
|
81
|
+
rpcClient: fakeHealthcheckRpcClient(),
|
|
82
|
+
});
|
|
83
|
+
expect(TestScriptOutputSchema.safeParse(out).success).toBe(true);
|
|
84
|
+
expect(out.result).toEqual(CANNED_RESULT.result);
|
|
85
|
+
expect(out.stdout).toBe(CANNED_RESULT.stdout);
|
|
86
|
+
expect(out.stderr).toBe(CANNED_RESULT.stderr);
|
|
87
|
+
expect(out.exitCode).toBe(CANNED_RESULT.exitCode);
|
|
88
|
+
expect(out.durationMs).toBe(CANNED_RESULT.durationMs);
|
|
89
|
+
expect(out.timedOut).toBe(false);
|
|
90
|
+
// sandboxDowngraded is computed from the active policy and always surfaced.
|
|
91
|
+
expect(typeof out.sandboxDowngraded).toBe("boolean");
|
|
92
|
+
});
|
|
93
|
+
});
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
import { SDK_EDITOR_BUNDLE_DTS } from "@checkstack/sdk/editor-bundle";
|
|
2
|
+
import { qualifyAccessRuleId } from "@checkstack/common";
|
|
3
|
+
import { resolveActiveSandboxPolicy } from "@checkstack/backend-api";
|
|
4
|
+
import {
|
|
5
|
+
HealthCheckApi,
|
|
6
|
+
CollectorScriptTestInputSchema,
|
|
7
|
+
healthCheckAccess,
|
|
8
|
+
pluginMetadata as healthcheckPluginMetadata,
|
|
9
|
+
} from "@checkstack/healthcheck-common";
|
|
10
|
+
import {
|
|
11
|
+
GetScriptContextOutputSchema,
|
|
12
|
+
TestScriptInputSchema,
|
|
13
|
+
TestScriptOutputSchema,
|
|
14
|
+
type GetScriptContextOutput,
|
|
15
|
+
type TestScriptOutput,
|
|
16
|
+
} from "@checkstack/ai-common";
|
|
17
|
+
import { resolveScriptContext } from "@checkstack/ai-backend";
|
|
18
|
+
import type { RegisteredAiTool } from "@checkstack/ai-backend";
|
|
19
|
+
import { z } from "zod";
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* The healthcheck script-context rule that gates BOTH script tools. These are
|
|
23
|
+
* single-context (healthcheck-only) tools, so the resolver gate by the
|
|
24
|
+
* healthcheck configuration-manage rule is the authority - there is no
|
|
25
|
+
* cross-context surface, so no in-execute context assertion is needed (the old
|
|
26
|
+
* cross-context tools needed one because they fanned out to multiple plugins;
|
|
27
|
+
* these only ever handle healthcheck contexts).
|
|
28
|
+
*/
|
|
29
|
+
const HEALTHCHECK_MANAGE_RULE = qualifyAccessRuleId(
|
|
30
|
+
healthcheckPluginMetadata,
|
|
31
|
+
healthCheckAccess.configuration.manage,
|
|
32
|
+
);
|
|
33
|
+
|
|
34
|
+
/** The two healthcheck script contexts this plugin's tools handle. */
|
|
35
|
+
const HealthcheckScriptContextSchema = z.enum([
|
|
36
|
+
"healthcheck-script",
|
|
37
|
+
"healthcheck-shell",
|
|
38
|
+
]);
|
|
39
|
+
|
|
40
|
+
export const HealthcheckGetScriptContextInputSchema = z.object({
|
|
41
|
+
context: HealthcheckScriptContextSchema,
|
|
42
|
+
});
|
|
43
|
+
export type HealthcheckGetScriptContextInput = z.infer<
|
|
44
|
+
typeof HealthcheckGetScriptContextInputSchema
|
|
45
|
+
>;
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* `healthcheck.getScriptContext` - return the SDK symbols / imports / type
|
|
49
|
+
* signatures for a HEALTHCHECK script context by PURE extraction from the
|
|
50
|
+
* generated SDK editor bundle (the same DTS Monaco mounts). `effect: "read"` -
|
|
51
|
+
* it composes a static build-time resource and persists nothing, so it
|
|
52
|
+
* auto-runs in chat.
|
|
53
|
+
*
|
|
54
|
+
* This is a single-context (healthcheck-only) tool, so it is gated directly by
|
|
55
|
+
* the healthcheck configuration-manage rule at the resolver - no in-execute
|
|
56
|
+
* context assertion is needed.
|
|
57
|
+
*/
|
|
58
|
+
export function createHealthcheckGetScriptContextTool(): RegisteredAiTool<
|
|
59
|
+
HealthcheckGetScriptContextInput,
|
|
60
|
+
GetScriptContextOutput
|
|
61
|
+
> {
|
|
62
|
+
return {
|
|
63
|
+
name: "healthcheck.getScriptContext",
|
|
64
|
+
description:
|
|
65
|
+
"Return the SDK symbols, imports, and type signatures available to a health-check script in a given context (healthcheck-script, healthcheck-shell). Use this before drafting or testing a script so you import the correct module and helper and match the runtime context shape.",
|
|
66
|
+
effect: "read",
|
|
67
|
+
input: HealthcheckGetScriptContextInputSchema,
|
|
68
|
+
output: GetScriptContextOutputSchema,
|
|
69
|
+
requiredAccessRules: [HEALTHCHECK_MANAGE_RULE],
|
|
70
|
+
async execute({ input }) {
|
|
71
|
+
const resolved = resolveScriptContext({
|
|
72
|
+
context: input.context,
|
|
73
|
+
bundle: SDK_EDITOR_BUNDLE_DTS,
|
|
74
|
+
});
|
|
75
|
+
return {
|
|
76
|
+
context: resolved.context,
|
|
77
|
+
language: resolved.language,
|
|
78
|
+
sdkModule: resolved.sdkModule,
|
|
79
|
+
helper: resolved.helper,
|
|
80
|
+
declarations: resolved.declarations,
|
|
81
|
+
shellEnv: resolved.shellEnv ? [...resolved.shellEnv] : undefined,
|
|
82
|
+
starterExample: resolved.starterExample,
|
|
83
|
+
allowsManagedPackages: resolved.allowsManagedPackages,
|
|
84
|
+
};
|
|
85
|
+
},
|
|
86
|
+
};
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
export const HealthcheckTestScriptInputSchema = TestScriptInputSchema.extend({
|
|
90
|
+
context: HealthcheckScriptContextSchema,
|
|
91
|
+
});
|
|
92
|
+
export type HealthcheckTestScriptInput = z.infer<
|
|
93
|
+
typeof HealthcheckTestScriptInputSchema
|
|
94
|
+
>;
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* Resolve whether the active GLOBAL sandbox policy fell back to the fail-closed
|
|
98
|
+
* profile (no provider, or the provider threw). Surfaced as `sandboxDowngraded`
|
|
99
|
+
* so the model/operator NEVER gets a silent downgrade. Pure read of the same
|
|
100
|
+
* global policy the runners themselves resolve, so it is pod-consistent; any
|
|
101
|
+
* read failure conservatively reports a downgrade rather than masking one.
|
|
102
|
+
*/
|
|
103
|
+
async function resolveSandboxDowngraded(): Promise<boolean> {
|
|
104
|
+
try {
|
|
105
|
+
const { failedClosed } = await resolveActiveSandboxPolicy();
|
|
106
|
+
return failedClosed;
|
|
107
|
+
} catch {
|
|
108
|
+
return true;
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* `healthcheck.testScript` - run a DRAFT health-check script through the
|
|
114
|
+
* EXISTING fail-closed sandbox by calling `healthCheckContract.testCollectorScript`
|
|
115
|
+
* via the USER-SCOPED client passed at call time, so handler-side authorization
|
|
116
|
+
* is enforced exactly as a direct UI/RPC call. No model call is made, so the
|
|
117
|
+
* spend ledger is untouched.
|
|
118
|
+
*
|
|
119
|
+
* `effect: "read"` - it persists NOTHING about platform config (no health
|
|
120
|
+
* check, no row). It still counts toward the per-principal tool budget
|
|
121
|
+
* (enforced by the chat loop around every tool call).
|
|
122
|
+
*
|
|
123
|
+
* This is a single-context (healthcheck-only) tool, gated directly by the
|
|
124
|
+
* healthcheck configuration-manage rule at the resolver - no in-execute context
|
|
125
|
+
* assertion is needed.
|
|
126
|
+
*
|
|
127
|
+
* Safety inherited from the RPC test path:
|
|
128
|
+
* - The fail-closed global sandbox enforces no egress / scratch FS / privilege
|
|
129
|
+
* drop; `sandboxDowngraded` surfaces a fallback so it is never silent.
|
|
130
|
+
* - This tool passes NO `secretOverrides` and NO `secretEnv`, so only
|
|
131
|
+
* `__SECRET_<NAME>__` placeholders are ever present - the model never
|
|
132
|
+
* supplies secret values.
|
|
133
|
+
* - `timeoutMs` is capped at 30s in the input (stricter than the RPC's 300s).
|
|
134
|
+
*/
|
|
135
|
+
export function createHealthcheckTestScriptTool(): RegisteredAiTool<
|
|
136
|
+
HealthcheckTestScriptInput,
|
|
137
|
+
TestScriptOutput
|
|
138
|
+
> {
|
|
139
|
+
return {
|
|
140
|
+
name: "healthcheck.testScript",
|
|
141
|
+
description:
|
|
142
|
+
"Run a drafted health-check script in the secure fail-closed sandbox and return its result, stdout/stderr, and any error - WITHOUT creating any health check. Use this to validate a draft before proposing it. Never pass real secret values; the sandbox injects placeholders only.",
|
|
143
|
+
effect: "read",
|
|
144
|
+
input: HealthcheckTestScriptInputSchema,
|
|
145
|
+
output: TestScriptOutputSchema,
|
|
146
|
+
requiredAccessRules: [HEALTHCHECK_MANAGE_RULE],
|
|
147
|
+
async execute({ input, rpcClient }) {
|
|
148
|
+
const healthCheckClient = rpcClient.forPlugin(HealthCheckApi);
|
|
149
|
+
const kind =
|
|
150
|
+
input.context === "healthcheck-script" ? "typescript" : "shell";
|
|
151
|
+
const sandboxDowngraded = await resolveSandboxDowngraded();
|
|
152
|
+
|
|
153
|
+
// Map the tool input -> CollectorScriptTestInputSchema, parsing the loose
|
|
154
|
+
// `sampleContext` through the RPC's own schema (unknown keys are stripped,
|
|
155
|
+
// types narrowed). NEVER pass secretEnv / secretOverrides: the model never
|
|
156
|
+
// supplies secret values, so only placeholders can ever appear in the run.
|
|
157
|
+
const rpcInput = CollectorScriptTestInputSchema.parse({
|
|
158
|
+
kind,
|
|
159
|
+
script: input.source,
|
|
160
|
+
config: input.config,
|
|
161
|
+
env: input.env,
|
|
162
|
+
runContext: input.sampleContext,
|
|
163
|
+
timeoutMs: input.timeoutMs,
|
|
164
|
+
});
|
|
165
|
+
const raw = await healthCheckClient.testCollectorScript(rpcInput);
|
|
166
|
+
|
|
167
|
+
return {
|
|
168
|
+
result: raw.result,
|
|
169
|
+
stdout: raw.stdout,
|
|
170
|
+
stderr: raw.stderr,
|
|
171
|
+
exitCode: raw.exitCode,
|
|
172
|
+
durationMs: raw.durationMs,
|
|
173
|
+
timedOut: raw.timedOut,
|
|
174
|
+
error: raw.error,
|
|
175
|
+
sandboxDowngraded,
|
|
176
|
+
};
|
|
177
|
+
},
|
|
178
|
+
};
|
|
179
|
+
}
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
import { describe, expect, test, mock } from "bun:test";
|
|
2
|
+
import type { AuthUser, RpcClient } from "@checkstack/backend-api";
|
|
3
|
+
import { createHealthcheckUpdateTool } from "./healthcheck-update";
|
|
4
|
+
|
|
5
|
+
const principal: AuthUser = {
|
|
6
|
+
type: "user",
|
|
7
|
+
id: "u1",
|
|
8
|
+
accessRules: ["healthcheck.healthcheck.manage"],
|
|
9
|
+
};
|
|
10
|
+
|
|
11
|
+
const httpStrategy = {
|
|
12
|
+
id: "healthcheck-http.http",
|
|
13
|
+
displayName: "HTTP",
|
|
14
|
+
description: "HTTP probe",
|
|
15
|
+
category: "network",
|
|
16
|
+
configSchema: { type: "object", properties: { url: { type: "string" } } },
|
|
17
|
+
};
|
|
18
|
+
|
|
19
|
+
const existing = {
|
|
20
|
+
id: "hc1",
|
|
21
|
+
name: "google-com-http",
|
|
22
|
+
strategyId: "healthcheck-http.http",
|
|
23
|
+
config: { url: "https://google.com" },
|
|
24
|
+
intervalSeconds: 60,
|
|
25
|
+
collectors: [],
|
|
26
|
+
paused: false,
|
|
27
|
+
createdAt: new Date(),
|
|
28
|
+
updatedAt: new Date(),
|
|
29
|
+
};
|
|
30
|
+
|
|
31
|
+
function fakeRpcClient(overrides: Record<string, ReturnType<typeof mock>>): {
|
|
32
|
+
rpcClient: RpcClient;
|
|
33
|
+
fns: Record<string, ReturnType<typeof mock>>;
|
|
34
|
+
} {
|
|
35
|
+
const fns = {
|
|
36
|
+
getConfiguration: mock(() => Promise.resolve(existing)),
|
|
37
|
+
validateConfiguration: mock(() =>
|
|
38
|
+
Promise.resolve({ valid: true, errors: [] }),
|
|
39
|
+
),
|
|
40
|
+
getStrategies: mock(() => Promise.resolve([httpStrategy])),
|
|
41
|
+
getCollectors: mock(() => Promise.resolve([])),
|
|
42
|
+
updateConfiguration: mock(() =>
|
|
43
|
+
Promise.resolve({ ...existing, intervalSeconds: 30 }),
|
|
44
|
+
),
|
|
45
|
+
...overrides,
|
|
46
|
+
};
|
|
47
|
+
return {
|
|
48
|
+
rpcClient: { forPlugin: () => fns } as unknown as RpcClient,
|
|
49
|
+
fns,
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
describe("healthcheck.update tool", () => {
|
|
54
|
+
test("declares mutate effect + the manage rule", () => {
|
|
55
|
+
const tool = createHealthcheckUpdateTool();
|
|
56
|
+
expect(tool.name).toBe("healthcheck.update");
|
|
57
|
+
expect(tool.effect).toBe("mutate");
|
|
58
|
+
expect(tool.requiredAccessRules).toEqual(["healthcheck.healthcheck.manage"]);
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
test("dryRun merges the partial body and deep-validates, NEVER updating", async () => {
|
|
62
|
+
const { rpcClient, fns } = fakeRpcClient({});
|
|
63
|
+
const tool = createHealthcheckUpdateTool();
|
|
64
|
+
const preview = await tool.dryRun!({
|
|
65
|
+
input: { id: "hc1", body: { intervalSeconds: 30 } },
|
|
66
|
+
principal,
|
|
67
|
+
rpcClient,
|
|
68
|
+
});
|
|
69
|
+
expect(fns.validateConfiguration).toHaveBeenCalledTimes(1);
|
|
70
|
+
expect(fns.updateConfiguration).not.toHaveBeenCalled();
|
|
71
|
+
expect(preview.summary).toContain("google-com-http");
|
|
72
|
+
expect(preview.summary).toContain("30s");
|
|
73
|
+
expect(preview.payload).toEqual({ id: "hc1", body: { intervalSeconds: 30 } });
|
|
74
|
+
// The before -> after diff captures exactly what changes.
|
|
75
|
+
expect(preview.diff).toEqual([
|
|
76
|
+
{ path: "intervalSeconds", before: 60, after: 30 },
|
|
77
|
+
]);
|
|
78
|
+
});
|
|
79
|
+
|
|
80
|
+
test("dryRun throws when the id is unknown", async () => {
|
|
81
|
+
const { rpcClient } = fakeRpcClient({
|
|
82
|
+
getConfiguration: mock(() => Promise.resolve(undefined)),
|
|
83
|
+
});
|
|
84
|
+
const tool = createHealthcheckUpdateTool();
|
|
85
|
+
await expect(
|
|
86
|
+
tool.dryRun!({ input: { id: "nope", body: {} }, principal, rpcClient }),
|
|
87
|
+
).rejects.toThrow(/No health check found/);
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
test("dryRun surfaces a deep validation error from the merged config", async () => {
|
|
91
|
+
const { rpcClient } = fakeRpcClient({
|
|
92
|
+
validateConfiguration: mock(() =>
|
|
93
|
+
Promise.resolve({
|
|
94
|
+
valid: false,
|
|
95
|
+
errors: [{ path: ["config", "url"], message: "Expected string" }],
|
|
96
|
+
}),
|
|
97
|
+
),
|
|
98
|
+
});
|
|
99
|
+
const tool = createHealthcheckUpdateTool();
|
|
100
|
+
await expect(
|
|
101
|
+
tool.dryRun!({
|
|
102
|
+
input: { id: "hc1", body: { config: { url: 1 } } },
|
|
103
|
+
principal,
|
|
104
|
+
rpcClient,
|
|
105
|
+
}),
|
|
106
|
+
).rejects.toThrow(/invalid/i);
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
test("execute (apply) updates via updateConfiguration", async () => {
|
|
110
|
+
const { rpcClient, fns } = fakeRpcClient({});
|
|
111
|
+
const tool = createHealthcheckUpdateTool();
|
|
112
|
+
const result = await tool.execute({
|
|
113
|
+
input: { id: "hc1", body: { intervalSeconds: 30 } },
|
|
114
|
+
principal,
|
|
115
|
+
rpcClient,
|
|
116
|
+
});
|
|
117
|
+
expect(fns.updateConfiguration).toHaveBeenCalledWith({
|
|
118
|
+
id: "hc1",
|
|
119
|
+
body: { intervalSeconds: 30 },
|
|
120
|
+
});
|
|
121
|
+
expect(result.configuration.id).toBe("hc1");
|
|
122
|
+
});
|
|
123
|
+
});
|