@checkstack/ai-backend 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +97 -0
- package/drizzle/0000_productive_jackpot.sql +26 -0
- package/drizzle/0001_puzzling_purple_man.sql +26 -0
- package/drizzle/0002_sparkling_paper_doll.sql +15 -0
- package/drizzle/0003_married_senator_kelly.sql +1 -0
- package/drizzle/0004_crazy_miek.sql +2 -0
- package/drizzle/0005_tearful_randall_flagg.sql +1 -0
- package/drizzle/meta/0000_snapshot.json +232 -0
- package/drizzle/meta/0001_snapshot.json +434 -0
- package/drizzle/meta/0002_snapshot.json +551 -0
- package/drizzle/meta/0003_snapshot.json +557 -0
- package/drizzle/meta/0004_snapshot.json +573 -0
- package/drizzle/meta/0005_snapshot.json +574 -0
- package/drizzle/meta/_journal.json +48 -0
- package/drizzle.config.ts +7 -0
- package/package.json +42 -0
- package/src/agent-runner.test.ts +262 -0
- package/src/agent-runner.ts +262 -0
- package/src/chat/agent-loop.test.ts +119 -0
- package/src/chat/agent-loop.ts +73 -0
- package/src/chat/auto-apply.test.ts +237 -0
- package/src/chat/chat-handler.ts +111 -0
- package/src/chat/chat-service.streamturn.test.ts +417 -0
- package/src/chat/chat-service.test.ts +250 -0
- package/src/chat/chat-service.ts +923 -0
- package/src/chat/classifier-service.ts +64 -0
- package/src/chat/classifier.logic.test.ts +92 -0
- package/src/chat/classifier.logic.ts +71 -0
- package/src/chat/conversation-store.it.test.ts +203 -0
- package/src/chat/conversation-store.test.ts +248 -0
- package/src/chat/conversation-store.ts +237 -0
- package/src/chat/decision.logic.test.ts +45 -0
- package/src/chat/decision.logic.ts +54 -0
- package/src/chat/llm-provider.test.ts +63 -0
- package/src/chat/llm-provider.ts +67 -0
- package/src/chat/model-error.logic.test.ts +60 -0
- package/src/chat/model-error.logic.ts +65 -0
- package/src/chat/normalize-messages.logic.test.ts +101 -0
- package/src/chat/normalize-messages.logic.ts +65 -0
- package/src/chat/permission-mode.logic.test.ts +70 -0
- package/src/chat/permission-mode.logic.ts +45 -0
- package/src/chat/read-invoker.ts +72 -0
- package/src/chat/replay.test.ts +174 -0
- package/src/chat/scrub-content.test.ts +183 -0
- package/src/chat/scrub-content.ts +154 -0
- package/src/chat/sdk-tools.test.ts +168 -0
- package/src/chat/sdk-tools.ts +181 -0
- package/src/chat/title-service.test.ts +146 -0
- package/src/chat/title-service.ts +111 -0
- package/src/chat/title.logic.test.ts +98 -0
- package/src/chat/title.logic.ts +102 -0
- package/src/extension-points.ts +41 -0
- package/src/generated/docs-index.ts +3020 -0
- package/src/hardening/handler-authz.test.ts +282 -0
- package/src/hardening/no-secret-leak.test.ts +303 -0
- package/src/hooks.ts +33 -0
- package/src/index.ts +542 -0
- package/src/mcp/connection-registry.test.ts +25 -0
- package/src/mcp/connection-registry.ts +54 -0
- package/src/mcp/mcp-conformance.it.test.ts +128 -0
- package/src/mcp/server.test.ts +285 -0
- package/src/mcp/server.ts +300 -0
- package/src/mcp/tool-invoker.ts +65 -0
- package/src/openai-provider.test.ts +64 -0
- package/src/openai-provider.ts +146 -0
- package/src/projection.test.ts +97 -0
- package/src/projection.ts +132 -0
- package/src/propose-apply/args-hash.test.ts +26 -0
- package/src/propose-apply/args-hash.ts +30 -0
- package/src/propose-apply/service.test.ts +423 -0
- package/src/propose-apply/service.ts +419 -0
- package/src/propose-apply/store.test.ts +136 -0
- package/src/propose-apply/store.ts +224 -0
- package/src/propose-apply/token.test.ts +52 -0
- package/src/propose-apply/token.ts +71 -0
- package/src/rate-limit/spend-ledger.it.test.ts +224 -0
- package/src/rate-limit/spend-ledger.test.ts +176 -0
- package/src/rate-limit/spend-ledger.ts +162 -0
- package/src/rate-limit/tool-budget.it.test.ts +173 -0
- package/src/rate-limit/tool-budget.test.ts +58 -0
- package/src/rate-limit/tool-budget.ts +107 -0
- package/src/registry-wiring.test.ts +131 -0
- package/src/registry-wiring.ts +68 -0
- package/src/resolver.test.ts +156 -0
- package/src/resolver.ts +78 -0
- package/src/router.test.ts +78 -0
- package/src/router.ts +345 -0
- package/src/schema.ts +284 -0
- package/src/serializer.test.ts +88 -0
- package/src/serializer.ts +42 -0
- package/src/tool-registry.ts +58 -0
- package/src/tools/composite-tools.ts +24 -0
- package/src/tools/docs-tools.test.ts +150 -0
- package/src/tools/docs-tools.ts +115 -0
- package/src/tools/probe-url.test.ts +51 -0
- package/src/tools/probe-url.ts +146 -0
- package/src/tools/rank-docs.test.ts +153 -0
- package/src/tools/rank-docs.ts +209 -0
- package/src/tools/script-context-extract.test.ts +93 -0
- package/src/tools/script-context-extract.ts +283 -0
- package/src/tools/ssrf-guard.test.ts +69 -0
- package/src/tools/ssrf-guard.ts +108 -0
- package/src/tools/tool-set.e2e.test.ts +64 -0
- package/src/user-rpc-client.test.ts +45 -0
- package/src/user-rpc-client.ts +60 -0
- package/tsconfig.json +26 -0
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
import { describe, expect, test } from "bun:test";
|
|
2
|
+
import { z } from "zod";
|
|
3
|
+
import { toJsonSchema, configString } from "@checkstack/backend-api";
|
|
4
|
+
import type { RegisteredAiTool } from "./tool-registry";
|
|
5
|
+
import { serializeTool, serializeTools } from "./serializer";
|
|
6
|
+
|
|
7
|
+
describe("serializeTool", () => {
|
|
8
|
+
const inputSchema = z.object({ status: z.string().optional() });
|
|
9
|
+
const tool: RegisteredAiTool = {
|
|
10
|
+
name: "incident.list",
|
|
11
|
+
description: "List incidents.",
|
|
12
|
+
effect: "read",
|
|
13
|
+
input: inputSchema,
|
|
14
|
+
requiredAccessRules: ["incident.incident.read"],
|
|
15
|
+
execute: () => Promise.resolve({}),
|
|
16
|
+
};
|
|
17
|
+
|
|
18
|
+
test("produces the SAME JSON Schema as toJsonSchema (no second serializer drift)", () => {
|
|
19
|
+
const descriptor = serializeTool({ tool });
|
|
20
|
+
// The serializer must wrap the platform's toJsonSchema verbatim, so a tool
|
|
21
|
+
// input schema and the same schema run through the OpenAPI substrate agree.
|
|
22
|
+
expect(descriptor.inputSchema).toEqual(toJsonSchema(inputSchema));
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
test("carries name, description, effect, and requiredAccessRules", () => {
|
|
26
|
+
const descriptor = serializeTool({ tool });
|
|
27
|
+
expect(descriptor.name).toBe("incident.list");
|
|
28
|
+
expect(descriptor.description).toBe("List incidents.");
|
|
29
|
+
expect(descriptor.effect).toBe("read");
|
|
30
|
+
expect(descriptor.requiredAccessRules).toEqual(["incident.incident.read"]);
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
test("includes outputSchema only when the tool declares an output", () => {
|
|
34
|
+
expect(serializeTool({ tool }).outputSchema).toBeUndefined();
|
|
35
|
+
const withOutput: RegisteredAiTool = {
|
|
36
|
+
...tool,
|
|
37
|
+
output: z.object({ count: z.number() }),
|
|
38
|
+
};
|
|
39
|
+
expect(serializeTool({ tool: withOutput }).outputSchema).toEqual(
|
|
40
|
+
toJsonSchema(z.object({ count: z.number() })),
|
|
41
|
+
);
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
test("never emits a secret VALUE into the descriptor", () => {
|
|
45
|
+
// A tool whose input has an x-secret field: the schema describes the field
|
|
46
|
+
// but the descriptor must never contain a concrete secret value.
|
|
47
|
+
const secretTool: RegisteredAiTool = {
|
|
48
|
+
name: "thing.configure",
|
|
49
|
+
description: "Configure a thing.",
|
|
50
|
+
effect: "read",
|
|
51
|
+
input: z.object({ apiKey: configString({ "x-secret": true }) }),
|
|
52
|
+
requiredAccessRules: ["ai.tools.manage"],
|
|
53
|
+
execute: () => Promise.resolve({}),
|
|
54
|
+
};
|
|
55
|
+
const serialized = JSON.stringify(serializeTool({ tool: secretTool }));
|
|
56
|
+
// The descriptor is pure schema metadata — it carries no runtime values.
|
|
57
|
+
expect(serialized).not.toContain("super-secret");
|
|
58
|
+
// The x-secret marker IS preserved (so transports can mask), but no value.
|
|
59
|
+
expect(serialized).toContain("x-secret");
|
|
60
|
+
});
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
describe("serializeTools", () => {
|
|
64
|
+
test("serializes each tool", () => {
|
|
65
|
+
const tools: RegisteredAiTool[] = [
|
|
66
|
+
{
|
|
67
|
+
name: "a.one",
|
|
68
|
+
description: "a",
|
|
69
|
+
effect: "read",
|
|
70
|
+
input: z.object({}),
|
|
71
|
+
requiredAccessRules: [],
|
|
72
|
+
execute: () => Promise.resolve({}),
|
|
73
|
+
},
|
|
74
|
+
{
|
|
75
|
+
name: "b.two",
|
|
76
|
+
description: "b",
|
|
77
|
+
effect: "mutate",
|
|
78
|
+
input: z.object({}),
|
|
79
|
+
requiredAccessRules: ["b.x.manage"],
|
|
80
|
+
execute: () => Promise.resolve({}),
|
|
81
|
+
},
|
|
82
|
+
];
|
|
83
|
+
expect(serializeTools({ tools }).map((d) => d.name)).toEqual([
|
|
84
|
+
"a.one",
|
|
85
|
+
"b.two",
|
|
86
|
+
]);
|
|
87
|
+
});
|
|
88
|
+
});
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import { toJsonSchema } from "@checkstack/backend-api";
|
|
2
|
+
import type { AiTool, AiToolDescriptor } from "@checkstack/ai-common";
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Serialize a registered {@link AiTool} into its transport-facing descriptor.
|
|
6
|
+
*
|
|
7
|
+
* This is the ONE serializer shared by both transports (OpenAI function
|
|
8
|
+
* schemas and MCP tool defs). It wraps the platform's `toJsonSchema()`
|
|
9
|
+
* (which itself wraps zod v4's native `toJSONSchema()` and stamps the `x-*`
|
|
10
|
+
* registry metadata), so the schema a tool exposes is byte-for-byte the same
|
|
11
|
+
* substrate the OpenAPI generator emits for the source procedure — there is no
|
|
12
|
+
* second serializer to drift.
|
|
13
|
+
*
|
|
14
|
+
* The descriptor deliberately carries NO executor closure and NO zod object,
|
|
15
|
+
* only JSON Schema + metadata, so it is safe to return over an RPC or MCP wire.
|
|
16
|
+
*/
|
|
17
|
+
export function serializeTool({
|
|
18
|
+
tool,
|
|
19
|
+
}: {
|
|
20
|
+
tool: AiTool;
|
|
21
|
+
}): AiToolDescriptor {
|
|
22
|
+
return {
|
|
23
|
+
name: tool.name,
|
|
24
|
+
description: tool.description,
|
|
25
|
+
effect: tool.effect,
|
|
26
|
+
inputSchema: toJsonSchema(tool.input),
|
|
27
|
+
outputSchema: tool.output ? toJsonSchema(tool.output) : undefined,
|
|
28
|
+
requiredAccessRules: [...tool.requiredAccessRules],
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Serialize many tools at once (convenience for the introspection RPC and the
|
|
34
|
+
* MCP `list_tools` response).
|
|
35
|
+
*/
|
|
36
|
+
export function serializeTools({
|
|
37
|
+
tools,
|
|
38
|
+
}: {
|
|
39
|
+
tools: AiTool[];
|
|
40
|
+
}): AiToolDescriptor[] {
|
|
41
|
+
return tools.map((tool) => serializeTool({ tool }));
|
|
42
|
+
}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import type { AuthUser, RpcClient } from "@checkstack/backend-api";
|
|
2
|
+
import type { AiTool } from "@checkstack/ai-common";
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* A tool whose executors run with a Checkstack {@link AuthUser} principal and
|
|
6
|
+
* receive a USER-SCOPED {@link RpcClient} (bound to the originating user) for any
|
|
7
|
+
* plugin call - so handler-side authz + per-resource/team scoping always apply.
|
|
8
|
+
*/
|
|
9
|
+
export type RegisteredAiTool<TInput = unknown, TOutput = unknown> = AiTool<
|
|
10
|
+
TInput,
|
|
11
|
+
TOutput,
|
|
12
|
+
AuthUser,
|
|
13
|
+
RpcClient
|
|
14
|
+
>;
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Registry for AI tools — the spine of the AI platform. Plugins contribute
|
|
18
|
+
* tools through `aiToolExtensionPoint` (hand-authored composite tools) and
|
|
19
|
+
* `aiToolProjectionExtensionPoint` (opt-in projections of existing oRPC
|
|
20
|
+
* procedures). Both transports (chat, MCP) resolve tools from this one
|
|
21
|
+
* registry, so no capability is implemented twice.
|
|
22
|
+
*
|
|
23
|
+
* Tool names are already fully qualified (`<plugin>.<tool>`) by the extension
|
|
24
|
+
* points before they reach `register`.
|
|
25
|
+
*/
|
|
26
|
+
export interface AiToolRegistry {
|
|
27
|
+
register(tool: RegisteredAiTool): void;
|
|
28
|
+
getTools(): RegisteredAiTool[];
|
|
29
|
+
getTool(name: string): RegisteredAiTool | undefined;
|
|
30
|
+
hasTool(name: string): boolean;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export function createAiToolRegistry(): AiToolRegistry {
|
|
34
|
+
const tools = new Map<string, RegisteredAiTool>();
|
|
35
|
+
|
|
36
|
+
return {
|
|
37
|
+
register(tool: RegisteredAiTool): void {
|
|
38
|
+
if (tools.has(tool.name)) {
|
|
39
|
+
throw new Error(
|
|
40
|
+
`AI tool ${tool.name} already registered — likely a duplicate registration.`,
|
|
41
|
+
);
|
|
42
|
+
}
|
|
43
|
+
tools.set(tool.name, tool);
|
|
44
|
+
},
|
|
45
|
+
|
|
46
|
+
getTools(): RegisteredAiTool[] {
|
|
47
|
+
return [...tools.values()];
|
|
48
|
+
},
|
|
49
|
+
|
|
50
|
+
getTool(name: string): RegisteredAiTool | undefined {
|
|
51
|
+
return tools.get(name);
|
|
52
|
+
},
|
|
53
|
+
|
|
54
|
+
hasTool(name: string): boolean {
|
|
55
|
+
return tools.has(name);
|
|
56
|
+
},
|
|
57
|
+
};
|
|
58
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import type { RegisteredAiTool } from "../tool-registry";
|
|
2
|
+
import { createDocsTools } from "./docs-tools";
|
|
3
|
+
import { createProbeUrlTool } from "./probe-url";
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* ai-backend's OWN platform AI tools, registered through `aiToolExtensionPoint`.
|
|
7
|
+
* These are genuinely cross-plugin / platform tools with no single owning plugin:
|
|
8
|
+
* documentation grounding (bundled docs) and URL probing.
|
|
9
|
+
*
|
|
10
|
+
* Plugin-specific tools (health-check / automation propose-update-delete,
|
|
11
|
+
* capability catalogs, and script-context tools) are owned by and registered
|
|
12
|
+
* FROM their plugins via the same `aiToolExtensionPoint` - ai-backend does not
|
|
13
|
+
* depend on any plugin's `*-common` to build them. The systemic-authz test in
|
|
14
|
+
* `tool-set.e2e.test.ts` covers this set plus the projected read tools.
|
|
15
|
+
*/
|
|
16
|
+
export function buildCompositeTools(): RegisteredAiTool[] {
|
|
17
|
+
return [
|
|
18
|
+
// Docs grounding (effect: "read"; composes the bundled docs index).
|
|
19
|
+
...createDocsTools(),
|
|
20
|
+
// URL introspection (effect: "read"): probe a public URL to see what it
|
|
21
|
+
// returns before drafting check assertions. SSRF-guarded (no internal hosts).
|
|
22
|
+
createProbeUrlTool(),
|
|
23
|
+
];
|
|
24
|
+
}
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
import { describe, expect, test } from "bun:test";
|
|
2
|
+
import type { AuthUser, RpcClient } from "@checkstack/backend-api";
|
|
3
|
+
import {
|
|
4
|
+
SearchDocsOutputSchema,
|
|
5
|
+
GetDocOutputSchema,
|
|
6
|
+
} from "@checkstack/ai-common";
|
|
7
|
+
import type { DocsIndexEntry } from "../generated/docs-index";
|
|
8
|
+
import { createAiToolRegistry } from "../tool-registry";
|
|
9
|
+
import { createAiToolResolver } from "../resolver";
|
|
10
|
+
import { createRegistryExtensionPoints } from "../registry-wiring";
|
|
11
|
+
import { pluginMetadata as aiPluginMetadata } from "@checkstack/ai-common";
|
|
12
|
+
import {
|
|
13
|
+
createDocsTools,
|
|
14
|
+
createGetDocTool,
|
|
15
|
+
createSearchDocsTool,
|
|
16
|
+
} from "./docs-tools";
|
|
17
|
+
|
|
18
|
+
const FIXTURE_INDEX: DocsIndexEntry[] = [
|
|
19
|
+
{
|
|
20
|
+
slug: "user-guide/concepts/health-checks",
|
|
21
|
+
title: "Health checks",
|
|
22
|
+
description: "How health checks work",
|
|
23
|
+
headings: ["Strategies", "Collectors"],
|
|
24
|
+
content: "A health check probes a target on a schedule.",
|
|
25
|
+
truncated: false,
|
|
26
|
+
},
|
|
27
|
+
{
|
|
28
|
+
slug: "developer-guide/ai/context-tools",
|
|
29
|
+
title: "Assistant context tools",
|
|
30
|
+
headings: ["searchDocs", "getDoc"],
|
|
31
|
+
content: "The assistant can search and read the docs.",
|
|
32
|
+
truncated: true,
|
|
33
|
+
},
|
|
34
|
+
];
|
|
35
|
+
|
|
36
|
+
/** A user with exactly the given access rules. */
|
|
37
|
+
function userWith(accessRules: string[]): AuthUser {
|
|
38
|
+
return { type: "user", id: "u1", accessRules };
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
const PRINCIPAL = userWith(["ai.chat.read"]);
|
|
42
|
+
|
|
43
|
+
// The docs tools read the bundled docs index, never the RPC client, so a
|
|
44
|
+
// never-called stub satisfies the user-scoped `rpcClient` arg of `execute`.
|
|
45
|
+
const rpcClient = {
|
|
46
|
+
forPlugin: () => {
|
|
47
|
+
throw new Error("docs tools must not call the RPC client");
|
|
48
|
+
},
|
|
49
|
+
} as unknown as RpcClient;
|
|
50
|
+
|
|
51
|
+
describe("ai.searchDocs tool", () => {
|
|
52
|
+
test("returns DocHit[] validating the wire schema", async () => {
|
|
53
|
+
const tool = createSearchDocsTool({ index: FIXTURE_INDEX });
|
|
54
|
+
const out = await tool.execute({
|
|
55
|
+
input: { query: "health check", limit: 5 },
|
|
56
|
+
principal: PRINCIPAL,
|
|
57
|
+
rpcClient,
|
|
58
|
+
});
|
|
59
|
+
expect(() => SearchDocsOutputSchema.parse(out)).not.toThrow();
|
|
60
|
+
expect(out.hits.length).toBeGreaterThan(0);
|
|
61
|
+
expect(out.hits[0]?.slug).toBe("user-guide/concepts/health-checks");
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
test("honours the limit cap", async () => {
|
|
65
|
+
const tool = createSearchDocsTool({ index: FIXTURE_INDEX });
|
|
66
|
+
const out = await tool.execute({
|
|
67
|
+
input: { query: "docs the assistant health", limit: 1 },
|
|
68
|
+
principal: PRINCIPAL,
|
|
69
|
+
rpcClient,
|
|
70
|
+
});
|
|
71
|
+
expect(out.hits.length).toBe(1);
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
test("is effect:read and gated by ai.chat.read", () => {
|
|
75
|
+
const tool = createSearchDocsTool();
|
|
76
|
+
expect(tool.effect).toBe("read");
|
|
77
|
+
expect(tool.requiredAccessRules).toEqual(["ai.chat.read"]);
|
|
78
|
+
expect(tool.dryRun).toBeUndefined();
|
|
79
|
+
});
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
describe("ai.getDoc tool", () => {
|
|
83
|
+
test("returns a known slug's content + truncated flag (schema-valid)", async () => {
|
|
84
|
+
const tool = createGetDocTool({ index: FIXTURE_INDEX });
|
|
85
|
+
const out = await tool.execute({
|
|
86
|
+
input: { slug: "developer-guide/ai/context-tools" },
|
|
87
|
+
principal: PRINCIPAL,
|
|
88
|
+
rpcClient,
|
|
89
|
+
});
|
|
90
|
+
expect(() => GetDocOutputSchema.parse(out)).not.toThrow();
|
|
91
|
+
expect(out.title).toBe("Assistant context tools");
|
|
92
|
+
expect(out.truncated).toBe(true);
|
|
93
|
+
expect(out.content).toContain("search and read the docs");
|
|
94
|
+
});
|
|
95
|
+
|
|
96
|
+
test("an unknown slug throws a clear, recoverable error", async () => {
|
|
97
|
+
const tool = createGetDocTool({ index: FIXTURE_INDEX });
|
|
98
|
+
await expect(
|
|
99
|
+
tool.execute({ input: { slug: "does/not/exist" }, principal: PRINCIPAL, rpcClient }),
|
|
100
|
+
).rejects.toThrow(/searchDocs/);
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
test("is effect:read and gated by ai.chat.read", () => {
|
|
104
|
+
const tool = createGetDocTool();
|
|
105
|
+
expect(tool.effect).toBe("read");
|
|
106
|
+
expect(tool.requiredAccessRules).toEqual(["ai.chat.read"]);
|
|
107
|
+
});
|
|
108
|
+
});
|
|
109
|
+
|
|
110
|
+
describe("docs tools registration + resolution", () => {
|
|
111
|
+
test("both qualify to ai.* and a chat user sees them", () => {
|
|
112
|
+
const registry = createAiToolRegistry();
|
|
113
|
+
const { toolExtensionPoint } = createRegistryExtensionPoints({ registry });
|
|
114
|
+
for (const tool of createDocsTools({ index: FIXTURE_INDEX })) {
|
|
115
|
+
toolExtensionPoint.registerTool(tool, aiPluginMetadata);
|
|
116
|
+
}
|
|
117
|
+
const resolver = createAiToolResolver({ registry });
|
|
118
|
+
|
|
119
|
+
const names = resolver
|
|
120
|
+
.resolveTools(userWith(["ai.chat.read"]))
|
|
121
|
+
.map((t) => t.name)
|
|
122
|
+
.sort();
|
|
123
|
+
expect(names).toEqual(["ai.getDoc", "ai.searchDocs"]);
|
|
124
|
+
});
|
|
125
|
+
|
|
126
|
+
test("a principal without ai.chat.read sees neither docs tool", () => {
|
|
127
|
+
const registry = createAiToolRegistry();
|
|
128
|
+
const { toolExtensionPoint } = createRegistryExtensionPoints({ registry });
|
|
129
|
+
for (const tool of createDocsTools({ index: FIXTURE_INDEX })) {
|
|
130
|
+
toolExtensionPoint.registerTool(tool, aiPluginMetadata);
|
|
131
|
+
}
|
|
132
|
+
const resolver = createAiToolResolver({ registry });
|
|
133
|
+
|
|
134
|
+
const names = resolver
|
|
135
|
+
.resolveTools(userWith(["incident.incident.read"]))
|
|
136
|
+
.map((t) => t.name);
|
|
137
|
+
expect(names).toEqual([]);
|
|
138
|
+
});
|
|
139
|
+
|
|
140
|
+
test("the bundled production index powers searchDocs (smoke test)", async () => {
|
|
141
|
+
// Uses the real DOCS_INDEX default; the AI docs page must be findable.
|
|
142
|
+
const tool = createSearchDocsTool();
|
|
143
|
+
const out = await tool.execute({
|
|
144
|
+
input: { query: "AI assistant documentation tools", limit: 10 },
|
|
145
|
+
principal: PRINCIPAL,
|
|
146
|
+
rpcClient,
|
|
147
|
+
});
|
|
148
|
+
expect(out.hits.length).toBeGreaterThan(0);
|
|
149
|
+
});
|
|
150
|
+
});
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
import { qualifyAccessRuleId } from "@checkstack/common";
|
|
2
|
+
import {
|
|
3
|
+
aiAccess,
|
|
4
|
+
pluginMetadata as aiPluginMetadata,
|
|
5
|
+
SearchDocsInputSchema,
|
|
6
|
+
SearchDocsOutputSchema,
|
|
7
|
+
GetDocInputSchema,
|
|
8
|
+
GetDocOutputSchema,
|
|
9
|
+
type SearchDocsInput,
|
|
10
|
+
type SearchDocsOutput,
|
|
11
|
+
type GetDocInput,
|
|
12
|
+
type GetDocOutput,
|
|
13
|
+
} from "@checkstack/ai-common";
|
|
14
|
+
import { DOCS_INDEX, type DocsIndexEntry } from "../generated/docs-index";
|
|
15
|
+
import { rankDocs } from "./rank-docs";
|
|
16
|
+
import type { RegisteredAiTool } from "../tool-registry";
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Documentation-grounding tools (`ai.searchDocs` + `ai.getDoc`, plan §2.5).
|
|
20
|
+
* Both are composite `effect: "read"` tools: they compose the BUILD-TIME bundled
|
|
21
|
+
* docs index (`DOCS_INDEX`, plan §3.4) rather than projecting a single oRPC
|
|
22
|
+
* procedure, so they run their own `execute` in the chat loop's composite-read
|
|
23
|
+
* path. Gated by `ai.chat.read`: any chat user may read the platform's own
|
|
24
|
+
* public documentation; the docs carry no per-tenant data.
|
|
25
|
+
*
|
|
26
|
+
* The bundled index is identical on every pod (it is part of the same build
|
|
27
|
+
* artifact), so a read returns the same answer everywhere — no pod-local state
|
|
28
|
+
* (plan §7.2).
|
|
29
|
+
*/
|
|
30
|
+
|
|
31
|
+
/** The qualified access rule both docs tools require: `ai.chat.read`. */
|
|
32
|
+
const AI_CHAT_READ = qualifyAccessRuleId(aiPluginMetadata, aiAccess.chatUse);
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Builds `ai.searchDocs`: keyword/BM25-ish ranking over the bundled index. The
|
|
36
|
+
* ranking itself is the pure, unit-tested {@link rankDocs}; this builder only
|
|
37
|
+
* adapts the input/output to the wire contract.
|
|
38
|
+
*
|
|
39
|
+
* `index` is injectable for tests; production uses the bundled `DOCS_INDEX`.
|
|
40
|
+
*/
|
|
41
|
+
export function createSearchDocsTool({
|
|
42
|
+
index = DOCS_INDEX,
|
|
43
|
+
}: {
|
|
44
|
+
index?: readonly DocsIndexEntry[];
|
|
45
|
+
} = {}): RegisteredAiTool<SearchDocsInput, SearchDocsOutput> {
|
|
46
|
+
return {
|
|
47
|
+
name: "searchDocs",
|
|
48
|
+
description:
|
|
49
|
+
"Search Checkstack's own documentation by keyword and return the most " +
|
|
50
|
+
"relevant pages with a short snippet from each. Use this FIRST to ground " +
|
|
51
|
+
"any how-to or conceptual answer about Checkstack in the real docs, then " +
|
|
52
|
+
"call getDoc to read a promising page in full. Read-only.",
|
|
53
|
+
effect: "read",
|
|
54
|
+
input: SearchDocsInputSchema,
|
|
55
|
+
output: SearchDocsOutputSchema,
|
|
56
|
+
requiredAccessRules: [AI_CHAT_READ],
|
|
57
|
+
async execute({ input }) {
|
|
58
|
+
const hits = rankDocs({ index, query: input.query, limit: input.limit });
|
|
59
|
+
return { hits };
|
|
60
|
+
},
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* Builds `ai.getDoc`: returns one documentation page's full (frontmatter-
|
|
66
|
+
* stripped, byte-capped) content by slug. An unknown slug yields a clear error
|
|
67
|
+
* the model can recover from (it can re-`searchDocs`).
|
|
68
|
+
*/
|
|
69
|
+
export function createGetDocTool({
|
|
70
|
+
index = DOCS_INDEX,
|
|
71
|
+
}: {
|
|
72
|
+
index?: readonly DocsIndexEntry[];
|
|
73
|
+
} = {}): RegisteredAiTool<GetDocInput, GetDocOutput> {
|
|
74
|
+
return {
|
|
75
|
+
name: "getDoc",
|
|
76
|
+
description:
|
|
77
|
+
"Read one Checkstack documentation page in full by its slug (as returned " +
|
|
78
|
+
"by searchDocs, e.g. \"user-guide/concepts/health-checks\"). Use this " +
|
|
79
|
+
"after searchDocs to ground an answer in the page's actual content. " +
|
|
80
|
+
"Read-only.",
|
|
81
|
+
effect: "read",
|
|
82
|
+
input: GetDocInputSchema,
|
|
83
|
+
output: GetDocOutputSchema,
|
|
84
|
+
requiredAccessRules: [AI_CHAT_READ],
|
|
85
|
+
async execute({ input }) {
|
|
86
|
+
const entry = index.find((e) => e.slug === input.slug);
|
|
87
|
+
if (!entry) {
|
|
88
|
+
throw new Error(
|
|
89
|
+
`No documentation page with slug "${input.slug}". Use searchDocs to ` +
|
|
90
|
+
"find a valid slug.",
|
|
91
|
+
);
|
|
92
|
+
}
|
|
93
|
+
return {
|
|
94
|
+
slug: entry.slug,
|
|
95
|
+
title: entry.title,
|
|
96
|
+
...(entry.description ? { description: entry.description } : {}),
|
|
97
|
+
content: entry.content,
|
|
98
|
+
truncated: entry.truncated,
|
|
99
|
+
};
|
|
100
|
+
},
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* Builds both docs-grounding tools (the Phase 1 registration unit). Returns the
|
|
106
|
+
* erased `RegisteredAiTool` form so callers can iterate + register uniformly
|
|
107
|
+
* (the two tools have different input/output shapes).
|
|
108
|
+
*/
|
|
109
|
+
export function createDocsTools({
|
|
110
|
+
index = DOCS_INDEX,
|
|
111
|
+
}: {
|
|
112
|
+
index?: readonly DocsIndexEntry[];
|
|
113
|
+
} = {}): RegisteredAiTool[] {
|
|
114
|
+
return [createSearchDocsTool({ index }), createGetDocTool({ index })];
|
|
115
|
+
}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import { describe, expect, test } from "bun:test";
|
|
2
|
+
import type { AuthUser, RpcClient } from "@checkstack/backend-api";
|
|
3
|
+
import { createProbeUrlTool } from "./probe-url";
|
|
4
|
+
|
|
5
|
+
const principal: AuthUser = {
|
|
6
|
+
type: "user",
|
|
7
|
+
id: "u1",
|
|
8
|
+
accessRules: ["ai.chat.read"],
|
|
9
|
+
};
|
|
10
|
+
|
|
11
|
+
// probeUrl never touches the RPC client (it only issues an outbound fetch), so a
|
|
12
|
+
// never-called stub satisfies the user-scoped `rpcClient` arg of `execute`.
|
|
13
|
+
const rpcClient = {
|
|
14
|
+
forPlugin: () => {
|
|
15
|
+
throw new Error("probeUrl must not call the RPC client");
|
|
16
|
+
},
|
|
17
|
+
} as unknown as RpcClient;
|
|
18
|
+
|
|
19
|
+
describe("ai.probeUrl tool", () => {
|
|
20
|
+
test("is a read tool gated by the chat-read surface", () => {
|
|
21
|
+
const tool = createProbeUrlTool();
|
|
22
|
+
expect(tool.name).toBe("probeUrl");
|
|
23
|
+
expect(tool.effect).toBe("read");
|
|
24
|
+
expect(tool.requiredAccessRules).toEqual(["ai.chat.read"]);
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
test("refuses an internal/loopback target before any network call", async () => {
|
|
28
|
+
const tool = createProbeUrlTool();
|
|
29
|
+
await expect(
|
|
30
|
+
tool.execute({ input: { url: "http://localhost:8080/", method: "GET" }, principal, rpcClient }),
|
|
31
|
+
).rejects.toThrow(/loopback|internal/);
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
test("refuses the cloud metadata IP before any network call", async () => {
|
|
35
|
+
const tool = createProbeUrlTool();
|
|
36
|
+
await expect(
|
|
37
|
+
tool.execute({
|
|
38
|
+
input: { url: "http://169.254.169.254/latest/meta-data", method: "GET" },
|
|
39
|
+
principal,
|
|
40
|
+
rpcClient,
|
|
41
|
+
}),
|
|
42
|
+
).rejects.toThrow(/private\/reserved/);
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
test("refuses a non-http(s) scheme", async () => {
|
|
46
|
+
const tool = createProbeUrlTool();
|
|
47
|
+
await expect(
|
|
48
|
+
tool.execute({ input: { url: "file:///etc/passwd", method: "GET" }, principal, rpcClient }),
|
|
49
|
+
).rejects.toThrow(/http and https/);
|
|
50
|
+
});
|
|
51
|
+
});
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
import { lookup } from "node:dns/promises";
|
|
2
|
+
import { z } from "zod";
|
|
3
|
+
import { qualifyAccessRuleId } from "@checkstack/common";
|
|
4
|
+
import type { AuthUser } from "@checkstack/backend-api";
|
|
5
|
+
import { aiAccess, pluginMetadata as aiPluginMetadata } from "@checkstack/ai-common";
|
|
6
|
+
import type { RegisteredAiTool } from "../tool-registry";
|
|
7
|
+
import {
|
|
8
|
+
isPrivateIp,
|
|
9
|
+
parseSafeProbeUrl,
|
|
10
|
+
PROBE_MAX_BODY_BYTES,
|
|
11
|
+
PROBE_TIMEOUT_MS,
|
|
12
|
+
} from "./ssrf-guard";
|
|
13
|
+
|
|
14
|
+
/** Input for `ai.probeUrl`: the URL to introspect and the HTTP method. */
|
|
15
|
+
export const ProbeUrlInputSchema = z.object({
|
|
16
|
+
url: z.string().min(1),
|
|
17
|
+
method: z.enum(["GET", "HEAD"]).default("GET"),
|
|
18
|
+
});
|
|
19
|
+
export type ProbeUrlInput = z.infer<typeof ProbeUrlInputSchema>;
|
|
20
|
+
|
|
21
|
+
/** What a probe reports back to the model (no secrets, body capped). */
|
|
22
|
+
export const ProbeUrlOutputSchema = z.object({
|
|
23
|
+
url: z.string(),
|
|
24
|
+
status: z.number(),
|
|
25
|
+
statusText: z.string(),
|
|
26
|
+
/** True for a 3xx; `location` carries the redirect target (not followed). */
|
|
27
|
+
redirected: z.boolean(),
|
|
28
|
+
location: z.string().optional(),
|
|
29
|
+
contentType: z.string().optional(),
|
|
30
|
+
/** A safe, small subset of response headers. */
|
|
31
|
+
headers: z.record(z.string(), z.string()),
|
|
32
|
+
/** Up to PROBE_MAX_BODY_BYTES of the body, decoded as text. */
|
|
33
|
+
bodySample: z.string(),
|
|
34
|
+
bodyTruncated: z.boolean(),
|
|
35
|
+
});
|
|
36
|
+
export type ProbeUrlOutput = z.infer<typeof ProbeUrlOutputSchema>;
|
|
37
|
+
|
|
38
|
+
/** Response headers we surface (never set-cookie / auth-bearing headers). */
|
|
39
|
+
const SAFE_HEADERS = ["content-type", "content-length", "server", "location"];
|
|
40
|
+
|
|
41
|
+
/** Read up to PROBE_MAX_BODY_BYTES of a response body as text (capped). */
|
|
42
|
+
async function readCappedText(
|
|
43
|
+
response: Response,
|
|
44
|
+
): Promise<{ text: string; truncated: boolean }> {
|
|
45
|
+
if (!response.body) return { text: "", truncated: false };
|
|
46
|
+
const reader = response.body.getReader();
|
|
47
|
+
const chunks: Uint8Array[] = [];
|
|
48
|
+
let total = 0;
|
|
49
|
+
let truncated = false;
|
|
50
|
+
for (;;) {
|
|
51
|
+
const { done, value } = await reader.read();
|
|
52
|
+
if (done) break;
|
|
53
|
+
if (!value) continue;
|
|
54
|
+
const remaining = PROBE_MAX_BODY_BYTES - total;
|
|
55
|
+
if (value.length >= remaining) {
|
|
56
|
+
chunks.push(value.slice(0, remaining));
|
|
57
|
+
total += remaining;
|
|
58
|
+
truncated = true;
|
|
59
|
+
await reader.cancel();
|
|
60
|
+
break;
|
|
61
|
+
}
|
|
62
|
+
chunks.push(value);
|
|
63
|
+
total += value.length;
|
|
64
|
+
}
|
|
65
|
+
const merged = new Uint8Array(total);
|
|
66
|
+
let offset = 0;
|
|
67
|
+
for (const chunk of chunks) {
|
|
68
|
+
merged.set(chunk, offset);
|
|
69
|
+
offset += chunk.length;
|
|
70
|
+
}
|
|
71
|
+
return { text: new TextDecoder().decode(merged), truncated };
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* `ai.probeUrl` - issue ONE outbound HTTP GET/HEAD so the assistant can see what
|
|
76
|
+
* a URL actually returns (status, content type, a body sample) before drafting a
|
|
77
|
+
* health check's assertions. `effect: "read"` (auto-runs).
|
|
78
|
+
*
|
|
79
|
+
* SSRF DEFENSE (the request leaves the CORE BACKEND): the URL shape is validated
|
|
80
|
+
* (`parseSafeProbeUrl` - http(s) only, no loopback/internal host, no private IP
|
|
81
|
+
* literal), the hostname is then DNS-resolved and EVERY resolved IP is checked
|
|
82
|
+
* against the private/reserved ranges (catching a public name that resolves to a
|
|
83
|
+
* private IP), redirects are NOT followed (a 3xx only reports its `location`),
|
|
84
|
+
* the request carries no credentials, times out at PROBE_TIMEOUT_MS, and the body
|
|
85
|
+
* is read capped at PROBE_MAX_BODY_BYTES.
|
|
86
|
+
*/
|
|
87
|
+
export function createProbeUrlTool(): RegisteredAiTool<
|
|
88
|
+
ProbeUrlInput,
|
|
89
|
+
ProbeUrlOutput
|
|
90
|
+
> {
|
|
91
|
+
return {
|
|
92
|
+
name: "probeUrl",
|
|
93
|
+
description:
|
|
94
|
+
"Make ONE outbound HTTP GET or HEAD request to a public URL and return its status code, content type, response headers, and a sample of the body. Use this to inspect what an endpoint returns (e.g. what GET https://foo.bar/status responds with) BEFORE drafting a health check's assertions, so you assert on the real response. Redirects are not followed (a 3xx returns its Location). Cannot reach private/internal addresses.",
|
|
95
|
+
effect: "read",
|
|
96
|
+
input: ProbeUrlInputSchema,
|
|
97
|
+
output: ProbeUrlOutputSchema,
|
|
98
|
+
// Same broad chat-read surface as the docs/capability read tools.
|
|
99
|
+
requiredAccessRules: [qualifyAccessRuleId(aiPluginMetadata, aiAccess.chatUse)],
|
|
100
|
+
async execute({ input }: { input: ProbeUrlInput; principal: AuthUser }) {
|
|
101
|
+
const { url, hostname } = parseSafeProbeUrl(input.url);
|
|
102
|
+
|
|
103
|
+
// DNS re-check: a public-looking hostname must not resolve to a private IP.
|
|
104
|
+
const resolved = await lookup(hostname, { all: true });
|
|
105
|
+
for (const { address } of resolved) {
|
|
106
|
+
if (isPrivateIp(address)) {
|
|
107
|
+
throw new Error(
|
|
108
|
+
`Refusing to probe "${hostname}": it resolves to a private/reserved address (${address}).`,
|
|
109
|
+
);
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
const response = await fetch(url, {
|
|
114
|
+
method: input.method,
|
|
115
|
+
redirect: "manual",
|
|
116
|
+
signal: AbortSignal.timeout(PROBE_TIMEOUT_MS),
|
|
117
|
+
// No credentials/cookies; advertise a benign accept.
|
|
118
|
+
headers: { accept: "*/*" },
|
|
119
|
+
});
|
|
120
|
+
|
|
121
|
+
const headers: Record<string, string> = {};
|
|
122
|
+
for (const name of SAFE_HEADERS) {
|
|
123
|
+
const value = response.headers.get(name);
|
|
124
|
+
if (value !== null) headers[name] = value;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
const redirected = response.status >= 300 && response.status < 400;
|
|
128
|
+
const { text, truncated } =
|
|
129
|
+
input.method === "HEAD"
|
|
130
|
+
? { text: "", truncated: false }
|
|
131
|
+
: await readCappedText(response);
|
|
132
|
+
|
|
133
|
+
return {
|
|
134
|
+
url: url.toString(),
|
|
135
|
+
status: response.status,
|
|
136
|
+
statusText: response.statusText,
|
|
137
|
+
redirected,
|
|
138
|
+
location: response.headers.get("location") ?? undefined,
|
|
139
|
+
contentType: response.headers.get("content-type") ?? undefined,
|
|
140
|
+
headers,
|
|
141
|
+
bodySample: text,
|
|
142
|
+
bodyTruncated: truncated,
|
|
143
|
+
};
|
|
144
|
+
},
|
|
145
|
+
};
|
|
146
|
+
}
|