@checkstack/ai-backend 0.1.0

package/src/agent-runner.test.ts

@@ -0,0 +1,262 @@
+import { describe, expect, it, mock } from "bun:test";
+import { z } from "zod";
+import type { AuthUser, RpcClient } from "@checkstack/backend-api";
+import type { OpenAiCompatibleConnection } from "@checkstack/ai-common";
+import { createAgentRunner } from "./agent-runner";
+import { createAiToolRegistry } from "./tool-registry";
+import { createAiToolResolver } from "./resolver";
+import { deferredProjectionExecute } from "./projection";
+import type { RegisteredAiTool } from "./tool-registry";
+
+/**
+ * Unit coverage for the headless agent runner. The model is injected, so this
+ * exercises the real tool-resolution + filtering + execution wiring without a
+ * live LLM.
+ */
+
+const connection: OpenAiCompatibleConnection = {
+  baseUrl: "https://example.test/v1",
+  apiKey: "sk-test",
+  defaultModel: "test-model",
+} as OpenAiCompatibleConnection;
+
+const principal: AuthUser = {
+  type: "application",
+  id: "svc-1",
+  name: "Svc",
+  accessRules: ["*"],
+  teamIds: [],
+};
+
+const rpcClient = { forPlugin: () => ({}) } as unknown as RpcClient;
+
+function readTool(name: string, exec: () => Promise<unknown>): RegisteredAiTool {
+  return {
+    name,
+    description: `read ${name}`,
+    effect: "read",
+    input: z.object({}),
+    requiredAccessRules: [],
+    execute: exec,
+  } as RegisteredAiTool;
+}
+
+describe("createAgentRunner", () => {
+  it("offers non-destructive, non-projected tools and runs the loop", async () => {
+    const registry = createAiToolRegistry();
+    const calls: string[] = [];
+    registry.register(
+      readTool("plugin.read", async () => {
+        calls.push("plugin.read");
+        return { ok: true };
+      }),
+    );
+    // A destructive tool must NOT be offered.
+    registry.register({
+      name: "plugin.delete",
+      description: "delete",
+      effect: "destructive",
+      input: z.object({}),
+      requiredAccessRules: [],
+      execute: async () => ({ deleted: true }),
+    } as RegisteredAiTool);
+    // A projected read (deferred sentinel) must NOT be offered in v1.
+    registry.register({
+      name: "plugin.projected",
+      description: "projected",
+      effect: "read",
+      input: z.object({}),
+      requiredAccessRules: [],
+      execute: deferredProjectionExecute,
+    } as RegisteredAiTool);
+
+    const resolver = createAiToolResolver({ registry });
+
+    let offeredToolNames: string[] = [];
+    const generateText = mock(async (args: { tools?: Record<string, unknown> }) => {
+      offeredToolNames = Object.keys(args.tools ?? {});
+      // Simulate the model calling the read tool once.
+      const t = (args.tools ?? {})["plugin.read"] as {
+        execute: (i: unknown) => Promise<unknown>;
+      };
+      await t.execute({});
+      return { text: "done", usage: {} };
+    });
+    const generateObject = mock(async () => ({ object: { severity: "high" }, usage: {} }));
+
+    const runner = createAgentRunner({
+      resolver,
+      resolveConnection: async () => connection,
+      modelFns: {
+        generateText: generateText as never,
+        generateObject: generateObject as never,
+      },
+    });
+
+    const result = await runner({
+      principal,
+      rpcClient,
+      connectionId: "conn-1",
+      prompt: "go",
+      outputSchema: z.object({ severity: z.string() }),
+    });
+
+    expect(offeredToolNames.sort()).toEqual(["plugin.read"]);
+    expect(calls).toEqual(["plugin.read"]);
+    expect(result.text).toBe("done");
+    expect(result.object).toEqual({ severity: "high" });
+    expect(result.toolCalls).toEqual([{ tool: "plugin.read", ok: true }]);
+  });
+
+  it("offers a projected read tool and routes it through the principal's client", async () => {
+    const registry = createAiToolRegistry();
+    registry.register({
+      name: "incident.list",
+      description: "list incidents",
+      effect: "read",
+      input: z.object({}),
+      requiredAccessRules: [],
+      execute: deferredProjectionExecute,
+    } as RegisteredAiTool);
+    const resolver = createAiToolResolver({ registry });
+
+    const procCalls: unknown[] = [];
+    const routedClient = {
+      forPlugin: (def: { pluginId: string }) => {
+        expect(def.pluginId).toBe("incident");
+        return {
+          listIncidents: async (i: unknown) => {
+            procCalls.push(i);
+            return { incidents: [] };
+          },
+        };
+      },
+    } as unknown as RpcClient;
+
+    let offered: string[] = [];
+    const generateText = mock(async (args: { tools?: Record<string, unknown> }) => {
+      offered = Object.keys(args.tools ?? {});
+      const t = (args.tools ?? {})["incident.list"] as {
+        execute: (i: unknown) => Promise<unknown>;
+      };
+      await t.execute({ status: "open" });
+      return { text: "ok", usage: {} };
+    });
+
+    const runner = createAgentRunner({
+      resolver,
+      resolveConnection: async () => connection,
+      getProjectionRoute: (name) =>
+        name === "incident.list"
+          ? { pluginId: "incident", procedureKey: "listIncidents" }
+          : undefined,
+      modelFns: { generateText: generateText as never },
+    });
+
+    const result = await runner({
+      principal,
+      rpcClient: routedClient,
+      connectionId: "conn-1",
+      prompt: "go",
+    });
+
+    expect(offered).toEqual(["incident.list"]);
+    expect(procCalls).toEqual([{ status: "open" }]);
+    expect(result.toolCalls).toEqual([{ tool: "incident.list", ok: true }]);
+  });
+
+  it("records a tool failure and surfaces it to the model instead of aborting", async () => {
+    const registry = createAiToolRegistry();
+    registry.register(
+      readTool("plugin.boom", async () => {
+        throw new Error("missing access: plugin.read");
+      }),
+    );
+    const resolver = createAiToolResolver({ registry });
+
+    let toolResult: unknown;
+    const generateText = mock(async (args: { tools?: Record<string, unknown> }) => {
+      const t = (args.tools ?? {})["plugin.boom"] as {
+        execute: (i: unknown) => Promise<unknown>;
+      };
+      toolResult = await t.execute({});
+      return { text: "handled", usage: {} };
+    });
+
+    const runner = createAgentRunner({
+      resolver,
+      resolveConnection: async () => connection,
+      modelFns: { generateText: generateText as never },
+    });
+
+    const result = await runner({
+      principal,
+      rpcClient,
+      connectionId: "conn-1",
+      prompt: "go",
+    });
+
+    expect(toolResult).toEqual({ error: "missing access: plugin.read" });
+    expect(result.toolCalls).toEqual([{ tool: "plugin.boom", ok: false }]);
+    expect(result.object).toBeUndefined();
+  });
+
+  it("calls recordToolCall for each invocation (ok and failure)", async () => {
+    const registry = createAiToolRegistry();
+    registry.register(readTool("plugin.ok", async () => ({ ok: true })));
+    registry.register(
+      readTool("plugin.boom", async () => {
+        throw new Error("nope");
+      }),
+    );
+    const resolver = createAiToolResolver({ registry });
+
+    const recorded: Array<{ toolName: string; ok: boolean; effect: string }> =
+      [];
+    const recordToolCall = async (a: {
+      toolName: string;
+      ok: boolean;
+      effect: string;
+    }) => {
+      recorded.push({ toolName: a.toolName, ok: a.ok, effect: a.effect });
+    };
+
+    const generateText = mock(async (args: { tools?: Record<string, unknown> }) => {
+      const tools = args.tools ?? {};
+      await (tools["plugin.ok"] as { execute: (i: unknown) => Promise<unknown> }).execute({});
+      await (tools["plugin.boom"] as { execute: (i: unknown) => Promise<unknown> }).execute({});
+      return { text: "x", usage: {} };
+    });
+
+    const runner = createAgentRunner({
+      resolver,
+      resolveConnection: async () => connection,
+      recordToolCall: recordToolCall as never,
+      modelFns: { generateText: generateText as never },
+    });
+    await runner({ principal, rpcClient, connectionId: "c", prompt: "go" });
+
+    expect(recorded).toContainEqual({
+      toolName: "plugin.ok",
+      ok: true,
+      effect: "read",
+    });
+    expect(recorded).toContainEqual({
+      toolName: "plugin.boom",
+      ok: false,
+      effect: "read",
+    });
+  });
+
+  it("throws a clear error when the connection is invalid", async () => {
+    const resolver = createAiToolResolver({ registry: createAiToolRegistry() });
+    const runner = createAgentRunner({
+      resolver,
+      resolveConnection: async () => undefined,
+      modelFns: { generateText: mock(async () => ({ text: "", usage: {} })) as never },
+    });
+    await expect(
+      runner({ principal, rpcClient, connectionId: "missing", prompt: "go" }),
+    ).rejects.toThrow(/connection "missing"/i);
+  });
+});

package/src/agent-runner.ts

@@ -0,0 +1,262 @@
+/**
+ * Headless AI agent runner.
+ *
+ * The chat agent loop is HTTP/streaming/conversation-coupled. This is the
+ * transport-agnostic core for running ONE bounded agent task with no human in
+ * the loop - the engine behind the automation "AI Action". It is exposed as a
+ * service ({@link aiAgentRunnerRef}) so `automation-backend` (which already
+ * depends on `ai-backend`) can drive it without ai-backend depending on
+ * automation-backend.
+ *
+ * Security: the runner resolves the allowed tools for the supplied PRINCIPAL
+ * (the automation's `runAs` service account) and executes them through the
+ * supplied `rpcClient` (bound to that same principal), so every call is
+ * authorized exactly as that bounded identity. Destructive tools are NEVER
+ * offered (no human to confirm), matching the chat invariant.
+ *
+ * Tools offered: hand-authored read + mutate tools (run via their own
+ * `execute`) AND projected read tools (routed through the live router as the
+ * principal via the supplied `getProjectionRoute`). Destructive tools are never
+ * offered. Mutating tools auto-apply (executed directly via the principal's
+ * client); the propose/apply token gate is a chat/MCP human gate, intentionally
+ * bypassed here (the run is unattended and bounded by the principal).
+ */
+import {
+  tool as aiTool,
+  stepCountIs,
+  generateText,
+  generateObject,
+  type Tool,
+  type LanguageModel,
+} from "ai";
+import { z } from "zod";
+import {
+  createServiceRef,
+  type AuthUser,
+  type RpcClient,
+} from "@checkstack/backend-api";
+import type {
+  OpenAiCompatibleConnection,
+  AiToolEffect,
+} from "@checkstack/ai-common";
+import { extractErrorMessage, type ClientDefinition } from "@checkstack/common";
+import type { AiToolResolver } from "./resolver";
+import { buildLanguageModel } from "./chat/llm-provider";
+import { deferredProjectionExecute } from "./projection";
+
+const DEFAULT_MAX_STEPS = 8;
+
+const DEFAULT_SYSTEM_PROMPT = [
+  "You are an automation agent acting UNATTENDED - there is no human to ask.",
+  "You run with a bounded service account; you can only do what its permissions allow.",
+  "Use the available tools to investigate and act decisively on the task.",
+  "If a tool call is refused, do not retry it - work within your permissions.",
+  "Never attempt destructive actions. Be concise.",
+].join(" ");
+
+/** One tool invocation outcome, surfaced in the action's artifact for audit. */
+export interface AgentTaskToolCall {
+  tool: string;
+  ok: boolean;
+}
+
+export interface AgentTaskInput {
+  /**
+   * The principal the task runs as (the automation's `runAs` service account,
+   * already enriched). The runner resolves the allowed tools from it and
+   * executes them through `rpcClient`, which must be bound to the SAME
+   * identity, so authorization is enforced exactly as that bounded principal.
+   */
+  principal: AuthUser;
+  /** RPC client bound to that same service account (the run's `context.rpcClient`). */
+  rpcClient: RpcClient;
+  /** Integration connection id for the OpenAI-compatible model. */
+  connectionId: string;
+  /** Optional model override (validated against the connection's allowlist). */
+  model?: string;
+  /** Optional system-prompt override. */
+  systemPrompt?: string;
+  /** The task prompt (the automation injects its trigger/artifact context here). */
+  prompt: string;
+  /**
+   * When provided, after the tool loop the runner runs a structured-output
+   * pass to fill this schema - the action's typed artifact.
+   */
+  outputSchema?: z.ZodType;
+  /** Max agent steps (tool-call rounds). Defaults to 8. */
+  maxSteps?: number;
+}
+
+export interface AgentTaskResult {
+  /** The model's free-text result of the loop. */
+  text: string;
+  /** The structured object, present iff `outputSchema` was provided. */
+  object?: unknown;
+  /** Per-tool-call outcomes. */
+  toolCalls: AgentTaskToolCall[];
+}
+
+export type AiAgentRunner = (input: AgentTaskInput) => Promise<AgentTaskResult>;
+
+/** Service ref so `automation-backend` can resolve the runner at action time. */
+export const aiAgentRunnerRef =
+  createServiceRef<AiAgentRunner>("ai.agentRunner");
+
+/** Injectable model functions (so the runner is unit-testable without a model). */
+export interface AgentRunnerModelFns {
+  generateText: typeof generateText;
+  generateObject: typeof generateObject;
+}
+
+export function createAgentRunner({
+  resolver,
+  resolveConnection,
+  getProjectionRoute,
+  recordToolCall,
+  modelFns,
+}: {
+  resolver: AiToolResolver;
+  resolveConnection: (
+    connectionId: string,
+  ) => Promise<OpenAiCompatibleConnection | undefined>;
+  /**
+   * Resolve a projected read tool's underlying procedure routing
+   * (`{ pluginId, procedureKey }`). When provided, projected read tools are
+   * offered and invoked through the principal's client; when omitted they are
+   * not offered (cannot be run headlessly).
+   */
+  getProjectionRoute?: (
+    toolName: string,
+  ) => { pluginId: string; procedureKey: string } | undefined;
+  /**
+   * Best-effort audit hook, called once per tool invocation. Lets the host
+   * record the call into the durable AI audit log (with the `automation`
+   * transport). Failures here never break the agent loop.
+   */
+  recordToolCall?: (args: {
+    principal: AuthUser;
+    toolName: string;
+    effect: AiToolEffect;
+    input: unknown;
+    ok: boolean;
+    error?: string;
+  }) => Promise<void>;
+  modelFns?: Partial<AgentRunnerModelFns>;
+}): AiAgentRunner {
+  const gen = modelFns?.generateText ?? generateText;
+  const genObj = modelFns?.generateObject ?? generateObject;
+
+  return async ({
+    principal,
+    rpcClient,
+    connectionId,
+    model,
+    systemPrompt,
+    prompt,
+    outputSchema,
+    maxSteps,
+  }) => {
+    const connection = await resolveConnection(connectionId);
+    if (!connection) {
+      throw new Error(
+        `AI connection "${connectionId}" not found or not a valid OpenAI-compatible connection.`,
+      );
+    }
+    const languageModel: LanguageModel = buildLanguageModel({
+      connection,
+      model,
+    });
+
+    // Offer the principal's read + mutate tools (never destructive - no human
+    // to confirm). Hand-authored tools run via their own `execute`; projected
+    // read tools (the deferred sentinel) are routed through the live router AS
+    // the principal via `rpcClient`, so handler-side authz still holds.
+    const offered = resolver
+      .resolveTools(principal)
+      .filter((t) => t.effect !== "destructive");
+
+    const toolCalls: AgentTaskToolCall[] = [];
+    const sdkTools: Record<string, Tool> = {};
+    for (const t of offered) {
+      const isProjected = t.execute === deferredProjectionExecute;
+      const route = isProjected ? getProjectionRoute?.(t.name) : undefined;
+      // A projected tool with no resolvable route cannot be invoked headlessly;
+      // skip it rather than calling the deferred sentinel (which throws).
+      if (isProjected && !route) continue;
+
+      const invoke = route
+        ? async (input: unknown) => {
+            // `forPlugin` only reads `.pluginId`; this re-enters the live router
+            // AS the principal for the projected read's underlying procedure,
+            // so handler-side authz applies exactly as a direct call.
+            const pluginClient = rpcClient.forPlugin({
+              pluginId: route.pluginId,
+            } as ClientDefinition) as Record<
+              string,
+              (i: unknown) => Promise<unknown>
+            >;
+            return pluginClient[route.procedureKey](input);
+          }
+        : async (input: unknown) => t.execute({ input, principal, rpcClient });
+
+      sdkTools[t.name] = aiTool({
+        description: t.description,
+        inputSchema: t.input as z.ZodType,
+        execute: async (input: unknown) => {
+          try {
+            const result = await invoke(input);
+            toolCalls.push({ tool: t.name, ok: true });
+            if (recordToolCall) {
+              await recordToolCall({
+                principal,
+                toolName: t.name,
+                effect: t.effect,
+                input,
+                ok: true,
+              }).catch(() => {});
+            }
+            return result;
+          } catch (error) {
+            // Surface the failure to the model (e.g. a missing-permission
+            // message) so it can adapt, rather than aborting the whole run.
+            const message = extractErrorMessage(error);
+            toolCalls.push({ tool: t.name, ok: false });
+            if (recordToolCall) {
+              await recordToolCall({
+                principal,
+                toolName: t.name,
+                effect: t.effect,
+                input,
+                ok: false,
+                error: message,
+              }).catch(() => {});
+            }
+            return { error: message };
+          }
+        },
+      });
+    }
+
+    const { text } = await gen({
+      model: languageModel,
+      system: systemPrompt ?? DEFAULT_SYSTEM_PROMPT,
+      prompt,
+      tools: sdkTools,
+      stopWhen: stepCountIs(maxSteps ?? DEFAULT_MAX_STEPS),
+    });
+
+    let object: unknown;
+    if (outputSchema) {
+      const res = await genObj({
+        model: languageModel,
+        schema: outputSchema,
+        system:
+          "Produce the structured result from the analysis below. Use only information present in it; do not invent values.",
+        prompt: `Task: ${prompt}\n\n--- Analysis ---\n${text}`,
+      });
+      object = res.object;
+    }
+
+    return { text, object, toolCalls };
+  };
+}

package/src/chat/agent-loop.test.ts

@@ -0,0 +1,119 @@
+import { describe, expect, test } from "bun:test";
+import { z } from "zod";
+import type { AuthUser } from "@checkstack/backend-api";
+import { createAiToolRegistry } from "../tool-registry";
+import { createAiToolResolver } from "../resolver";
+import type { RegisteredAiTool } from "../tool-registry";
+import { disposeAgentTool, offeredTools } from "./agent-loop";
+
+function tool(
+  name: string,
+  effect: RegisteredAiTool["effect"],
+  rule: string,
+): RegisteredAiTool {
+  return {
+    name,
+    description: name,
+    effect,
+    input: z.object({}),
+    requiredAccessRules: [rule],
+    ...(effect === "read"
+      ? {}
+      : {
+          dryRun: async () => ({ summary: `would ${name}`, payload: {} }),
+        }),
+    execute: () => Promise.resolve({ ok: true }),
+  };
+}
+
+function setup() {
+  const registry = createAiToolRegistry();
+  const read = tool("incident.list", "read", "incident.incident.read");
+  const mutate = tool("automation.propose", "mutate", "automation.automation.manage");
+  const destroy = tool("incident.delete", "destructive", "incident.incident.manage");
+  registry.register(read);
+  registry.register(mutate);
+  registry.register(destroy);
+  const resolver = createAiToolResolver({ registry });
+  return { registry, resolver };
+}
+
+const limited: AuthUser = {
+  type: "user",
+  id: "u1",
+  accessRules: ["incident.incident.read"],
+};
+const power: AuthUser = {
+  type: "user",
+  id: "u2",
+  accessRules: [
+    "incident.incident.read",
+    "automation.automation.manage",
+    "incident.incident.manage",
+  ],
+};
+
+describe("agent loop tool gating (matrix #14)", () => {
+  test("the loop only offers resolver-allowed tools", () => {
+    const { resolver } = setup();
+    const offered = offeredTools({ principal: limited, resolver }).map((t) => t.name);
+    expect(offered).toEqual(["incident.list"]);
+    expect(offered).not.toContain("automation.propose");
+    expect(offered).not.toContain("incident.delete");
+  });
+
+  test("a model-requested tool OUTSIDE the principal's set is refused server-side", () => {
+    const { resolver, registry } = setup();
+    const d = disposeAgentTool({
+      toolName: "automation.propose",
+      principal: limited,
+      resolver,
+      getTool: (n) => registry.getTool(n),
+    });
+    expect(d.kind).toBe("refused");
+  });
+
+  test("an unknown tool name is refused", () => {
+    const { resolver, registry } = setup();
+    const d = disposeAgentTool({
+      toolName: "does.not.exist",
+      principal: power,
+      resolver,
+      getTool: (n) => registry.getTool(n),
+    });
+    expect(d.kind).toBe("refused");
+  });
+
+  test("a read tool auto-runs", () => {
+    const { resolver, registry } = setup();
+    const d = disposeAgentTool({
+      toolName: "incident.list",
+      principal: limited,
+      resolver,
+      getTool: (n) => registry.getTool(n),
+    });
+    expect(d.kind).toBe("run");
+  });
+
+  test("a mutate tool requires a confirm card (never silently mutates)", () => {
+    const { resolver, registry } = setup();
+    const d = disposeAgentTool({
+      toolName: "automation.propose",
+      principal: power,
+      resolver,
+      getTool: (n) => registry.getTool(n),
+    });
+    expect(d.kind).toBe("confirm");
+  });
+
+  test("a destructive tool requires a confirm card", () => {
+    const { resolver, registry } = setup();
+    const d = disposeAgentTool({
+      toolName: "incident.delete",
+      principal: power,
+      resolver,
+      getTool: (n) => registry.getTool(n),
+    });
+    expect(d.kind).toBe("confirm");
+  });
+});