@flink-app/flink 0.14.3 → 2.0.0-alpha.100

package/spec/AgentObserver.spec.ts

@@ -0,0 +1,266 @@
+import { z } from "zod";
+import {
+    AgentExecuteInput,
+    AgentObserver,
+    FlinkAgent,
+} from "../src/ai/FlinkAgent";
+import { FlinkToolProps } from "../src/ai/FlinkTool";
+import { LLMAdapter, LLMMessage, LLMStreamChunk } from "../src/ai/LLMAdapter";
+import { ToolExecutor } from "../src/ai/ToolExecutor";
+import { FlinkContext } from "../src/FlinkContext";
+import { createStreamingMock } from "./testHelpers";
+
+function makeAgent(opts: {
+    adapter: LLMAdapter;
+    tools?: { [id: string]: ToolExecutor<any> };
+    declaredToolNames?: string[];
+    observer?: AgentObserver;
+    compact?: boolean;
+    ctx?: FlinkContext;
+    permissions?: string | string[] | ((user?: any) => boolean);
+}) {
+    const ctx: FlinkContext = opts.ctx ?? { repos: {}, plugins: {}, agents: {} };
+    const declared = opts.declaredToolNames ?? Object.keys(opts.tools ?? {});
+
+    class TestAgent extends FlinkAgent<FlinkContext> {
+        id = "test-agent";
+        description = "Test agent";
+        instructions() {
+            return "Test instructions";
+        }
+        tools: string[] = declared;
+
+        permissions = opts.permissions;
+
+        // Force compaction to a single-message window to verify onLlmCall sees post-compaction state
+        protected shouldCompact = opts.compact ? () => true : undefined;
+        protected compactHistory = opts.compact ? (msgs: LLMMessage[]) => msgs.slice(-1) : undefined;
+
+        async query(input: AgentExecuteInput) {
+            const response = this.execute(input);
+            return await response.result;
+        }
+    }
+
+    const agent = new TestAgent();
+    (agent as any).ctx = ctx;
+    agent.__init(
+        new Map([["default", opts.adapter]]),
+        opts.tools ?? {},
+        opts.observer
+    );
+    return agent;
+}
+
+describe("AgentObserver", () => {
+    let mockCtx: FlinkContext;
+
+    beforeEach(() => {
+        mockCtx = { repos: {}, plugins: {}, agents: {} };
+    });
+
+    it("fires onRun once with resolved instructions and initial messages", async () => {
+        const onRun = jasmine.createSpy("onRun");
+        const adapter = createStreamingMock([
+            { textContent: "Hello", toolCalls: [], usage: { inputTokens: 1, outputTokens: 2 }, stopReason: "end_turn" },
+        ]);
+        const agent = makeAgent({ adapter, observer: { onRun }, ctx: mockCtx });
+
+        await (agent as any).query({ message: "hi" });
+
+        expect(onRun).toHaveBeenCalledTimes(1);
+        const event = onRun.calls.mostRecent().args[0];
+        expect(event.agentId).toBe("test-agent");
+        expect(event.instructions).toBe("Test instructions");
+        expect(event.messages.length).toBe(1);
+        expect(event.messages[0].content).toBe("hi");
+        expect(typeof event.runId).toBe("string");
+        expect(event.runId.length).toBeGreaterThan(0);
+    });
+
+    it("fires onLlmCall per step with post-compaction messages", async () => {
+        const onLlmCall = jasmine.createSpy("onLlmCall");
+        const adapter = createStreamingMock([
+            { textContent: "done", toolCalls: [], usage: { inputTokens: 1, outputTokens: 2 }, stopReason: "end_turn" },
+        ]);
+        const agent = makeAgent({ adapter, observer: { onLlmCall }, compact: true, ctx: mockCtx });
+
+        await (agent as any).query({
+            message: "new",
+            history: [
+                { role: "user", content: "1" },
+                { role: "assistant", content: "2" },
+                { role: "user", content: "3" },
+            ],
+        });
+
+        expect(onLlmCall).toHaveBeenCalledTimes(1);
+        const event = onLlmCall.calls.mostRecent().args[0];
+        // Compaction takes slice(-1) so only one message is sent to the LLM
+        expect(event.messages.length).toBe(1);
+        expect(event.step).toBe(1);
+    });
+
+    it("fires onStep per step with assistantText, per-step toolCalls, usage", async () => {
+        const onStep = jasmine.createSpy("onStep");
+
+        const toolProps: FlinkToolProps = {
+            id: "t",
+            description: "test",
+            inputSchema: z.object({}),
+        };
+        const toolFn = jasmine.createSpy("toolFn").and.returnValue(Promise.resolve({ success: true, data: { ok: true } }));
+        const toolExecutor = new ToolExecutor(toolProps, toolFn as any, mockCtx);
+
+        const adapter = createStreamingMock([
+            {
+                textContent: "thinking",
+                toolCalls: [{ id: "a", name: "t", input: {} }],
+                usage: { inputTokens: 10, outputTokens: 5 },
+                stopReason: "tool_use",
+            },
+            {
+                textContent: "done",
+                toolCalls: [],
+                usage: { inputTokens: 2, outputTokens: 3 },
+                stopReason: "end_turn",
+            },
+        ]);
+
+        const agent = makeAgent({
+            adapter,
+            observer: { onStep },
+            tools: { t: toolExecutor },
+            declaredToolNames: ["t"],
+            ctx: mockCtx,
+        });
+
+        await (agent as any).query({ message: "go" });
+
+        expect(onStep).toHaveBeenCalledTimes(2);
+        const step1 = onStep.calls.all()[0].args[0];
+        const step2 = onStep.calls.all()[1].args[0];
+
+        expect(step1.step).toBe(1);
+        expect(step1.assistantText).toBe("thinking");
+        expect(step1.toolCalls.length).toBe(1);
+        expect(step1.toolCalls[0].name).toBe("t");
+        expect(step1.usage).toEqual(jasmine.objectContaining({ inputTokens: 10, outputTokens: 5 }));
+
+        expect(step2.step).toBe(2);
+        expect(step2.assistantText).toBe("done");
+        // step2 has no new tool calls
+        expect(step2.toolCalls.length).toBe(0);
+    });
+
+    it("fires onFinish with result.runId matching earlier events' runId", async () => {
+        const events: any[] = [];
+        const adapter = createStreamingMock([
+            { textContent: "ok", toolCalls: [], usage: { inputTokens: 1, outputTokens: 2 }, stopReason: "end_turn" },
+        ]);
+        const observer: AgentObserver = {
+            onRun: (e) => { events.push({ kind: "run", runId: e.runId }); },
+            onLlmCall: (e) => { events.push({ kind: "llm", runId: e.runId }); },
+            onStep: (e) => { events.push({ kind: "step", runId: e.runId }); },
+            onFinish: (e) => { events.push({ kind: "finish", runId: e.runId, resultRunId: e.result.runId }); },
+        };
+        const agent = makeAgent({ adapter, observer, ctx: mockCtx });
+
+        const result = await (agent as any).query({ message: "hi" });
+
+        const runIds = new Set(events.map((e) => e.runId));
+        expect(runIds.size).toBe(1);
+        const runId = events[0].runId;
+        expect(result.runId).toBe(runId);
+        const finish = events.find((e) => e.kind === "finish");
+        expect(finish.resultRunId).toBe(runId);
+    });
+
+    it("fires onFinish with error populated when adapter throws", async () => {
+        const onFinish = jasmine.createSpy("onFinish");
+        const adapter: LLMAdapter = {
+            stream: jasmine.createSpy("stream").and.callFake(async function* () {
+                throw new Error("adapter boom");
+                yield {} as LLMStreamChunk; // unreachable, makes TS happy
+            }),
+        };
+        const agent = makeAgent({ adapter, observer: { onFinish }, ctx: mockCtx });
+
+        await expectAsync((agent as any).query({ message: "hi" })).toBeRejectedWithError(/adapter boom/);
+
+        expect(onFinish).toHaveBeenCalledTimes(1);
+        const event = onFinish.calls.mostRecent().args[0];
+        expect(event.error).toMatch(/adapter boom/);
+        expect(typeof event.runId).toBe("string");
+    });
+
+    it("swallowed observer errors do not break execution", async () => {
+        const adapter = createStreamingMock([
+            { textContent: "ok", toolCalls: [], usage: { inputTokens: 1, outputTokens: 2 }, stopReason: "end_turn" },
+        ]);
+        const observer: AgentObserver = {
+            onRun: () => { throw new Error("sync boom"); },
+            onLlmCall: async () => { throw new Error("async boom"); },
+            onStep: () => { throw new Error("step boom"); },
+            onFinish: () => { throw new Error("finish boom"); },
+        };
+        const agent = makeAgent({ adapter, observer, ctx: mockCtx });
+
+        const result = await (agent as any).query({ message: "hi" });
+        expect(result.message).toBe("ok");
+        expect(result.runId).toBeDefined();
+    });
+
+    it("AgentExecuteResult.runId is populated on success", async () => {
+        const adapter = createStreamingMock([
+            { textContent: "ok", toolCalls: [], usage: { inputTokens: 1, outputTokens: 2 }, stopReason: "end_turn" },
+        ]);
+        const agent = makeAgent({ adapter, ctx: mockCtx });
+
+        const result = await (agent as any).query({ message: "hi" });
+        expect(typeof result.runId).toBe("string");
+        expect(result.runId.length).toBeGreaterThan(0);
+    });
+
+    it("onLlmCall.tools reflects permission filtering", async () => {
+        const onLlmCall = jasmine.createSpy("onLlmCall");
+
+        const allowedProps: FlinkToolProps = {
+            id: "allowed",
+            description: "no perms required",
+            inputSchema: z.object({}),
+        };
+        const deniedProps: FlinkToolProps = {
+            id: "denied",
+            description: "requires admin",
+            inputSchema: z.object({}),
+            permissions: ["admin"],
+        };
+        const allowedFn = jasmine.createSpy("allowedFn").and.returnValue(Promise.resolve({ success: true, data: {} }));
+        const deniedFn = jasmine.createSpy("deniedFn").and.returnValue(Promise.resolve({ success: true, data: {} }));
+
+        const allowed = new ToolExecutor(allowedProps, allowedFn as any, mockCtx);
+        const denied = new ToolExecutor(deniedProps, deniedFn as any, mockCtx);
+
+        const adapter = createStreamingMock([
+            { textContent: "ok", toolCalls: [], usage: { inputTokens: 1, outputTokens: 2 }, stopReason: "end_turn" },
+        ]);
+        const agent = makeAgent({
+            adapter,
+            observer: { onLlmCall },
+            tools: { allowed, denied },
+            declaredToolNames: ["allowed", "denied"],
+            ctx: mockCtx,
+        });
+
+        await (agent as any).query({
+            message: "hi",
+            user: { id: "u1" },
+            userPermissions: [], // no admin perm → denied tool filtered out
+        });
+
+        const event = onLlmCall.calls.mostRecent().args[0];
+        expect(event.tools).toContain("allowed");
+        expect(event.tools).not.toContain("denied");
+    });
+});