@flink-app/flink 2.0.0-alpha.91 → 2.0.0-alpha.93

package/dist/src/ai/FlinkAgent.d.ts

@@ -230,6 +230,7 @@ export declare abstract class FlinkAgent<Ctx extends FlinkContext, ConversationC
     private _boundConversationContext?;
     private _llmAdapters?;
     private _tools?;
+    private _observer?;
     abstract id: string;
     abstract description: string;
     /**
@@ -294,7 +295,7 @@ export declare abstract class FlinkAgent<Ctx extends FlinkContext, ConversationC
      */
     __init(llmAdapters: Map<string, any>, tools: {
         [x: string]: ToolExecutor<Ctx>;
-    }): void;
+    }, observer?: AgentObserver): void;
     /**
      * Bind a user to this agent for permission checks
      *
@@ -539,6 +540,12 @@ export interface AgentExecuteInput<ConversationCtx = any> {
     };
 }
 export interface AgentExecuteResult {
+    /**
+     * Framework-generated unique ID for this agent execution.
+     * Matches the `runId` emitted in AgentObserver events, enabling apps to
+     * correlate persisted results with observer traces.
+     */
+    runId: string;
     message: string;
     toolCalls: Array<{
         name: string;
@@ -589,3 +596,95 @@ export interface AgentResponse {
     textStream: AsyncGenerator<string>;
     fullStream: AsyncGenerator<StreamChunk>;
 }
+/**
+ * Event fired once per agent execution, before the first LLM call.
+ * Contains the resolved system instructions and the initial message array
+ * (post-history, pre-compaction, pre-tool-filtering).
+ */
+export interface AgentObserverRunEvent {
+    runId: string;
+    agentId: string;
+    instructions: string;
+    input: AgentExecuteInput;
+    messages: LLMMessage[];
+    tools: string[];
+    model: {
+        adapterId?: string;
+        maxTokens?: number;
+        temperature?: number;
+    };
+    context: AgentExecuteContext;
+}
+/**
+ * Event fired immediately before each LLM call in the agentic loop.
+ * Reflects the messages and tools actually sent to the model after
+ * compaction and per-step permission filtering.
+ */
+export interface AgentObserverLlmCallEvent {
+    runId: string;
+    agentId: string;
+    step: number;
+    maxSteps: number;
+    instructions: string;
+    messages: LLMMessage[];
+    tools: string[];
+    model: {
+        adapterId?: string;
+        maxTokens?: number;
+        temperature?: number;
+    };
+    context: AgentExecuteContext;
+}
+/**
+ * Event fired after each step (LLM call + tool executions) completes.
+ * `toolCalls` contains only the tool calls executed during this step.
+ */
+export interface AgentObserverStepEvent {
+    runId: string;
+    agentId: string;
+    step: number;
+    maxSteps: number;
+    messages: LLMMessage[];
+    assistantText?: string;
+    toolCalls: AgentExecuteResult["toolCalls"];
+    usage?: import("./LLMAdapter").LLMUsage;
+    context: AgentExecuteContext;
+}
+/**
+ * Event fired when an agent execution completes (successfully or with error).
+ * On error, `result` contains whatever was accumulated before the throw.
+ */
+export interface AgentObserverFinishEvent {
+    runId: string;
+    agentId: string;
+    result: AgentExecuteResult;
+    messages: LLMMessage[];
+    durationMs: number;
+    error?: string;
+    context: AgentExecuteContext;
+}
+/**
+ * Global agent observer for app-level tracing, APM, cost accounting, dev tools, etc.
+ *
+ * Register once on `FlinkOptions.ai.observer`; fires for every agent execution in the app.
+ *
+ * Observer callbacks are invoked fire-and-forget: they may return a Promise but the
+ * framework does not await them, and any thrown/rejected errors are caught and logged
+ * without affecting agent execution. Observers are read-only — they must not mutate
+ * inputs, messages, or results. Use the per-agent `beforeRun`/`onStep`/`afterRun` hooks
+ * for business logic that needs to block or mutate.
+ *
+ * Typical use cases:
+ * - Persisted traces for a dev tools page (correlate via `context.metadata`)
+ * - OpenTelemetry / Sentry integration
+ * - Cost accounting and token-usage dashboards
+ *
+ * All events share a stable `runId` per execution so persisted records can be joined
+ * across events. The same `runId` is returned on `AgentExecuteResult.runId`.
+ */
+export interface AgentObserver {
+    onRun?(event: AgentObserverRunEvent): void | Promise<void>;
+    onLlmCall?(event: AgentObserverLlmCallEvent): void | Promise<void>;
+    onStep?(event: AgentObserverStepEvent): void | Promise<void>;
+    onFinish?(event: AgentObserverFinishEvent): void | Promise<void>;
+}

package/dist/src/ai/FlinkAgent.js

@@ -162,9 +162,10 @@ var FlinkAgent = /** @class */ (function () {
      * Internal initialization called by FlinkApp
      * @internal
      */
-    FlinkAgent.prototype.__init = function (llmAdapters, tools) {
+    FlinkAgent.prototype.__init = function (llmAdapters, tools, observer) {
         this._llmAdapters = llmAdapters;
         this._tools = tools;
+        this._observer = observer;
     };
     /**
      * Bind a user to this agent for permission checks
@@ -196,6 +197,9 @@ var FlinkAgent = /** @class */ (function () {
         if (this._tools) {
             bound._tools = this._tools;
         }
+        if (this._observer) {
+            bound._observer = this._observer;
+        }
         if (this._boundUserPermissions !== undefined) {
             bound._boundUserPermissions = this._boundUserPermissions;
         }
@@ -234,6 +238,9 @@ var FlinkAgent = /** @class */ (function () {
         if (this._tools) {
             bound._tools = this._tools;
         }
+        if (this._observer) {
+            bound._observer = this._observer;
+        }
         if (this._boundUser !== undefined) {
             bound._boundUser = this._boundUser;
         }
@@ -271,6 +278,9 @@ var FlinkAgent = /** @class */ (function () {
         if (this._tools) {
             bound._tools = this._tools;
         }
+        if (this._observer) {
+            bound._observer = this._observer;
+        }
         if (this._boundUser !== undefined) {
             bound._boundUser = this._boundUser;
         }
@@ -310,6 +320,9 @@ var FlinkAgent = /** @class */ (function () {
         if (this._tools) {
             bound._tools = this._tools;
         }
+        if (this._observer) {
+            bound._observer = this._observer;
+        }
         if (this._boundUser !== undefined) {
             bound._boundUser = this._boundUser;
         }
@@ -606,8 +619,8 @@ var FlinkAgent = /** @class */ (function () {
             // Get tools map and LLM adapters from internal properties
             var toolsMap = this.resolveTools();
             var llmAdapters = this._llmAdapters;
-            this.runner = new AgentRunner_1.AgentRunner(this.toAgentProps(), toolsMap, llmAdapters, this.getAgentId(), this.ctx // Pass ctx to runner so callbacks can access it
-            );
+            this.runner = new AgentRunner_1.AgentRunner(this.toAgentProps(), toolsMap, llmAdapters, this.getAgentId(), this.ctx, // Pass ctx to runner so callbacks can access it
+            this._observer);
         }
         return this.runner;
     };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@flink-app/flink",
-  "version": "2.0.0-alpha.91",
+  "version": "2.0.0-alpha.93",
   "description": "Typescript only framework for creating REST-like APIs on top of Express and mongodb",
   "types": "dist/src/index.d.ts",
   "main": "dist/src/index.js",

package/spec/AgentObserver.spec.ts

@@ -0,0 +1,266 @@
+import { z } from "zod";
+import {
+    AgentExecuteInput,
+    AgentObserver,
+    FlinkAgent,
+} from "../src/ai/FlinkAgent";
+import { FlinkToolProps } from "../src/ai/FlinkTool";
+import { LLMAdapter, LLMMessage, LLMStreamChunk } from "../src/ai/LLMAdapter";
+import { ToolExecutor } from "../src/ai/ToolExecutor";
+import { FlinkContext } from "../src/FlinkContext";
+import { createStreamingMock } from "./testHelpers";
+
+function makeAgent(opts: {
+    adapter: LLMAdapter;
+    tools?: { [id: string]: ToolExecutor<any> };
+    declaredToolNames?: string[];
+    observer?: AgentObserver;
+    compact?: boolean;
+    ctx?: FlinkContext;
+    permissions?: string | string[] | ((user?: any) => boolean);
+}) {
+    const ctx: FlinkContext = opts.ctx ?? { repos: {}, plugins: {}, agents: {} };
+    const declared = opts.declaredToolNames ?? Object.keys(opts.tools ?? {});
+
+    class TestAgent extends FlinkAgent<FlinkContext> {
+        id = "test-agent";
+        description = "Test agent";
+        instructions() {
+            return "Test instructions";
+        }
+        tools: string[] = declared;
+
+        permissions = opts.permissions;
+
+        // Force compaction to a single-message window to verify onLlmCall sees post-compaction state
+        protected shouldCompact = opts.compact ? () => true : undefined;
+        protected compactHistory = opts.compact ? (msgs: LLMMessage[]) => msgs.slice(-1) : undefined;
+
+        async query(input: AgentExecuteInput) {
+            const response = this.execute(input);
+            return await response.result;
+        }
+    }
+
+    const agent = new TestAgent();
+    (agent as any).ctx = ctx;
+    agent.__init(
+        new Map([["default", opts.adapter]]),
+        opts.tools ?? {},
+        opts.observer
+    );
+    return agent;
+}
+
+describe("AgentObserver", () => {
+    let mockCtx: FlinkContext;
+
+    beforeEach(() => {
+        mockCtx = { repos: {}, plugins: {}, agents: {} };
+    });
+
+    it("fires onRun once with resolved instructions and initial messages", async () => {
+        const onRun = jasmine.createSpy("onRun");
+        const adapter = createStreamingMock([
+            { textContent: "Hello", toolCalls: [], usage: { inputTokens: 1, outputTokens: 2 }, stopReason: "end_turn" },
+        ]);
+        const agent = makeAgent({ adapter, observer: { onRun }, ctx: mockCtx });
+
+        await (agent as any).query({ message: "hi" });
+
+        expect(onRun).toHaveBeenCalledTimes(1);
+        const event = onRun.calls.mostRecent().args[0];
+        expect(event.agentId).toBe("test-agent");
+        expect(event.instructions).toBe("Test instructions");
+        expect(event.messages.length).toBe(1);
+        expect(event.messages[0].content).toBe("hi");
+        expect(typeof event.runId).toBe("string");
+        expect(event.runId.length).toBeGreaterThan(0);
+    });
+
+    it("fires onLlmCall per step with post-compaction messages", async () => {
+        const onLlmCall = jasmine.createSpy("onLlmCall");
+        const adapter = createStreamingMock([
+            { textContent: "done", toolCalls: [], usage: { inputTokens: 1, outputTokens: 2 }, stopReason: "end_turn" },
+        ]);
+        const agent = makeAgent({ adapter, observer: { onLlmCall }, compact: true, ctx: mockCtx });
+
+        await (agent as any).query({
+            message: "new",
+            history: [
+                { role: "user", content: "1" },
+                { role: "assistant", content: "2" },
+                { role: "user", content: "3" },
+            ],
+        });
+
+        expect(onLlmCall).toHaveBeenCalledTimes(1);
+        const event = onLlmCall.calls.mostRecent().args[0];
+        // Compaction takes slice(-1) so only one message is sent to the LLM
+        expect(event.messages.length).toBe(1);
+        expect(event.step).toBe(1);
+    });
+
+    it("fires onStep per step with assistantText, per-step toolCalls, usage", async () => {
+        const onStep = jasmine.createSpy("onStep");
+
+        const toolProps: FlinkToolProps = {
+            id: "t",
+            description: "test",
+            inputSchema: z.object({}),
+        };
+        const toolFn = jasmine.createSpy("toolFn").and.returnValue(Promise.resolve({ success: true, data: { ok: true } }));
+        const toolExecutor = new ToolExecutor(toolProps, toolFn as any, mockCtx);
+
+        const adapter = createStreamingMock([
+            {
+                textContent: "thinking",
+                toolCalls: [{ id: "a", name: "t", input: {} }],
+                usage: { inputTokens: 10, outputTokens: 5 },
+                stopReason: "tool_use",
+            },
+            {
+                textContent: "done",
+                toolCalls: [],
+                usage: { inputTokens: 2, outputTokens: 3 },
+                stopReason: "end_turn",
+            },
+        ]);
+
+        const agent = makeAgent({
+            adapter,
+            observer: { onStep },
+            tools: { t: toolExecutor },
+            declaredToolNames: ["t"],
+            ctx: mockCtx,
+        });
+
+        await (agent as any).query({ message: "go" });
+
+        expect(onStep).toHaveBeenCalledTimes(2);
+        const step1 = onStep.calls.all()[0].args[0];
+        const step2 = onStep.calls.all()[1].args[0];
+
+        expect(step1.step).toBe(1);
+        expect(step1.assistantText).toBe("thinking");
+        expect(step1.toolCalls.length).toBe(1);
+        expect(step1.toolCalls[0].name).toBe("t");
+        expect(step1.usage).toEqual(jasmine.objectContaining({ inputTokens: 10, outputTokens: 5 }));
+
+        expect(step2.step).toBe(2);
+        expect(step2.assistantText).toBe("done");
+        // step2 has no new tool calls
+        expect(step2.toolCalls.length).toBe(0);
+    });
+
+    it("fires onFinish with result.runId matching earlier events' runId", async () => {
+        const events: any[] = [];
+        const adapter = createStreamingMock([
+            { textContent: "ok", toolCalls: [], usage: { inputTokens: 1, outputTokens: 2 }, stopReason: "end_turn" },
+        ]);
+        const observer: AgentObserver = {
+            onRun: (e) => { events.push({ kind: "run", runId: e.runId }); },
+            onLlmCall: (e) => { events.push({ kind: "llm", runId: e.runId }); },
+            onStep: (e) => { events.push({ kind: "step", runId: e.runId }); },
+            onFinish: (e) => { events.push({ kind: "finish", runId: e.runId, resultRunId: e.result.runId }); },
+        };
+        const agent = makeAgent({ adapter, observer, ctx: mockCtx });
+
+        const result = await (agent as any).query({ message: "hi" });
+
+        const runIds = new Set(events.map((e) => e.runId));
+        expect(runIds.size).toBe(1);
+        const runId = events[0].runId;
+        expect(result.runId).toBe(runId);
+        const finish = events.find((e) => e.kind === "finish");
+        expect(finish.resultRunId).toBe(runId);
+    });
+
+    it("fires onFinish with error populated when adapter throws", async () => {
+        const onFinish = jasmine.createSpy("onFinish");
+        const adapter: LLMAdapter = {
+            stream: jasmine.createSpy("stream").and.callFake(async function* () {
+                throw new Error("adapter boom");
+                yield {} as LLMStreamChunk; // unreachable, makes TS happy
+            }),
+        };
+        const agent = makeAgent({ adapter, observer: { onFinish }, ctx: mockCtx });
+
+        await expectAsync((agent as any).query({ message: "hi" })).toBeRejectedWithError(/adapter boom/);
+
+        expect(onFinish).toHaveBeenCalledTimes(1);
+        const event = onFinish.calls.mostRecent().args[0];
+        expect(event.error).toMatch(/adapter boom/);
+        expect(typeof event.runId).toBe("string");
+    });
+
+    it("swallowed observer errors do not break execution", async () => {
+        const adapter = createStreamingMock([
+            { textContent: "ok", toolCalls: [], usage: { inputTokens: 1, outputTokens: 2 }, stopReason: "end_turn" },
+        ]);
+        const observer: AgentObserver = {
+            onRun: () => { throw new Error("sync boom"); },
+            onLlmCall: async () => { throw new Error("async boom"); },
+            onStep: () => { throw new Error("step boom"); },
+            onFinish: () => { throw new Error("finish boom"); },
+        };
+        const agent = makeAgent({ adapter, observer, ctx: mockCtx });
+
+        const result = await (agent as any).query({ message: "hi" });
+        expect(result.message).toBe("ok");
+        expect(result.runId).toBeDefined();
+    });
+
+    it("AgentExecuteResult.runId is populated on success", async () => {
+        const adapter = createStreamingMock([
+            { textContent: "ok", toolCalls: [], usage: { inputTokens: 1, outputTokens: 2 }, stopReason: "end_turn" },
+        ]);
+        const agent = makeAgent({ adapter, ctx: mockCtx });
+
+        const result = await (agent as any).query({ message: "hi" });
+        expect(typeof result.runId).toBe("string");
+        expect(result.runId.length).toBeGreaterThan(0);
+    });
+
+    it("onLlmCall.tools reflects permission filtering", async () => {
+        const onLlmCall = jasmine.createSpy("onLlmCall");
+
+        const allowedProps: FlinkToolProps = {
+            id: "allowed",
+            description: "no perms required",
+            inputSchema: z.object({}),
+        };
+        const deniedProps: FlinkToolProps = {
+            id: "denied",
+            description: "requires admin",
+            inputSchema: z.object({}),
+            permissions: ["admin"],
+        };
+        const allowedFn = jasmine.createSpy("allowedFn").and.returnValue(Promise.resolve({ success: true, data: {} }));
+        const deniedFn = jasmine.createSpy("deniedFn").and.returnValue(Promise.resolve({ success: true, data: {} }));
+
+        const allowed = new ToolExecutor(allowedProps, allowedFn as any, mockCtx);
+        const denied = new ToolExecutor(deniedProps, deniedFn as any, mockCtx);
+
+        const adapter = createStreamingMock([
+            { textContent: "ok", toolCalls: [], usage: { inputTokens: 1, outputTokens: 2 }, stopReason: "end_turn" },
+        ]);
+        const agent = makeAgent({
+            adapter,
+            observer: { onLlmCall },
+            tools: { allowed, denied },
+            declaredToolNames: ["allowed", "denied"],
+            ctx: mockCtx,
+        });
+
+        await (agent as any).query({
+            message: "hi",
+            user: { id: "u1" },
+            userPermissions: [], // no admin perm → denied tool filtered out
+        });
+
+        const event = onLlmCall.calls.mostRecent().args[0];
+        expect(event.tools).toContain("allowed");
+        expect(event.tools).not.toContain("denied");
+    });
+});

package/src/FlinkApp.ts

@@ -11,6 +11,7 @@ import { AsyncTask, CronJob, SimpleIntervalJob, ToadScheduler } from "toad-sched
 import { v4 } from "uuid";
 import { FlinkAgentFile } from "./ai/FlinkAgent";
 import { FlinkToolFile } from "./ai/FlinkTool";
+import { AgentObserver } from "./ai/FlinkAgent";
 import { LLMAdapter } from "./ai/LLMAdapter";
 import { ToolExecutor } from "./ai/ToolExecutor";
 import { FlinkAuthPlugin } from "./auth/FlinkAuthPlugin";
@@ -227,6 +228,22 @@ export interface FlinkOptions {
      */
     ai?: {
         llms?: { [id: string]: LLMAdapter };
+
+        /**
+         * Global agent observer for app-level tracing, APM, cost accounting, dev tools, etc.
+         *
+         * Fires for every agent execution in the app. Observer callbacks are invoked
+         * fire-and-forget — they may return a Promise but the framework does not await
+         * them, and any thrown/rejected errors are caught and logged without affecting
+         * agent execution.
+         *
+         * Events: `onRun` (pre-loop), `onLlmCall` (per step, pre-adapter call),
+         * `onStep` (per step end), `onFinish` (post-loop, including error path).
+         *
+         * For agent-local business logic (conversation persistence, guardrails) use the
+         * per-agent `beforeRun` / `onStep` / `afterRun` hooks on `FlinkAgent` instead.
+         */
+        observer?: AgentObserver;
     };
 
     /**
@@ -351,6 +368,7 @@ export class FlinkApp<C extends FlinkContext> {
     private services: { [x: string]: FlinkService<C> } = {};
 
     private llmAdapters: Map<string, LLMAdapter> = new Map();
+    private agentObserver?: AgentObserver;
     private tools: { [x: string]: ToolExecutor<C> } = {};
     private agents: { [x: string]: any } = {}; // FlinkAgent<C> instances
 
@@ -395,6 +413,9 @@ export class FlinkApp<C extends FlinkContext> {
             // Convert plain object to Map for internal use
             this.llmAdapters = new Map(Object.entries(opts.ai.llms));
         }
+
+        // Register global agent observer if configured
+        this.agentObserver = opts.ai?.observer;
     }
 
     get ctx() {
@@ -1502,7 +1523,7 @@ export class FlinkApp<C extends FlinkContext> {
         // Inject context and initialize agents
         for (const agent of Object.values(this.agents)) {
             agent.ctx = this.ctx;
-            agent.__init(this.llmAdapters, this.tools);
+            agent.__init(this.llmAdapters, this.tools, this.agentObserver);
         }
     }