failproofai 1.0.7-beta.1 → 1.0.7

package/.next/standalone/sdk/typescript/test/tracker-bounds.test.ts

@@ -0,0 +1,191 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+import { runExitClosers } from "../src/exit.js";
+import * as core from "../src/integrations/core.js";
+import { runtime } from "../src/runtime.js";
+import { PAUSED_SESSION_TTL_MS, _stats, adapter, langchainHandler } from "../src/integrations/langchain.js";
+import { flushed, useSpool } from "./helpers.js";
+import type { Spool } from "./helpers.js";
+
+/**
+ * What a LONG-RUNNING process keeps in memory, and what that does to events.
+ *
+ * Every other adapter test runs a handful of runs and looks at the trace. These
+ * run tens of thousands, because the failures here only exist at volume: a
+ * table that is never pruned on normal completion fills to its cap, and a
+ * FIFO cap then evicts the entries of runs that are still LIVE — a model call
+ * that has been running for ten seconds on a busy server loses its
+ * `model_response`, with one warning, while every test with three runs in it
+ * passes.
+ */
+
+type Handler = Record<string, (...args: unknown[]) => unknown>;
+
+let spool: Spool;
+let h: Handler;
+
+beforeEach(() => {
+  spool = useSpool();
+  core.resetFailures();
+  h = langchainHandler() as Handler;
+});
+afterEach(async () => {
+  vi.useRealTimers();
+  adapter.uninstall();
+  await spool.cleanup();
+});
+
+const start = (id: string, parent: string | undefined, name: string, meta: Record<string, unknown> = {}, tags: string[] = []) =>
+  h.handleChainStart!({ name }, {}, id, parent, tags, meta, undefined, name);
+const end = (id: string) => h.handleChainEnd!({}, id);
+
+/** One ordinary request: a graph root and one node, both completed. */
+async function request(i: number): Promise<void> {
+  await start(`r${i}`, undefined, "g", { failproofai_sdk_session_id: `S${i}` });
+  await start(`c${i}`, `r${i}`, "n", { langgraph_node: "n", langgraph_checkpoint_ns: `n:${i}` }, ["graph:step:1"]);
+  await end(`c${i}`);
+  await end(`r${i}`);
+}
+
+describe("RunTracker", () => {
+  it("forgets a link once the run it belongs to has closed", () => {
+    const tracker = new core.RunTracker("t");
+    tracker.startAgent("root", { agentId: "a", sessionId: "s" });
+    tracker.emit("toolUse", "tool", { parentKey: "root", toolName: "x", toolCallId: "1" });
+    expect(tracker.stats().links).toBe(1);
+    tracker.emit("toolResult", "tool", { toolName: "x", toolCallId: "1" });
+    expect(tracker.stats().links).toBe(0);
+    tracker.endAgent("root");
+    expect(tracker.stats()).toEqual({ runs: 0, links: 0 });
+  });
+
+  it("answers isOpen without copying every open agent", () => {
+    const tracker = new core.RunTracker("t");
+    tracker.startAgent("root", { agentId: "a", sessionId: "s" });
+    expect(tracker.isOpen("root")).toBe(true);
+    expect(tracker.isOpen("other")).toBe(false);
+    tracker.forget("root");
+    expect(tracker.isOpen("root")).toBe(false);
+  });
+
+  it("closes open agents at exit, but leaves one paused on a human for its resumer", async () => {
+    // A LangGraph interrupt or a suspended Mastra workflow is resumed by
+    // whichever process takes the answer; ending it here would end a run that
+    // is not over. Everything else a dying process holds is abandoned.
+    const tracker = new core.RunTracker("t");
+    tracker.startAgent("busy", { agentId: "busy", sessionId: "s1" });
+    tracker.startAgent("waiting", { agentId: "waiting", sessionId: "s2" });
+    tracker.emit("agentPause", "waiting", { pauseId: "p1" });
+    tracker.startAgent("resumed", { agentId: "resumed", sessionId: "s3" });
+    tracker.emit("agentPause", "resumed", { pauseId: "p2" });
+    tracker.emit("agentResume", "resumed", { pauseId: "p2" });
+    tracker.closeAtExit();
+    const ends = (await flushed(spool)).filter((e) => e.type === "agent_end");
+    expect(ends.map((e) => [e.agent_id, e.outcome])).toEqual([
+      ["resumed", "failed"],
+      ["busy", "failed"],
+    ]);
+    expect(tracker.isOpen("waiting")).toBe(true);
+  });
+
+  it("closes an adapter's open tool, hook and model call at exit, then its agent with an error", async () => {
+    // Live on LangGraph, Mastra, LlamaIndex and the AI SDK: SIGTERM mid-tool
+    // closed the agent and left the tool_use, the node's hook_triggered and
+    // the model_request open — spans the dashboard shows as running forever.
+    const tracker = new core.RunTracker("t");
+    tracker.startAgent("root", { agentId: "svc", sessionId: "exit-s" });
+    tracker.emit("hookTriggered", "node", { parentKey: "root", hookName: "tools", hookId: "h1" });
+    tracker.emit("modelRequest", "llm", { parentKey: "root", requestId: "r1", model: "m" });
+    tracker.emit("toolUse", "tool", { parentKey: "node", toolName: "slow", toolCallId: "c1" });
+    tracker.emit("toolUse", "done", { parentKey: "node", toolName: "fast", toolCallId: "c0" });
+    tracker.emit("toolResult", "done", { toolName: "fast", toolCallId: "c0" });
+    runExitClosers(143); // both phases, as the writer's exit hook runs them
+    // This session only: the exit closes whatever else this test process left open too.
+    const events = (await flushed(spool)).filter((e) => e.session_id === "exit-s").slice(6); // start, hook, model, two tool_use, one tool_result
+    expect(events.map((e) => [e.type, e.tool_call_id ?? e.hook_id ?? e.request_id ?? null])).toEqual([
+      ["tool_result", "c1"],
+      ["model_response", "r1"],
+      ["hook_completed", "h1"],
+      ["error", null],
+      ["agent_end", null],
+    ]);
+    expect(events[0]!.error).toMatch(/^ProcessExit: the process exited \(code 143\)/);
+    expect(events[1]!.stop_reason).toBe("error");
+    expect(events[2]!.outcome).toBe("failed");
+    expect(events[3]!.error_type).toBe("ProcessExit");
+    expect(events[4]!.outcome).toBe("failed");
+  });
+});
+
+describe("langchain under load", () => {
+  it("keeps a long-running model call attributable while thousands of runs complete", async () => {
+    await start("rootA", undefined, "graph", { failproofai_sdk_session_id: "A" });
+    await start("nodeA", "rootA", "agent", { langgraph_node: "agent", langgraph_checkpoint_ns: "agent:1" }, ["graph:step:1"]);
+    await h.handleChatModelStart!({ name: "ChatX" }, [[]], "modelA", "nodeA", {}, [], { langgraph_node: "agent" }, "ChatX");
+
+    for (let i = 0; i < 12_000; i++) {
+      await request(i);
+      // A real server yields to the writer's flush timer constantly; this tight
+      // loop never does, and would otherwise hit the writer's OWN queue cap.
+      if (i % 1_000 === 0) await runtime.writer.flushNow();
+    }
+
+    await h.handleLLMEnd!(
+      { generations: [[{ text: "hi", message: { content: "hi", usage_metadata: { input_tokens: 3, output_tokens: 1 } } }]] },
+      "modelA",
+    );
+    await end("nodeA");
+    await end("rootA");
+
+    const types = (await flushed(spool)).filter((e) => e.session_id === "A").map((e) => e.type);
+    expect(types).toEqual([
+      "agent_start",
+      "hook_triggered",
+      "model_request",
+      "model_response",
+      "hook_completed",
+      "agent_end",
+    ]);
+  });
+
+  it("holds nothing for a request once it has completed", async () => {
+    for (let i = 0; i < 5_000; i++) await request(i);
+    expect(_stats()).toEqual({ runs: 0, sessions: 0, tracker: { runs: 0, links: 0 } });
+  });
+
+  // A resume arriving here after the TTL takes the cross-worker path, which the
+  // integration suite proves with two real processes (`remote-resume`).
+  it("forgets a paused run after the TTL", async () => {
+    vi.useFakeTimers({ toFake: ["Date"] });
+    const interrupt = (id: string) =>
+      Object.assign(new Error("interrupt"), { name: "GraphInterrupt", is_bubble_up: true, interrupts: [{ id, value: "approve?" }] });
+
+    // Paused here, resumed (if ever) by another worker — the ordinary shape.
+    for (let i = 0; i < 50; i++) {
+      await start(`p${i}`, undefined, "g", { thread_id: `t${i}` });
+      await start(`pn${i}`, `p${i}`, "ask", { langgraph_node: "ask", langgraph_checkpoint_ns: `ask:${i}` }, ["graph:step:1"]);
+      await h.handleChainError!(interrupt(`i${i}`), `pn${i}`);
+      await h.handleChainError!(interrupt(`i${i}`), `p${i}`);
+    }
+    expect(_stats().tracker.runs).toBe(50);
+
+    vi.setSystemTime(Date.now() + PAUSED_SESSION_TTL_MS + 1);
+    await request(0);
+    expect(_stats()).toEqual({ runs: 0, sessions: 0, tracker: { runs: 0, links: 0 } });
+  });
+
+  it("never lets paused runs crowd live ones out of the tracker", async () => {
+    const interrupt = (id: string) =>
+      Object.assign(new Error("interrupt"), { name: "GraphInterrupt", is_bubble_up: true, interrupts: [{ id, value: "approve?" }] });
+    for (let i = 0; i < 3_000; i++) {
+      await start(`p${i}`, undefined, "g", { thread_id: `t${i}` });
+      await start(`pn${i}`, `p${i}`, "ask", { langgraph_node: "ask", langgraph_checkpoint_ns: `ask:${i}` }, ["graph:step:1"]);
+      await h.handleChainError!(interrupt(`i${i}`), `pn${i}`);
+      await h.handleChainError!(interrupt(`i${i}`), `p${i}`);
+    }
+    // Bounded by the session cap, not by the tracker's — the tracker's room is
+    // for runs that are still executing.
+    expect(_stats().tracker.runs).toBeLessThanOrEqual(1_000);
+    expect(_stats().sessions).toBeLessThanOrEqual(1_000);
+  });
+});

package/.next/standalone/sdk/typescript/test/wire-format.test.ts

@@ -0,0 +1,214 @@
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+
+import { setEnvironment } from "../src/environment.js";
+import * as schema from "../src/schema.js";
+
+/**
+ * The wire format, frozen.
+ *
+ * These assertions exist to FAIL when somebody reorders a field or renames one,
+ * because the other end of this pipe is a Rust daemon, an ingest endpoint whose
+ * dedup key hashes the canonical payload, and a Python SDK writing into the
+ * same directories. None of those three will tell us when we drift; this file
+ * is the only thing that will.
+ *
+ * Key ORDER is asserted, not just membership. The identity block comes first,
+ * then `environment`, then the declared optionals that were supplied, then the
+ * caller's extras — the same order `failproofai_sdk/_schema.py` produces.
+ */
+
+const IDENTITY = {
+  timestamp: "2026-01-01T00:00:00.000000Z",
+  sessionId: "s1",
+  agentId: "main",
+} as const;
+
+describe("event wire format", () => {
+  beforeEach(() => {
+    setEnvironment("test-env");
+  });
+  afterEach(() => {
+    setEnvironment(null);
+  });
+
+  it("puts the identity block first, then environment, then optionals, then extras", () => {
+    const event = schema.toolUseEvent({
+      ...IDENTITY,
+      toolName: "search",
+      toolCallId: "c1",
+      input: { q: "kites" },
+      extraFields: { fw_run_id: "r1" },
+    });
+    expect(Object.keys(event)).toEqual([
+      "timestamp",
+      "session_id",
+      "agent_id",
+      "type",
+      "tool_name",
+      "tool_call_id",
+      "environment",
+      "input",
+      "fw_run_id",
+    ]);
+    expect(event).toEqual({
+      timestamp: "2026-01-01T00:00:00.000000Z",
+      session_id: "s1",
+      agent_id: "main",
+      type: "tool_use",
+      tool_name: "search",
+      tool_call_id: "c1",
+      environment: "test-env",
+      input: { q: "kites" },
+      fw_run_id: "r1",
+    });
+  });
+
+  it("omits an optional that was not supplied rather than writing null", () => {
+    const event = schema.toolResultEvent({ ...IDENTITY, toolName: "s", toolCallId: "c1" });
+    expect(Object.keys(event)).toEqual([
+      "timestamp",
+      "session_id",
+      "agent_id",
+      "type",
+      "tool_name",
+      "tool_call_id",
+      "environment",
+    ]);
+    expect("output" in event).toBe(false);
+    expect("duration_ms" in event).toBe(false);
+  });
+
+  it("treats null and undefined identically for optionals", () => {
+    const withNull = schema.agentEndEvent({ ...IDENTITY, outcome: null, summary: null });
+    const withUndefined = schema.agentEndEvent({ ...IDENTITY });
+    expect(withNull).toEqual(withUndefined);
+  });
+
+  it("appends request_id LAST on model events, so older events are byte-identical", () => {
+    const withoutRequestId = schema.modelResponseEvent({ ...IDENTITY, model: "m", role: "assistant" });
+    const withRequestId = schema.modelResponseEvent({
+      ...IDENTITY,
+      model: "m",
+      role: "assistant",
+      requestId: "r1",
+    });
+    expect(Object.keys(withRequestId)).toEqual([...Object.keys(withoutRequestId), "request_id"]);
+    expect(JSON.stringify(withRequestId).startsWith(JSON.stringify(withoutRequestId).slice(0, -1))).toBe(
+      true,
+    );
+  });
+
+  it("lets extras override nothing that is declared — they are merged last by design", () => {
+    // This is the hazard `guardExtras` and the reserved-name check exist for:
+    // the schema itself does NOT protect the declared field.
+    const event = schema.toolUseEvent({
+      ...IDENTITY,
+      toolName: "declared",
+      toolCallId: "c1",
+      extraFields: { tool_name: "overwritten" },
+    });
+    expect(event.tool_name).toBe("overwritten");
+  });
+
+  it("covers all 15 event types with the type discriminator the server reads", () => {
+    const types = [
+      schema.toolUseEvent({ ...IDENTITY, toolName: "t", toolCallId: "c" }),
+      schema.toolResultEvent({ ...IDENTITY, toolName: "t", toolCallId: "c" }),
+      schema.modelRequestEvent({ ...IDENTITY }),
+      schema.modelResponseEvent({ ...IDENTITY }),
+      schema.agentStartEvent({ ...IDENTITY }),
+      schema.agentEndEvent({ ...IDENTITY }),
+      schema.agentPauseEvent({ ...IDENTITY, pauseId: "p" }),
+      schema.agentResumeEvent({ ...IDENTITY, pauseId: "p" }),
+      schema.hookTriggeredEvent({ ...IDENTITY, hookName: "h", hookId: "h1" }),
+      schema.hookCompletedEvent({ ...IDENTITY, hookName: "h", hookId: "h1" }),
+      schema.errorEvent({ ...IDENTITY, errorType: "E", message: "m" }),
+      schema.humanWaitEvent({ ...IDENTITY, inputId: "i" }),
+      schema.humanInputEvent({ ...IDENTITY, inputId: "i" }),
+      schema.humanPauseEvent({ ...IDENTITY }),
+      schema.humanInterruptEvent({ ...IDENTITY }),
+    ].map((event) => event.type);
+
+    expect(types).toEqual([
+      "tool_use",
+      "tool_result",
+      "model_request",
+      "model_response",
+      "agent_start",
+      "agent_end",
+      "agent_pause",
+      "agent_resume",
+      "hook_triggered",
+      "hook_completed",
+      "error",
+      "human_wait",
+      "human_input",
+      "human_pause",
+      "human_interrupt",
+    ]);
+  });
+
+  it("names every declared field, so integrations/core.ts cannot hold a stale copy", () => {
+    // `DECLARED_FIELD_NAMES` drives `FORBIDDEN_EXTRAS`. A field added above and
+    // missing here is a field an adapter could silently overwrite.
+    const declared = new Set<string>();
+    const identity = { ...IDENTITY };
+    const builders = [
+      schema.toolUseEvent({ ...identity, toolName: "t", toolCallId: "c", input: {} }),
+      schema.toolResultEvent({
+        ...identity,
+        toolName: "t",
+        toolCallId: "c",
+        output: 1,
+        error: "e",
+        durationMs: 1,
+      }),
+      schema.modelRequestEvent({
+        ...identity,
+        model: "m",
+        messages: [],
+        system: "s",
+        tools: [],
+        requestId: "r",
+      }),
+      schema.modelResponseEvent({
+        ...identity,
+        model: "m",
+        stopReason: "s",
+        inputTokens: 1,
+        outputTokens: 1,
+        content: "c",
+        role: "assistant",
+        requestId: "r",
+      }),
+      schema.agentStartEvent({ ...identity, goal: "g", parentId: "p" }),
+      schema.agentEndEvent({ ...identity, outcome: "success", summary: "s" }),
+      schema.agentPauseEvent({ ...identity, pauseId: "p", reason: "r", userId: "u" }),
+      schema.agentResumeEvent({ ...identity, pauseId: "p", durationMs: 1, reason: "r", userId: "u" }),
+      schema.hookTriggeredEvent({
+        ...identity,
+        hookName: "h",
+        hookId: "h",
+        triggerEvent: "t",
+        input: 1,
+      }),
+      schema.hookCompletedEvent({
+        ...identity,
+        hookName: "h",
+        hookId: "h",
+        outcome: "o",
+        output: 1,
+        error: "e",
+        durationMs: 1,
+      }),
+      schema.errorEvent({ ...identity, errorType: "E", message: "m", traceback: "t" }),
+      schema.humanWaitEvent({ ...identity, inputId: "i", prompt: "p", options: ["a"], reason: "r" }),
+      schema.humanInputEvent({ ...identity, inputId: "i", response: "r", durationMs: 1 }),
+      schema.humanPauseEvent({ ...identity, reason: "r", userId: "u" }),
+      schema.humanInterruptEvent({ ...identity, reason: "r", userId: "u", atStep: "s" }),
+    ];
+    for (const event of builders) for (const key of Object.keys(event)) declared.add(key);
+
+    expect([...declared].sort()).toEqual([...schema.DECLARED_FIELD_NAMES].sort());
+  });
+});