failproofai 1.0.7-beta.1 → 1.0.7

package/.next/standalone/sdk/typescript/integration/llamaindex.test.ts

@@ -0,0 +1,709 @@
+import { describe, expect, it } from "vitest";
+
+import {
+  FORMATS,
+  count,
+  describeTrace,
+  ofType,
+  runAgent,
+  traceViolations,
+  typecheck,
+  type Event,
+} from "./harness.js";
+
+/**
+ * LlamaIndex.TS, against real releases.
+ *
+ * The expected traces follow the Python SDK's LlamaIndex adapter, captured from
+ * llama-index-core 0.14.24 running the equivalent `FunctionAgent` program with a
+ * scripted model (same tool, same tool call id, same two turns). The two SDKs
+ * write into one pipe, so for the same program they must draw the same tree
+ * (`sdk/python/skill/references/frameworks.md`):
+ *
+ *   * a workflow run is the agent — session + `agent_start`/`agent_end` — named
+ *     after the agent (`FunctionAgent.name`, `"Agent"` by default), never an id;
+ *   * each workflow step is a HOOK (`trigger_event="workflow_step"`);
+ *   * a multi-agent workflow is `AgentWorkflow` with one nested agent per agent
+ *     holding the turn;
+ *   * a bare model call outside any run is its own run, named after the class;
+ *   * a tool failure lives on its `tool_result`, not in an `error` event.
+ *
+ * Where TS genuinely differs, it is the framework and not the adapter: the step
+ * NAMES are LlamaIndex.TS's own handler names (`runAgentStep`, where Python has
+ * `run_agent_step`), TS executes every tool call of a turn in ONE
+ * `executeToolCalls` step (Python runs a `call_tool` step per call), and
+ * `parseAgentOutput` starts INSIDE `runAgentStep` because the TS runtime
+ * dispatches the next step synchronously from `sendEvent`.
+ */
+
+/**
+ * The supported floor and the latest release. 0.11.4 is the first `llamaindex`
+ * on `@llamaindex/workflow` 1.1 (then still `@llama-flow/core` underneath, with
+ * step handlers called as `handler(event)` and no bus event for a tool call);
+ * 0.9–0.11.3 ship workflow 1.0, a different runtime whose `AgentWorkflow` has no
+ * `runStream` at all.
+ */
+const FIXTURES = ["llamaindex-0.11", "llamaindex-0.12"] as const;
+
+/** One line per event: `agent_id type [hook|tool]`. */
+const shape = (events: Event[]): string[] =>
+  events.map((e) => [e.agent_id, e.type, (e.hook_name ?? e.tool_name ?? "") as string].join(" ").trim());
+
+const hook = (agentId: string, name: string, body: string[] = []): string[] => [
+  `${agentId} hook_triggered ${name}`,
+  ...body,
+  `${agentId} hook_completed ${name}`,
+];
+
+const MODEL = (agentId: string) => [`${agentId} model_request`, `${agentId} model_response`];
+
+/** One `FunctionAgent` turn that asks for `tool`, then the turn that answers. */
+const agentLoop = (agentId: string, tool: string): string[] => [
+  ...hook(agentId, "setupAgent"),
+  `${agentId} hook_triggered runAgentStep`,
+  ...MODEL(agentId),
+  ...hook(agentId, "parseAgentOutput"),
+  `${agentId} hook_completed runAgentStep`,
+  ...hook(agentId, "executeToolCalls", [`${agentId} tool_use ${tool}`, `${agentId} tool_result ${tool}`]),
+  ...hook(agentId, "processToolResults"),
+  ...hook(agentId, "setupAgent"),
+  `${agentId} hook_triggered runAgentStep`,
+  ...MODEL(agentId),
+  `${agentId} hook_triggered parseAgentOutput`,
+  `${agentId} hook_completed runAgentStep`,
+  `${agentId} hook_completed parseAgentOutput`,
+];
+
+const WORKFLOW = (agentId = "Agent", tool = "get_weather"): string[] => [
+  `${agentId} agent_start`,
+  ...hook(agentId, "handleInputStep"),
+  ...agentLoop(agentId, tool),
+  `${agentId} agent_end`,
+];
+
+describe.each(FIXTURES)("%s", (fixture) => {
+  it("typechecks as a customer's nodenext ES-module project", () => {
+    expect(typecheck(fixture)).toBe("");
+  });
+
+  describe.each(FORMATS)("as %s", (format) => {
+    const run = (scenario: string, env: Record<string, string> = {}) => {
+      const result = runAgent(fixture, format, scenario, env);
+      expect(result.status, describeTrace(result)).toBe(0);
+      // Instrumenting must never load a second copy of the framework: LlamaIndex
+      // detects it and prints this to the customer's terminal.
+      expect(result.stderr).not.toContain("already imported");
+      expect(result.stdout).toContain('"instrumented":["llamaindex"]');
+      return result;
+    };
+
+    it("records a workflow agent run as one agent with its steps as hooks", () => {
+      const result = run("workflow");
+      expect(shape(result.events), describeTrace(result)).toEqual(WORKFLOW());
+      expect(traceViolations(result.events), describeTrace(result)).toEqual([]);
+      expect(new Set(result.events.map((e) => e.session_id)).size).toBe(1);
+      for (const event of ofType(result.events, "hook_triggered")) {
+        expect(event.trigger_event).toBe("workflow_step");
+      }
+      const responses = ofType(result.events, "model_response");
+      expect(responses.map((e) => [e.input_tokens, e.output_tokens])).toEqual([
+        [12, 5],
+        [30, 7],
+      ]);
+      for (const event of [...ofType(result.events, "model_request"), ...responses]) {
+        expect(event.model).toBe("scripted-1");
+      }
+      // The model's own tool call id, so a tool_use joins to the tool call that
+      // asked for it.
+      expect(ofType(result.events, "tool_use")[0]!.tool_call_id).toBe("call_1");
+      expect(ofType(result.events, "tool_result")[0]!.output).toBe("sunny in Paris");
+      const end = ofType(result.events, "agent_end")[0]!;
+      expect(end.outcome).toBe("success");
+      expect(end.summary).toBe("It is sunny in Paris.");
+      expect(ofType(result.events, "agent_start")[0]!.goal).toBe("weather in Paris?");
+      for (const event of result.events) {
+        expect(event.framework).toBe("llamaindex");
+        expect(typeof event.framework_version).toBe("string");
+      }
+      expect(result.stdout).toContain("It is sunny in Paris.");
+    });
+
+    it("names the run after the agent", () => {
+      const result = run("named");
+      expect(shape(result.events), describeTrace(result)).toEqual(WORKFLOW("weather_bot"));
+    });
+
+    it("records an agent built before instrument()", () => {
+      const result = run("early");
+      expect(shape(result.events), describeTrace(result)).toEqual(WORKFLOW());
+    });
+
+    it("records runStream() exactly like run()", () => {
+      const result = run("stream");
+      expect(shape(result.events), describeTrace(result)).toEqual(WORKFLOW());
+      expect(traceViolations(result.events), describeTrace(result)).toEqual([]);
+    });
+
+    it("records a multi-agent handoff as nested agents under the workflow", () => {
+      const result = run("handoff");
+      expect(shape(result.events), describeTrace(result)).toEqual([
+        "AgentWorkflow agent_start",
+        ...hook("AgentWorkflow", "handleInputStep"),
+        "triage agent_start",
+        ...hook("triage", "setupAgent"),
+        "triage hook_triggered runAgentStep",
+        ...MODEL("triage"),
+        ...hook("triage", "parseAgentOutput"),
+        "triage hook_completed runAgentStep",
+        ...hook("triage", "executeToolCalls", ["triage tool_use handOff", "triage tool_result handOff"]),
+        ...hook("triage", "processToolResults"),
+        "triage agent_end",
+        "forecaster agent_start",
+        ...agentLoop("forecaster", "get_weather"),
+        "forecaster agent_end",
+        "AgentWorkflow agent_end",
+      ]);
+      expect(traceViolations(result.events), describeTrace(result)).toEqual([]);
+      for (const start of ofType(result.events, "agent_start").slice(1)) {
+        expect(start.parent_id).toBe("AgentWorkflow");
+      }
+    });
+
+    it("records a legacy LLMAgent as one agent across all its steps", () => {
+      const result = run("legacy");
+      expect(shape(result.events), describeTrace(result)).toEqual([
+        "LLMAgent agent_start",
+        ...MODEL("LLMAgent"),
+        "LLMAgent tool_use get_weather",
+        "LLMAgent tool_result get_weather",
+        ...MODEL("LLMAgent"),
+        "LLMAgent agent_end",
+      ]);
+      expect(traceViolations(result.events), describeTrace(result)).toEqual([]);
+      expect(new Set(result.events.map((e) => e.session_id)).size).toBe(1);
+      expect(ofType(result.events, "model_response").map((e) => [e.input_tokens, e.output_tokens])).toEqual([
+        [12, 5],
+        [30, 7],
+      ]);
+      expect(ofType(result.events, "model_request")[0]!.model).toBe("scripted-1");
+    });
+
+    it("records a bare model call as its own run", () => {
+      const result = run("model");
+      expect(shape(result.events), describeTrace(result)).toEqual([
+        "ScriptedLLM agent_start",
+        ...MODEL("ScriptedLLM"),
+        "ScriptedLLM agent_end",
+      ]);
+      expect(traceViolations(result.events), describeTrace(result)).toEqual([]);
+      const response = ofType(result.events, "model_response")[0]!;
+      expect([response.input_tokens, response.output_tokens]).toEqual([3, 1]);
+      expect(response.model).toBe("scripted-1");
+    });
+
+    it("nests the run under an enclosing agent() scope", () => {
+      const result = run("scope");
+      expect(shape(result.events), describeTrace(result)).toEqual([
+        "planner agent_start",
+        ...WORKFLOW(),
+        "planner agent_end",
+      ]);
+      expect(new Set(result.events.map((e) => e.session_id))).toEqual(new Set(["req-1"]));
+      expect(ofType(result.events, "agent_start")[1]!.parent_id).toBe("planner");
+      expect(traceViolations(result.events), describeTrace(result)).toEqual([]);
+    });
+
+    it("records a failing tool on its tool_result, once", () => {
+      const result = run("tool-error");
+      expect(shape(result.events), describeTrace(result)).toEqual(WORKFLOW("Agent", "broken"));
+      expect(traceViolations(result.events), describeTrace(result)).toEqual([]);
+      expect(ofType(result.events, "tool_result")[0]!.error).toMatch(/tool exploded/);
+      expect(ofType(result.events, "hook_completed").every((e) => e.outcome === "success")).toBe(true);
+      expect(ofType(result.events, "agent_end")[0]!.outcome).toBe("success");
+      expect(count(result.events, "error")).toBe(0);
+    });
+
+    /**
+     * Two requests in flight on ONE object built at startup (before
+     * instrument()), Paris slow and Rome fast, so Rome starts and ends inside
+     * Paris. Each must be its own session with its own run, and every event in
+     * a session must belong to that session's request.
+     */
+    const concurrent = (scenario: string, expected: string[], agentId: string) => {
+      const result = run(scenario);
+      expect(traceViolations(result.events), describeTrace(result)).toEqual([]);
+      const sessions = new Map<string, Event[]>();
+      for (const e of result.events) sessions.set(e.session_id, [...(sessions.get(e.session_id) ?? []), e]);
+      expect(sessions.size, describeTrace(result)).toBe(2);
+      const byCity = new Map<string, Event[]>();
+      for (const list of sessions.values()) {
+        expect(shape(list), describeTrace(result)).toEqual(expected);
+        const start = list[0]!;
+        expect(start.agent_id).toBe(agentId);
+        expect(start.parent_id ?? null).toBeNull();
+        const city = String(start.goal).includes("Rome") ? "Rome" : "Paris";
+        byCity.set(city, list);
+      }
+      expect([...byCity.keys()].sort()).toEqual(["Paris", "Rome"]);
+      for (const [city, list] of byCity) {
+        const other = city === "Paris" ? "Rome" : "Paris";
+        // Nothing of the other request in this session.
+        expect(JSON.stringify(list), describeTrace(result)).not.toContain(other);
+        for (const response of ofType(list, "model_response")) {
+          expect([response.input_tokens, response.output_tokens]).toEqual(city === "Paris" ? [11, 2] : [21, 3]);
+          expect(response.model).toBe("echo-1");
+        }
+        const end = ofType(list, "agent_end")[0]!;
+        expect(end.outcome).toBe("success");
+        expect(end.summary).toBe(`It is sunny in ${city}.`);
+      }
+      // They really did overlap: Rome started after Paris and ended before it.
+      const index = (city: string, type: string) => result.events.indexOf(ofType(byCity.get(city)!, type)[0]!);
+      expect(index("Rome", "agent_start")).toBeGreaterThan(index("Paris", "agent_start"));
+      expect(index("Rome", "agent_end")).toBeLessThan(index("Paris", "agent_end"));
+      return byCity;
+    };
+
+    it("keeps concurrent queries on one shared query engine in separate sessions", () => {
+      const byCity = concurrent(
+        "shared-query",
+        [
+          "RetrieverQueryEngine agent_start",
+          "RetrieverQueryEngine tool_use NotesRetriever",
+          "RetrieverQueryEngine tool_result NotesRetriever",
+          ...MODEL("RetrieverQueryEngine"),
+          "RetrieverQueryEngine agent_end",
+        ],
+        "RetrieverQueryEngine",
+      );
+      for (const [city, list] of byCity) {
+        expect(ofType(list, "tool_use")[0]!.input).toEqual({ query: `weather in ${city}?` });
+      }
+    });
+
+    it("keeps concurrent chats on one shared legacy LLMAgent in separate sessions", () => {
+      const byCity = concurrent(
+        "shared-legacy",
+        [
+          "LLMAgent agent_start",
+          ...MODEL("LLMAgent"),
+          "LLMAgent tool_use get_weather",
+          "LLMAgent tool_result get_weather",
+          ...MODEL("LLMAgent"),
+          "LLMAgent agent_end",
+        ],
+        "LLMAgent",
+      );
+      for (const [city, list] of byCity) {
+        expect(ofType(list, "tool_use")[0]!.tool_call_id).toBe(`call_${city}`);
+        expect(ofType(list, "tool_result")[0]!.output).toBe(`sunny in ${city}`);
+      }
+    });
+
+    it("keeps concurrent runs of one shared workflow agent in separate sessions", () => {
+      const byCity = concurrent("shared-workflow", WORKFLOW(), "Agent");
+      for (const [city, list] of byCity) {
+        expect(ofType(list, "tool_use")[0]!.tool_call_id).toBe(`call_${city}`);
+        expect(ofType(list, "tool_result")[0]!.output).toBe(`sunny in ${city}`);
+      }
+    });
+
+    it("records nothing after uninstrument()", () => {
+      const result = run("uninstrument");
+      expect(shape(result.events), describeTrace(result)).toEqual(WORKFLOW());
+      expect(result.stdout).toContain('"removed":["llamaindex"]');
+    });
+
+    // -- the coverage sweep ------------------------------------------------------
+    //
+    // Every commonly used surface, each against the Python adapter's golden
+    // trace for the same program (llama-index-core 0.14.24, mock LLM and
+    // embeddings): a top-level chat engine, query engine or retriever call is
+    // the session's root agent named after its class, and a retrieval is a
+    // `tool_use`/`tool_result` named after the retriever's class, its output
+    // summarised. (Python's MockLLM records each call twice, chat inside
+    // complete; that is the mock, not the mapping.)
+
+    /** The fewest checks every trace must pass: sound, one session, the framework's name. */
+    const sound = (result: ReturnType<typeof run>, sessions = 1, scopes: string[] = []) => {
+      expect(traceViolations(result.events), describeTrace(result)).toEqual([]);
+      expect(new Set(result.events.map((e) => e.session_id)).size, describeTrace(result)).toBe(sessions);
+      // The application's own agent() scope is not the framework's.
+      for (const event of result.events) {
+        if (!scopes.includes(event.agent_id)) expect(event.framework).toBe("llamaindex");
+      }
+    };
+    const RETRIEVAL = (agentId: string, retriever = "VectorIndexRetriever") => [
+      `${agentId} tool_use ${retriever}`,
+      `${agentId} tool_result ${retriever}`,
+    ];
+    /** The retrieval found the Paris note, summarised: count, score, text prefix. */
+    const retrievedParis = (event: Event) => {
+      const output = event.output as { num_nodes: number; top: Array<{ text: string; score: number }> };
+      expect(output.num_nodes).toBe(1);
+      expect(output.top[0]!.text).toBe("Paris is sunny.");
+      expect(typeof output.top[0]!.score).toBe("number");
+    };
+
+    describe.each(["chat-simple", "chat-simple-stream"])("SimpleChatEngine (%s)", (scenario) => {
+      it("is one run named after the engine, with the chat history in the model call", () => {
+        const result = run(scenario);
+        expect(shape(result.events), describeTrace(result)).toEqual([
+          "SimpleChatEngine agent_start",
+          ...MODEL("SimpleChatEngine"),
+          "SimpleChatEngine agent_end",
+        ]);
+        sound(result);
+        const request = ofType(result.events, "model_request")[0]!;
+        expect(request.model).toBe("scripted-1");
+        expect(request.messages).toEqual([
+          { role: "user", content: "hi" },
+          { role: "assistant", content: "hello" },
+          { role: "user", content: "weather in Paris?" },
+        ]);
+        const response = ofType(result.events, "model_response")[0]!;
+        expect([response.input_tokens, response.output_tokens]).toEqual([9, 4]);
+        expect(ofType(result.events, "agent_end")[0]).toMatchObject({ outcome: "success", summary: "It is sunny in Paris." });
+      });
+    });
+
+    describe.each(["chat-context", "chat-context-stream"])("ContextChatEngine (%s)", (scenario) => {
+      it("is one run: the retrieval, then the model call with the history and the context", () => {
+        const result = run(scenario);
+        expect(shape(result.events), describeTrace(result)).toEqual([
+          "ContextChatEngine agent_start",
+          ...RETRIEVAL("ContextChatEngine"),
+          ...MODEL("ContextChatEngine"),
+          "ContextChatEngine agent_end",
+        ]);
+        sound(result);
+        expect(ofType(result.events, "tool_use")[0]!.input).toEqual({ query: "weather in Paris?" });
+        retrievedParis(ofType(result.events, "tool_result")[0]!);
+        const messages = JSON.stringify(ofType(result.events, "model_request")[0]!.messages);
+        expect(messages).toContain("Paris is sunny.");
+        expect(messages).toContain("hello");
+        const response = ofType(result.events, "model_response")[0]!;
+        expect([response.input_tokens, response.output_tokens]).toEqual([9, 4]);
+        expect(ofType(result.events, "agent_end")[0]).toMatchObject({ outcome: "success", summary: "It is sunny in Paris." });
+      });
+    });
+
+    describe.each(["chat-condense", "chat-condense-stream"])("CondenseQuestionChatEngine (%s)", (scenario) => {
+      it("is one run: the condensing model call, the query's retrieval and its answer", () => {
+        const result = run(scenario);
+        expect(shape(result.events), describeTrace(result)).toEqual([
+          "CondenseQuestionChatEngine agent_start",
+          ...MODEL("CondenseQuestionChatEngine"),
+          ...RETRIEVAL("CondenseQuestionChatEngine"),
+          ...MODEL("CondenseQuestionChatEngine"),
+          "CondenseQuestionChatEngine agent_end",
+        ]);
+        sound(result);
+        // The history reaches the model through the condense prompt.
+        expect(JSON.stringify(ofType(result.events, "model_request")[0]!.messages)).toContain("hello");
+        expect(ofType(result.events, "tool_use")[0]!.input).toEqual({ query: "What is the weather in Paris?" });
+        retrievedParis(ofType(result.events, "tool_result")[0]!);
+        expect(ofType(result.events, "model_response").map((e) => [e.input_tokens, e.output_tokens])).toEqual([
+          [6, 3],
+          [9, 4],
+        ]);
+        expect(ofType(result.events, "agent_end")[0]).toMatchObject({ outcome: "success", summary: "It is sunny in Paris." });
+      });
+    });
+
+    it("records a retriever used directly as its own run", () => {
+      const result = run("retriever");
+      expect(shape(result.events), describeTrace(result)).toEqual([
+        "VectorIndexRetriever agent_start",
+        ...RETRIEVAL("VectorIndexRetriever"),
+        "VectorIndexRetriever agent_end",
+      ]);
+      sound(result);
+      expect(ofType(result.events, "tool_use")[0]!.input).toEqual({ query: "weather in Paris?" });
+      retrievedParis(ofType(result.events, "tool_result")[0]!);
+    });
+
+    describe.each(["index-query", "index-query-stream"])("VectorStoreIndex.asQueryEngine() (%s)", (scenario) => {
+      it("is one run named after the engine, with its retrieval and its answer", () => {
+        const result = run(scenario);
+        expect(shape(result.events), describeTrace(result)).toEqual([
+          "RetrieverQueryEngine agent_start",
+          ...RETRIEVAL("RetrieverQueryEngine"),
+          ...MODEL("RetrieverQueryEngine"),
+          "RetrieverQueryEngine agent_end",
+        ]);
+        sound(result);
+        expect(ofType(result.events, "agent_start")[0]!.goal).toBe("weather in Paris?");
+        retrievedParis(ofType(result.events, "tool_result")[0]!);
+        const response = ofType(result.events, "model_response")[0]!;
+        expect([response.input_tokens, response.output_tokens]).toEqual([9, 4]);
+        expect(ofType(result.events, "agent_end")[0]).toMatchObject({ outcome: "success", summary: "It is sunny in Paris." });
+      });
+    });
+
+    it("records nothing for building an index, with or without embeddings: true", () => {
+      // LlamaIndex.TS dispatches no embedding event on its bus, so there is
+      // nothing for `embeddings: true` to record; the option is accepted and
+      // changes nothing. (Python records embedding calls under it.)
+      expect(run("index-build").events).toEqual([]);
+      expect(run("index-build", { FAILPROOFAI_IT_OPTIONS: JSON.stringify({ embeddings: true }) }).events).toEqual([]);
+    });
+
+    /** workflow-core ≥1.1 exposes the hook plain workflows are recorded through; the floor's runtime does not. */
+    const plainWorkflows = fixture !== "llamaindex-0.11";
+
+    it(
+      plainWorkflows
+        ? "records a createWorkflow() workflow as a run with its steps as hooks"
+        : "leaves a createWorkflow() workflow's model call a run of its own on the floor's runtime",
+      () => {
+        const result = run("custom-workflow");
+        sound(result);
+        if (plainWorkflows) {
+          expect(shape(result.events), describeTrace(result)).toEqual([
+            "Workflow agent_start",
+            ...hook("Workflow", "research"),
+            ...hook("Workflow", "answer", MODEL("Workflow")),
+            "Workflow agent_end",
+          ]);
+          expect(ofType(result.events, "agent_start")[0]!.goal).toBe("weather in Paris?");
+          for (const event of ofType(result.events, "hook_triggered")) expect(event.trigger_event).toBe("workflow_step");
+          expect(ofType(result.events, "agent_end")[0]).toMatchObject({ outcome: "success", summary: "It is sunny in Paris." });
+        } else {
+          // @llama-flow/core binds its handler context in a closure: there is
+          // no run boundary and no step to observe, only the model call.
+          expect(shape(result.events), describeTrace(result)).toEqual([
+            "ScriptedLLM agent_start",
+            ...MODEL("ScriptedLLM"),
+            "ScriptedLLM agent_end",
+          ]);
+        }
+      },
+    );
+
+    it("nests a createWorkflow() workflow under an enclosing agent() scope", () => {
+      const result = run("custom-workflow-scoped");
+      sound(result, 1, ["forecast_flow"]);
+      const inner = plainWorkflows
+        ? ["Workflow agent_start", ...hook("Workflow", "research"), ...hook("Workflow", "answer", MODEL("Workflow")), "Workflow agent_end"]
+        : ["ScriptedLLM agent_start", ...MODEL("ScriptedLLM"), "ScriptedLLM agent_end"];
+      expect(shape(result.events), describeTrace(result)).toEqual(["forecast_flow agent_start", ...inner, "forecast_flow agent_end"]);
+      expect(ofType(result.events, "agent_start")[1]!.parent_id).toBe("forecast_flow");
+    });
+
+    it("records a three-agent handoff chain as nested agents under the workflow", () => {
+      const result = run("handoff3");
+      const handingOff = (agentId: string) => [
+        `${agentId} agent_start`,
+        ...hook(agentId, "setupAgent"),
+        `${agentId} hook_triggered runAgentStep`,
+        ...MODEL(agentId),
+        ...hook(agentId, "parseAgentOutput"),
+        `${agentId} hook_completed runAgentStep`,
+        ...hook(agentId, "executeToolCalls", [`${agentId} tool_use handOff`, `${agentId} tool_result handOff`]),
+        ...hook(agentId, "processToolResults"),
+        `${agentId} agent_end`,
+      ];
+      expect(shape(result.events), describeTrace(result)).toEqual([
+        "AgentWorkflow agent_start",
+        ...hook("AgentWorkflow", "handleInputStep"),
+        ...handingOff("triage"),
+        ...handingOff("researcher"),
+        "forecaster agent_start",
+        ...agentLoop("forecaster", "get_weather"),
+        "forecaster agent_end",
+        "AgentWorkflow agent_end",
+      ]);
+      sound(result);
+      for (const start of ofType(result.events, "agent_start").slice(1)) expect(start.parent_id).toBe("AgentWorkflow");
+      expect(ofType(result.events, "tool_use").map((e) => e.tool_call_id)).toEqual(["call_h1", "call_h2", "call_1"]);
+      expect(ofType(result.events, "agent_end").at(-1)!.summary).toBe("It is sunny in Paris.");
+    });
+
+    // `responseFormat` arrived after the floor (@llamaindex/workflow 1.1.5 ignores it).
+    it.skipIf(fixture === "llamaindex-0.11")("records agent() with responseFormat: the structured-output call and tool", () => {
+      const result = run("structured");
+      expect(shape(result.events), describeTrace(result)).toEqual([
+        "Agent agent_start",
+        ...hook("Agent", "handleInputStep"),
+        ...hook("Agent", "setupAgent"),
+        "Agent hook_triggered runAgentStep",
+        ...MODEL("Agent"),
+        "Agent hook_triggered parseAgentOutput",
+        "Agent hook_completed runAgentStep",
+        ...MODEL("Agent"),
+        "Agent tool_use format_output",
+        "Agent tool_result format_output",
+        "Agent hook_completed parseAgentOutput",
+        "Agent agent_end",
+      ]);
+      sound(result);
+      expect(ofType(result.events, "tool_result")[0]!.output).toEqual({ city: "Paris", sky: "sunny" });
+      expect(result.stdout).toContain('"object":{"city":"Paris","sky":"sunny"}');
+    });
+
+    it("records a FunctionTool.from() tool like a tool() one", () => {
+      const result = run("function-tool");
+      expect(shape(result.events), describeTrace(result)).toEqual(WORKFLOW("Agent", "lookup_weather"));
+      sound(result);
+      expect(ofType(result.events, "tool_use")[0]).toMatchObject({ tool_call_id: "call_1", input: { city: "Paris" } });
+      expect(ofType(result.events, "tool_result")[0]!.output).toBe("sunny in Paris");
+    });
+
+    it("records a QueryEngineTool with the query's retrieval and model call inside it", () => {
+      const result = run("query-engine-tool");
+      expect(shape(result.events), describeTrace(result)).toEqual([
+        "Agent agent_start",
+        ...hook("Agent", "handleInputStep"),
+        ...hook("Agent", "setupAgent"),
+        "Agent hook_triggered runAgentStep",
+        ...MODEL("Agent"),
+        ...hook("Agent", "parseAgentOutput"),
+        "Agent hook_completed runAgentStep",
+        ...hook("Agent", "executeToolCalls", [
+          "Agent tool_use city_notes",
+          ...RETRIEVAL("Agent"),
+          ...MODEL("Agent"),
+          "Agent tool_result city_notes",
+        ]),
+        ...hook("Agent", "processToolResults"),
+        ...hook("Agent", "setupAgent"),
+        "Agent hook_triggered runAgentStep",
+        ...MODEL("Agent"),
+        "Agent hook_triggered parseAgentOutput",
+        "Agent hook_completed runAgentStep",
+        "Agent hook_completed parseAgentOutput",
+        "Agent agent_end",
+      ]);
+      sound(result);
+      expect(ofType(result.events, "tool_result").at(-1)!.output).toEqual({ content: "Paris is sunny." });
+      expect(ofType(result.events, "model_response").map((e) => [e.input_tokens, e.output_tokens])).toEqual([
+        [12, 5],
+        [7, 3],
+        [30, 7],
+      ]);
+    });
+
+    it("records parallel tool calls from one model turn, each on its own id", () => {
+      const result = run("parallel-tools");
+      expect(shape(result.events), describeTrace(result)).toEqual([
+        "Agent agent_start",
+        ...hook("Agent", "handleInputStep"),
+        ...hook("Agent", "setupAgent"),
+        "Agent hook_triggered runAgentStep",
+        ...MODEL("Agent"),
+        ...hook("Agent", "parseAgentOutput"),
+        "Agent hook_completed runAgentStep",
+        ...hook("Agent", "executeToolCalls", [
+          "Agent tool_use get_weather",
+          "Agent tool_result get_weather",
+          "Agent tool_use get_time",
+          "Agent tool_result get_time",
+        ]),
+        ...hook("Agent", "processToolResults"),
+        ...hook("Agent", "setupAgent"),
+        "Agent hook_triggered runAgentStep",
+        ...MODEL("Agent"),
+        "Agent hook_triggered parseAgentOutput",
+        "Agent hook_completed runAgentStep",
+        "Agent hook_completed parseAgentOutput",
+        "Agent agent_end",
+      ]);
+      sound(result);
+      expect(ofType(result.events, "tool_result").map((e) => [e.tool_call_id, e.output])).toEqual([
+        ["call_1", "sunny in Paris"],
+        ["call_2", "noon in Paris"],
+      ]);
+    });
+
+    it("closes the run as failed when the provider fails mid-stream (an unhandled rejection in LlamaIndex)", () => {
+      // The workflow runtime lets this error escape as an unhandled rejection
+      // and `run()` never settles — LlamaIndex's behaviour, with or without
+      // the SDK. The failed step is the signal: the run still closes, failed.
+      const result = run("stream-error");
+      expect(shape(result.events), describeTrace(result)).toEqual([
+        "Agent agent_start",
+        ...hook("Agent", "handleInputStep"),
+        ...hook("Agent", "setupAgent"),
+        "Agent hook_triggered runAgentStep",
+        ...MODEL("Agent"),
+        "Agent hook_completed runAgentStep",
+        "Agent agent_end",
+      ]);
+      sound(result);
+      expect(result.stdout).toContain('"settled":"pending"');
+      expect(result.stdout).toContain('"unhandled":["Error: provider dropped the stream"]');
+      expect(ofType(result.events, "model_response")[0]!.error).toMatch(/provider dropped the stream/);
+      expect(ofType(result.events, "hook_completed").at(-1)).toMatchObject({ outcome: "failed" });
+      expect(ofType(result.events, "agent_end")[0]).toMatchObject({ outcome: "failed" });
+      expect(count(result.events, "error")).toBe(0);
+    });
+
+    /**
+     * Ten requests in flight at once on ONE object: each its own session, and
+     * nothing of any other request in it. CityLLM's usage is per city
+     * (`[100 + n, n]`), so every model response says which request it answered.
+     */
+    const CITIES = ["Paris", "Rome", "Oslo", "Lima", "Cairo", "Delhi", "Tokyo", "Quito", "Accra", "Hanoi"];
+    const tenWay = (scenario: string, expected: (agentId: string) => string[], agentId: string) => {
+      const result = run(scenario);
+      sound(result, CITIES.length);
+      const byCity = new Map<string, Event[]>();
+      for (const e of result.events) {
+        const list = [...result.events.filter((x) => x.session_id === e.session_id)];
+        const text = JSON.stringify(list);
+        const city = CITIES.find((c) => text.includes(`weather in ${c}?`));
+        expect(city, describeTrace(result)).toBeDefined();
+        byCity.set(city!, list);
+      }
+      expect([...byCity.keys()].sort()).toEqual([...CITIES].sort());
+      for (const [city, list] of byCity) {
+        expect(shape(list), describeTrace(result)).toEqual(expected(agentId));
+        expect(list[0]!.agent_id).toBe(agentId);
+        expect(list[0]!.parent_id ?? null).toBeNull();
+        const text = JSON.stringify(list);
+        for (const other of CITIES) if (other !== city) expect(text, `${city}'s session mentions ${other}`).not.toContain(other);
+        const n = CITIES.indexOf(city);
+        for (const response of ofType(list, "model_response")) {
+          expect([response.input_tokens, response.output_tokens]).toEqual([100 + n, n]);
+          expect(response.model).toBe("city-1");
+        }
+        expect(ofType(list, "agent_end")[0]).toMatchObject({ outcome: "success", summary: `It is sunny in ${city}.` });
+      }
+      return byCity;
+    };
+
+    it("keeps 10 concurrent run()s of one agent object apart", () => {
+      const byCity = tenWay("concurrent-agents", (id) => WORKFLOW(id), "Agent");
+      for (const [city, list] of byCity) {
+        expect(ofType(list, "tool_use")[0]!.tool_call_id).toBe(`call_${city}`);
+        expect(ofType(list, "tool_result")[0]!.output).toBe(`sunny in ${city}`);
+      }
+    });
+
+    it("keeps 10 concurrent queries on one shared query engine apart", () => {
+      const byCity = tenWay(
+        "concurrent-queries",
+        (id) => [`${id} agent_start`, ...RETRIEVAL(id), ...MODEL(id), `${id} agent_end`],
+        "RetrieverQueryEngine",
+      );
+      for (const [city, list] of byCity) {
+        expect(ofType(list, "tool_use")[0]!.input).toEqual({ query: `weather in ${city}?` });
+        expect((ofType(list, "tool_result")[0]!.output as { top: Array<{ text: string }> }).top[0]!.text).toBe(
+          `${city} is sunny.`,
+        );
+      }
+    });
+
+    it("keeps 10 concurrent chats on one shared chat engine apart", () => {
+      const byCity = tenWay(
+        "concurrent-chats",
+        (id) => [`${id} agent_start`, ...RETRIEVAL(id), ...MODEL(id), `${id} agent_end`],
+        "ContextChatEngine",
+      );
+      for (const [city, list] of byCity) {
+        expect(ofType(list, "tool_use")[0]!.input).toEqual({ query: `weather in ${city}?` });
+      }
+    });
+  });
+});