@selesai/code 0.13.29 → 0.13.30

package/dist/extensions/capability-gateway/integration.test.ts

@@ -2,16 +2,19 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { fileURLToPath } from "node:url";
-import { afterEach, describe, expect, it } from "vitest";
+import { afterEach, describe, expect, it, vi } from "vitest";
 import { fauxProvider } from "@earendil-works/pi-ai/providers/faux";
 import {
 	createAgentSession,
+	createEventBus,
 	DefaultResourceLoader,
 	ModelRuntime,
 	SessionManager,
 	SettingsManager,
 	type AgentSession,
+	type EventBus,
 } from "@selesai/code";
+import { allowNetwork } from "../../../test/test-network-env.ts";
 
 const EXTENSIONS_DIR = fileURLToPath(new URL("../../", import.meta.url));
 const GATEWAY_DIR = fileURLToPath(new URL(".", import.meta.url));
@@ -21,14 +24,34 @@ const INLINE_SKILLS_FILE = fileURLToPath(new URL("../inline-skills.ts", import.m
 
 interface Harness {
 	session: AgentSession;
+	/** Everything the gateway emitted on its telemetry channel. */
+	telemetry: Array<Record<string, unknown>>;
 	dispose: () => Promise<void>;
 }
 
-async function createGatewaySession(options: {
+interface HarnessOptions {
 	enabled: boolean;
 	withSkill?: boolean;
 	extensions?: string[];
-}): Promise<Harness> {
+	/** Written to `capabilityGateway.routing.jev` in the session's settings.json. */
+	jev?: Record<string, unknown>;
+	/** Make the gateway's telemetry channel throw on every emit. */
+	telemetryDown?: boolean;
+}
+
+/** A bus that fails only on the gateway's own telemetry channel. */
+function telemetryDownBus(): EventBus {
+	const bus = createEventBus();
+	return {
+		emit: (channel, data) => {
+			if (channel === "capability-gateway") throw new Error("telemetry channel down");
+			bus.emit(channel, data);
+		},
+		on: bus.on,
+	};
+}
+
+async function createGatewaySession(options: HarnessOptions): Promise<Harness> {
 	const extensionPaths = options.extensions ?? [EXTENSIONS_DIR];
 	const cwd = mkdtempSync(join(tmpdir(), "gw-cwd-"));
 	const home = mkdtempSync(join(tmpdir(), "gw-home-"));
@@ -45,6 +68,13 @@ async function createGatewaySession(options: {
 	if (options.enabled) delete process.env.SELESAI_CAPABILITY_GATEWAY;
 	else process.env.SELESAI_CAPABILITY_GATEWAY = "0";
 
+	if (options.jev) {
+		writeFileSync(
+			join(home, "settings.json"),
+			JSON.stringify({ capabilityGateway: { routing: { jev: options.jev } } }),
+		);
+	}
+
 	if (options.withSkill) {
 		mkdirSync(join(home, "skills", "research"), { recursive: true });
 		writeFileSync(
@@ -71,11 +101,20 @@ async function createGatewaySession(options: {
 	if (!model) throw new Error("faux model not registered");
 
 	const settingsManager = SettingsManager.inMemory({ compaction: { enabled: false }, retry: { enabled: false } });
+	// Subscribe to the gateway's telemetry channel before the extension can emit.
+	const eventBus = options.telemetryDown ? telemetryDownBus() : createEventBus();
+	const telemetry: Array<Record<string, unknown>> = [];
+	if (!options.telemetryDown) {
+		eventBus.on("capability-gateway", (data) => {
+			telemetry.push(data as Record<string, unknown>);
+		});
+	}
 	const loader = new DefaultResourceLoader({
 		cwd,
 		agentDir: home,
 		settingsManager,
 		additionalExtensionPaths: extensionPaths,
+		eventBus,
 		noPromptTemplates: true,
 		noThemes: true,
 		noContextFiles: true,
@@ -114,12 +153,13 @@ async function createGatewaySession(options: {
 		rmSync(home, { recursive: true, force: true });
 	};
 
-	return { session, dispose };
+	return { session, telemetry, dispose };
 }
 
 const harnesses: Harness[] = [];
 afterEach(async () => {
 	for (const h of harnesses.splice(0)) await h.dispose();
+	vi.unstubAllGlobals();
 });
 
 describe("capability gateway integration", () => {
@@ -236,7 +276,7 @@ describe("capability gateway integration", () => {
 			h.session.systemPrompt,
 			{ cwd: process.cwd() } as never,
 		);
-		expect(result).toBeUndefined();
+		expect(result.messages).toEqual([]);
 		expect(h.session.getActiveToolNames()).toContain("grep_app_search");
 	});
 
@@ -253,7 +293,7 @@ describe("capability gateway integration", () => {
 			h.session.systemPrompt,
 			{ cwd: process.cwd() } as never,
 		);
-		expect(result).toBeUndefined();
+		expect(result.messages).toEqual([]);
 		expect(h.session.systemPrompt).not.toContain("Research instructions body.");
 	});
 
@@ -266,7 +306,7 @@ describe("capability gateway integration", () => {
 			h.session.systemPrompt,
 			{ cwd: process.cwd() } as never,
 		);
-		expect(result).toBeUndefined();
+		expect(result.messages).toEqual([]);
 	});
 
 	it("does not recommend capability_skill_show for an inline-loaded $skill", async () => {
@@ -288,6 +328,352 @@ describe("capability gateway integration", () => {
 			h.session.systemPrompt,
 			{ cwd: process.cwd() } as never,
 		);
-		expect(result).toBeUndefined();
+		expect(result.messages).toEqual([]);
+	});
+});
+
+// ---------------------------------------------------------------------------
+// Jev-assisted routing (opt-in): capabilityGateway.routing.jev.
+// ---------------------------------------------------------------------------
+
+describe("capability gateway Jev routing", () => {
+	// "...github..." only weakly suggests both grep-app tools: the deterministic router returns a
+	// two-candidate hint, which is the only Jev trigger.
+	const HINT_PROMPT = "look at this github repo";
+	// No tool name, alias, or summary token matches: the router has no lexical signal and Jev
+	// must never be consulted.
+	const NO_SIGNAL_PROMPT = "continue where we left off last time";
+
+	const jevSettings = (overrides: Record<string, unknown> = {}): Record<string, unknown> => ({
+		enabled: true,
+		provider: "faux-gw",
+		baseUrl: "http://jev.test/v1",
+		minConfidence: 0.5,
+		timeoutMs: 2_000,
+		...overrides,
+	});
+
+	function jevSseResponse(choice: string, confidence: number): Response {
+		const body = JSON.stringify({ answers: { capability: { choice, confidence } } });
+		const chunk = (delta: unknown, finish: string | null) =>
+			`data: ${JSON.stringify({
+				id: "1",
+				object: "chat.completion.chunk",
+				created: 0,
+				model: "jev-1.13",
+				choices: [{ index: 0, delta, finish_reason: finish }],
+			})}\n\n`;
+		return new Response(chunk({ role: "assistant", content: body }, null) + chunk({}, "stop") + "data: [DONE]\n\n", {
+			status: 200,
+			headers: { "content-type": "text/event-stream" },
+		});
+	}
+
+	function stubJev(choice: string, confidence = 0.95): { requests: string[]; fetchMock: ReturnType<typeof vi.fn> } {
+		const requests: string[] = [];
+		const fetchMock = vi.fn(async (_url: unknown, init?: { body?: unknown }) => {
+			requests.push(typeof init?.body === "string" ? init.body : "");
+			return jevSseResponse(choice, confidence);
+		});
+		vi.stubGlobal("fetch", fetchMock);
+		return { requests, fetchMock };
+	}
+
+	async function route(h: Harness, prompt: string): Promise<void> {
+		const result = await h.session.extensionRunner.emitBeforeAgentStart(
+			prompt,
+			undefined,
+			h.session.systemPrompt,
+			{ cwd: process.cwd() } as never,
+		);
+		expect(result.messages).toEqual([]);
+	}
+
+	function routeEvents(h: Harness): Array<Record<string, unknown>> {
+		return h.telemetry.filter((event) => event.event === "route");
+	}
+
+	// Telemetry may carry route metadata and canonical tool names, never prompt
+	// text, conversation turns, credentials, raw Jev output, or tool arguments.
+	const TELEMETRY_KEYS = new Set([
+		"event",
+		"source",
+		"outcome",
+		"reason",
+		"tool",
+		"confidence",
+		"candidates",
+		"durationMs",
+		"baselineCount",
+		"dormantCount",
+		"activeCount",
+		"activated",
+		"used",
+		"results",
+		"skill",
+	]);
+
+	it("activates the tool Jev selects for the run, then restores the baseline", async () => {
+		allowNetwork();
+		const { requests } = stubJev("grep_app_search", 0.91);
+		const h = await createGatewaySession({
+			enabled: true,
+			extensions: [GATEWAY_DIR, GREP_APP_DIR, GRAFT_DIR],
+			jev: jevSettings(),
+		});
+		harnesses.push(h);
+		const baseline = h.session.getActiveToolNames();
+		expect(baseline).not.toContain("grep_app_search");
+
+		await route(h, HINT_PROMPT);
+
+		expect(h.session.getActiveToolNames()).toContain("grep_app_search");
+		// Built-ins and always-active code-context tools stay active.
+		for (const name of ["read", "bash", "capability_catalog", "graft_find_code"]) {
+			expect(h.session.getActiveToolNames()).toContain(name);
+		}
+		expect(routeEvents(h)).toContainEqual(
+			expect.objectContaining({ source: "jev", outcome: "attempt", candidates: 2 }),
+		);
+		expect(routeEvents(h)).toContainEqual(
+			expect.objectContaining({
+				source: "jev",
+				outcome: "activated",
+				tool: "grep_app_search",
+				confidence: "high",
+				candidates: 2,
+				durationMs: expect.any(Number),
+			}),
+		);
+		expect(requests).toHaveLength(1);
+
+		// The activation was temporary: settlement restores the baseline set.
+		await h.session.extensionRunner.emit({ type: "agent_settled" });
+		expect(h.session.getActiveToolNames()).toEqual(baseline);
+		expect(h.telemetry).toContainEqual(
+			expect.objectContaining({ event: "reset", activated: 1, used: 0 }),
+		);
+	});
+
+	it("exposes the selected schema for the same turn and keeps the other hinted candidate dormant", async () => {
+		allowNetwork();
+		stubJev("grep_app_search", 0.9);
+		const h = await createGatewaySession({
+			enabled: true,
+			extensions: [GATEWAY_DIR, GREP_APP_DIR],
+			jev: jevSettings(),
+		});
+		harnesses.push(h);
+
+		await route(h, HINT_PROMPT);
+
+		// The live loadout is what the resulting turn uses, and the selected tool's
+		// schema is registered for it.
+		expect(h.session.getActiveToolNames()).toContain("grep_app_search");
+		const definition = h.session.getToolDefinition("grep_app_search");
+		expect(definition).toBeDefined();
+		expect(definition?.parameters).toBeDefined();
+		// The other hinted candidate (and every nonselected tool) stays dormant.
+		expect(h.session.getActiveToolNames()).not.toContain("grep_app_fetch");
+	});
+
+	it("does not call Jev without a lexical tool signal or for a skill-only match", async () => {
+		allowNetwork();
+		const { fetchMock } = stubJev("grep_app_search");
+		const h = await createGatewaySession({
+			enabled: true,
+			withSkill: true,
+			extensions: [GATEWAY_DIR, GREP_APP_DIR],
+			jev: jevSettings(),
+		});
+		harnesses.push(h);
+
+		await route(h, NO_SIGNAL_PROMPT);
+		expect(fetchMock).not.toHaveBeenCalled();
+		expect(routeEvents(h)).toEqual([]);
+		expect(h.session.getActiveToolNames()).not.toContain("grep_app_search");
+
+		// A unique skill match is a recommendation, never a Jev auto-selection.
+		await route(h, "do research on this topic");
+		expect(fetchMock).not.toHaveBeenCalled();
+		expect(routeEvents(h)).toEqual([]);
+		expect(h.session.getActiveToolNames()).not.toContain("research");
+	});
+
+	it("does not call Jev when the deterministic router already activated a tool", async () => {
+		allowNetwork();
+		const deterministic = stubJev("grep_app_search");
+		const h = await createGatewaySession({
+			enabled: true,
+			extensions: [GATEWAY_DIR, GREP_APP_DIR],
+			jev: jevSettings(),
+		});
+		harnesses.push(h);
+
+		await route(h, "search github code with grep_app_search");
+		expect(h.session.getActiveToolNames()).toContain("grep_app_search");
+		expect(deterministic.fetchMock).not.toHaveBeenCalled();
+		expect(routeEvents(h)).toContainEqual(
+			expect.objectContaining({ source: "deterministic", outcome: "activated", tool: "grep_app_search" }),
+		);
+	});
+
+	it("leaves routing deterministic when the Jev route is not configured", async () => {
+		allowNetwork();
+		const disabled = stubJev("grep_app_search");
+		const h = await createGatewaySession({ enabled: true, extensions: [GATEWAY_DIR, GREP_APP_DIR] });
+		harnesses.push(h);
+
+		await route(h, HINT_PROMPT);
+
+		expect(disabled.fetchMock).not.toHaveBeenCalled();
+		expect(h.session.getActiveToolNames()).not.toContain("grep_app_search");
+		expect(routeEvents(h)).toEqual([]);
+	});
+
+	it("offers Jev only the current prompt and the hinted candidates, never the full catalog", async () => {
+		allowNetwork();
+		const { requests } = stubJev("none", 1);
+		const h = await createGatewaySession({
+			enabled: true,
+			withSkill: true,
+			extensions: [GATEWAY_DIR, GREP_APP_DIR, GRAFT_DIR],
+			jev: jevSettings(),
+		});
+		harnesses.push(h);
+
+		await route(h, HINT_PROMPT);
+
+		const sent = JSON.parse(requests[0]!) as { messages: Array<{ content: string }> };
+		const payload = JSON.parse(sent.messages[0]!.content) as {
+			state: { conversation: unknown[]; system_prompt?: string };
+			questions: Record<string, { criteria: Record<string, string> }>;
+		};
+		const criteria = Object.keys(payload.questions.capability!.criteria);
+		expect(criteria).toEqual(["none", "grep_app_search", "grep_app_fetch"]);
+		// Built-ins, gateway tools, always-active Graft tools, and skills are not selectable.
+		for (const name of ["read", "capability_catalog", "capability_skill_show", "graft_find_code", "research"]) {
+			expect(criteria).not.toContain(name);
+		}
+		// Only the current prompt travels; never conversation history or a system prompt.
+		expect(payload.state.conversation).toEqual([{ role: "user", text: HINT_PROMPT }]);
+		expect(payload.state.system_prompt).toBeUndefined();
+	});
+
+	it("abstains on `none` and stays non-blocking when Jev is unavailable", async () => {
+		allowNetwork();
+		const { requests } = stubJev("none", 1);
+		const h = await createGatewaySession({
+			enabled: true,
+			extensions: [GATEWAY_DIR, GREP_APP_DIR],
+			jev: jevSettings(),
+		});
+		harnesses.push(h);
+
+		await route(h, HINT_PROMPT);
+		expect(h.session.getActiveToolNames()).not.toContain("grep_app_search");
+		expect(routeEvents(h)).toContainEqual(
+			expect.objectContaining({ source: "jev", outcome: "abstained", reason: "none", candidates: 2 }),
+		);
+		expect(requests).toHaveLength(1);
+
+		// A provider outage is an ordinary fallback: nothing activates, and the
+		// telemetry reports the route without recording any content.
+		vi.stubGlobal(
+			"fetch",
+			vi.fn(async () => {
+				throw new Error("connection reset");
+			}),
+		);
+		await route(h, HINT_PROMPT);
+		expect(h.session.getActiveToolNames()).not.toContain("grep_app_search");
+		expect(routeEvents(h)).toContainEqual(
+			expect.objectContaining({ source: "jev", outcome: "unavailable", reason: "transport" }),
+		);
+
+		const serialized = JSON.stringify(h.telemetry);
+		expect(serialized).not.toContain(HINT_PROMPT);
+		expect(serialized).not.toContain(NO_SIGNAL_PROMPT);
+		expect(serialized).not.toContain("answers");
+		expect(serialized).not.toContain("faux");
+		for (const event of h.telemetry) {
+			for (const key of Object.keys(event)) expect(TELEMETRY_KEYS.has(key)).toBe(true);
+		}
+	});
+
+	it("is invisible without a Jev provider template or subscription", async () => {
+		allowNetwork();
+		const { fetchMock } = stubJev("grep_app_search");
+		const h = await createGatewaySession({
+			enabled: true,
+			extensions: [GATEWAY_DIR, GREP_APP_DIR],
+			jev: jevSettings({ provider: "no-such-provider", baseUrl: undefined }),
+		});
+		harnesses.push(h);
+
+		await route(h, HINT_PROMPT);
+
+		expect(fetchMock).not.toHaveBeenCalled();
+		expect(h.session.getActiveToolNames()).not.toContain("grep_app_search");
+		expect(routeEvents(h)).toContainEqual(
+			expect.objectContaining({ source: "jev", outcome: "unavailable", reason: "no-template" }),
+		);
+	});
+
+	it("records whether an activated tool was actually used, then resets", async () => {
+		allowNetwork();
+		stubJev("grep_app_search", 0.9);
+		const h = await createGatewaySession({
+			enabled: true,
+			extensions: [GATEWAY_DIR, GREP_APP_DIR],
+			jev: jevSettings(),
+		});
+		harnesses.push(h);
+
+		await route(h, HINT_PROMPT);
+		expect(h.session.getActiveToolNames()).toContain("grep_app_search");
+
+		// An unrelated tool produces no use telemetry.
+		await h.session.extensionRunner.emit({
+			type: "tool_execution_start",
+			toolCallId: "t0",
+			toolName: "read",
+			args: {},
+		});
+		expect(h.telemetry.filter((event) => event.event === "use")).toEqual([]);
+
+		// The selected tool is called (twice): one use event per activation.
+		for (const toolCallId of ["t1", "t2"]) {
+			await h.session.extensionRunner.emit({
+				type: "tool_execution_start",
+				toolCallId,
+				toolName: "grep_app_search",
+				args: {},
+			});
+		}
+		const uses = h.telemetry.filter((event) => event.event === "use");
+		expect(uses).toEqual([{ event: "use", tool: "grep_app_search", source: "jev" }]);
+
+		await h.session.extensionRunner.emit({ type: "agent_settled" });
+		expect(h.telemetry).toContainEqual(
+			expect.objectContaining({ event: "reset", activated: 1, used: 1 }),
+		);
+	});
+
+	it("completes the run when telemetry cannot be emitted", async () => {
+		allowNetwork();
+		stubJev("grep_app_search", 0.95);
+		const h = await createGatewaySession({
+			enabled: true,
+			extensions: [GATEWAY_DIR, GREP_APP_DIR],
+			jev: jevSettings(),
+			telemetryDown: true,
+		});
+		harnesses.push(h);
+
+		await route(h, HINT_PROMPT);
+
+		expect(h.telemetry).toEqual([]);
+		expect(h.session.getActiveToolNames()).toContain("grep_app_search");
 	});
 });