@yibie/pi-jev-browser 0.1.0

package/src/types.ts

@@ -0,0 +1,79 @@
+import type { Browser, BrowserContext, Page, Video } from "playwright";
+
+export type PolicyName = "pi" | "typesafe";
+
+export interface JevBrowserConfig {
+	policy: PolicyName;
+	allowedOrigins: string[];
+	headless: boolean;
+	recordVideo: boolean;
+	showCursor: boolean;
+	showClickIndicators: boolean;
+	outputDir: string;
+	viewport: { width: number; height: number };
+	stream: { enabled: boolean; intervalMs: number };
+}
+
+export type BrowserAction =
+	| {
+			type: "click" | "double_click";
+			x: number;
+			y: number;
+			button?: "left" | "right" | "wheel";
+			keys?: string[];
+	  }
+	| { type: "scroll"; x?: number; y?: number; deltaX: number; deltaY: number }
+	| { type: "type"; text: string }
+	| { type: "wait"; ms?: number }
+	| { type: "keypress"; keys: string[] }
+	| {
+			type: "drag";
+			path: Array<{ x: number; y: number } | [number, number]>;
+			button?: "left" | "right" | "wheel";
+	  }
+	| { type: "move"; x: number; y: number }
+	| { type: "screenshot" }
+	| { type: "navigate"; url: string }
+	| { type: "back" | "forward" | "reload" };
+
+export interface BrowserLogEntry {
+	id: number;
+	timestamp: string;
+	type:
+		| "console"
+		| "pageerror"
+		| "requestfailed"
+		| "download"
+		| "navigation"
+		| "security";
+	level: string;
+	text: string;
+	url?: string;
+}
+
+export interface BrowserState {
+	active: boolean;
+	currentUrl?: string;
+	pageTitle?: string;
+	pages: Array<{ index: number; title: string; url: string }>;
+	startedAt?: string;
+	viewport: { width: number; height: number };
+}
+
+export interface ActiveBrowserSession {
+	browser: Browser;
+	context: BrowserContext;
+	page: Page;
+	video?: Video;
+	id: string;
+	outputDir: string;
+	startedAt: string;
+	logs: BrowserLogEntry[];
+	nextLogId: number;
+	stream?: StreamController;
+}
+
+export interface StreamController {
+	url: string;
+	stop(): Promise<void>;
+}

package/src/typesafe.ts

@@ -0,0 +1,137 @@
+import type { Observation, ObservedTarget } from "./jev-browser.ts";
+import { buildQuestions, type Decision, type JevPolicy, parseText } from "./jev-model.ts";
+import { buildTextPrompt, type ModelCall, TEXT_SYSTEM } from "./pi-model.ts";
+
+export const TYPESAFE_ENDPOINT = "https://api.typesafe.ai/v1/systemone";
+export const DEFAULT_TYPESAFE_MODEL = "jev-latest";
+
+export interface TypesafeChoice {
+	choice: string;
+	probability?: number;
+	confidence?: number;
+}
+
+/**
+ * The TypeSafe API returns a choice, a probability for every offered option, and
+ * a confidence for the selected one. Only the choice is load-bearing: an answer
+ * that names no offered option tells the loop nothing, so that is an error. A
+ * distribution the loop cannot fully trust must never kill a run that has
+already taken actions, so anything else about it degrades to "probability
+ * unknown" instead.
+ */
+export function parseChoiceAnswer(
+	value: unknown,
+	valid: readonly string[],
+): TypesafeChoice {
+	const answers = (value as { answers?: unknown } | null | undefined)?.answers;
+	if (!answers || typeof answers !== "object")
+		throw new Error("TypeSafe returned no answers.");
+	const action = (answers as Record<string, unknown>).action;
+	if (!action || typeof action !== "object")
+		throw new Error("TypeSafe returned no answer for the action question.");
+	const { type, choice, probabilities, confidence } = action as {
+		type?: unknown;
+		choice?: unknown;
+		probabilities?: unknown;
+		confidence?: unknown;
+	};
+	if (type !== "choice")
+		throw new Error(
+			`TypeSafe answered the action question with type ${describe(type)}.`,
+		);
+	if (typeof choice !== "string" || !valid.includes(choice))
+		throw new Error(
+			`TypeSafe selected an unoffered option (${describe(choice)}). Offered: ${valid.slice(0, 10).join(", ")}${valid.length > 10 ? ", …" : ""}.`,
+		);
+	return {
+		choice,
+		probability: reportedProbability(probabilities, choice),
+		confidence:
+			typeof confidence === "number" && Number.isFinite(confidence)
+				? confidence
+				: undefined,
+	};
+}
+
+/** The selected option's own probability, if it is a usable number. */
+function reportedProbability(value: unknown, choice: string): number | undefined {
+	if (!value || typeof value !== "object" || Array.isArray(value))
+		return undefined;
+	const reported = (value as Record<string, unknown>)[choice];
+	return typeof reported === "number" &&
+		Number.isFinite(reported) &&
+		reported >= 0 &&
+		reported <= 1
+		? reported
+		: undefined;
+}
+
+function describe(value: unknown) {
+	return (JSON.stringify(value) ?? String(value)).slice(0, 60);
+}
+
+/**
+ * Direct TypeSafe transport: no gateway, no second vendor. Jev generates no
+ * text, so field values still come from a model — the one pi has configured. */
+export function createTypesafePolicy(options: {
+	apiKey: string;
+	model?: string;
+	text: ModelCall;
+	fetchImpl?: typeof fetch;
+}): JevPolicy {
+	const model = options.model ?? DEFAULT_TYPESAFE_MODEL;
+	const request = options.fetchImpl ?? fetch;
+	return {
+		async choose(observation, goal, history, signal): Promise<Decision> {
+			const questions = buildQuestions(observation, goal);
+			const response = await request(TYPESAFE_ENDPOINT, {
+				method: "POST",
+				headers: {
+					authorization: `Bearer ${options.apiKey}`,
+					"content-type": "application/json",
+				},
+				body: JSON.stringify({
+					state: JSON.stringify({
+						page: observation,
+						recentActions: history.slice(-10),
+					}),
+					model,
+					questions,
+				}),
+				signal,
+			});
+			if (!response.ok) {
+				// Keep the body out of the message only when it is unreadable; the
+				// status code is what the loop classifies.
+				const detail = await response.text().catch(() => "");
+				throw Object.assign(
+					new Error(
+						`TypeSafe request failed with ${response.status}${detail ? `: ${detail.slice(0, 200)}` : ""}`,
+					),
+					{ statusCode: response.status },
+				);
+			}
+			const { choice, probability, confidence } = parseChoiceAnswer(
+				await response.json(),
+				Object.keys(questions.action.criteria),
+			);
+			const target: ObservedTarget | undefined = observation.targets.find(
+				(entry) => `${entry.operation}:${entry.id}` === choice,
+			);
+			return {
+				operation: target?.operation ?? choice,
+				target,
+				probability,
+				providerConfidence: confidence,
+			};
+		},
+		async text(observation, goal, target, history, signal) {
+			const answer = await options.text({
+				system: TEXT_SYSTEM,
+				prompt: buildTextPrompt(observation, goal, target, history),
+				signal,
+			});
+			return parseText(answer);
+		},
+	};
+}

package/test/browser-setup.test.ts

@@ -0,0 +1,32 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { createBrowserSetup } from "../src/browser-setup.ts";
+
+test("browser setup shares installation and caches success", async () => {
+	let calls = 0;
+	let finish!: () => void;
+	const ensure = createBrowserSetup(async () => {
+		calls++;
+		await new Promise<void>((resolve) => {
+			finish = resolve;
+		});
+	});
+	const first = ensure();
+	assert.equal(ensure(), first);
+	await Promise.resolve();
+	assert.equal(calls, 1);
+	finish();
+	await first;
+	await ensure();
+	assert.equal(calls, 1);
+});
+
+test("failed browser setup reports failure and allows a later retry", async () => {
+	let calls = 0;
+	const ensure = createBrowserSetup(async () => {
+		if (++calls === 1) throw new Error("offline");
+	});
+	await assert.rejects(ensure(), /offline/);
+	await ensure();
+	assert.equal(calls, 2);
+});

package/test/credentials.test.ts

@@ -0,0 +1,53 @@
+import assert from "node:assert/strict";
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import test from "node:test";
+import { readTypesafeCredentials } from "../src/credentials.ts";
+
+test("credential file handles JSON syntax, precedence, reloads and missing keys", () => {
+	const directory = mkdtempSync(join(tmpdir(), "jev-credentials-"));
+	const path = join(directory, "config.json");
+	try {
+		assert.throws(() => readTypesafeCredentials({ path, env: {} }), /TYPESAFE_API_KEY/);
+		writeFileSync(
+			path,
+			JSON.stringify({
+				typesafe: { apiKey: "file-test-key", model: "jev-latest" },
+			}),
+			{ mode: 0o600 },
+		);
+		assert.deepEqual(readTypesafeCredentials({ path, env: {} }), {
+			apiKey: "file-test-key",
+			model: "jev-latest",
+		});
+		assert.deepEqual(
+			readTypesafeCredentials({
+				path,
+				env: { TYPESAFE_API_KEY: "env-test-key", TYPESAFE_MODEL: "other-model" },
+			}),
+			{ apiKey: "env-test-key", model: "other-model" },
+		);
+		// A blank environment value must not shadow a configured file value.
+		assert.equal(
+			readTypesafeCredentials({ path, env: { TYPESAFE_API_KEY: "  " } }).apiKey,
+			"file-test-key",
+		);
+		// The model falls back to the current default when nothing sets it.
+		writeFileSync(path, JSON.stringify({ typesafe: { apiKey: "changed-test-key" } }));
+		assert.deepEqual(readTypesafeCredentials({ path, env: {} }), {
+			apiKey: "changed-test-key",
+			model: "jev-latest",
+		});
+		writeFileSync(path, "{}");
+		assert.throws(() => readTypesafeCredentials({ path, env: {} }), /TYPESAFE_API_KEY/);
+		writeFileSync(path, "{invalid");
+		assert.throws(() => readTypesafeCredentials({ path, env: {} }), /JSON syntax/);
+		assert.throws(
+			() => readTypesafeCredentials({ path: directory, env: {} }),
+			/Cannot read/,
+		);
+	} finally {
+		rmSync(directory, { recursive: true, force: true });
+	}
+});

package/test/extension.test.ts

@@ -0,0 +1,180 @@
+import assert from "node:assert/strict";
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import test from "node:test";
+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import extension, {
+	modelAcceptsImages,
+	verificationHint,
+} from "../extensions/jev-browser.ts";
+import { normalizeKey } from "../src/actions.ts";
+import { isUrlAllowed, readConfig } from "../src/config.ts";
+
+interface RegisteredTool {
+	name: string;
+	description: string;
+	promptSnippet?: string;
+	promptGuidelines?: string[];
+	executionMode?: string;
+	parameters: {
+		type: string;
+		required?: string[];
+		properties: Record<string, any>;
+	};
+}
+
+function loadExtension() {
+	const tools = new Map<string, RegisteredTool>();
+	const events = new Map<string, unknown>();
+	extension({
+		registerTool(tool: RegisteredTool) {
+			tools.set(tool.name, tool);
+		},
+		on(event: string, handler: unknown) {
+			events.set(event, handler);
+		},
+	} as unknown as ExtensionAPI);
+	return { tools, events };
+}
+
+test("registers the complete Jev Browser surface", () => {
+	const { tools, events } = loadExtension();
+	assert.deepEqual([...tools.keys()], [
+		"jev_run",
+		"jev_actions",
+		"jev_state",
+		"jev_logs",
+		"jev_stream",
+		"jev_stop",
+	]);
+	// The browser must be released when pi replaces or exits the session.
+	assert.ok(events.has("session_shutdown"));
+	// Serialized: these drive one shared page, and pi may call tools concurrently.
+	for (const name of ["jev_run", "jev_actions", "jev_stream", "jev_stop"])
+		assert.equal(tools.get(name)?.executionMode, "sequential", name);
+	// Safety rules live on the tools they constrain, and name them.
+	for (const tool of tools.values())
+		for (const guideline of tool.promptGuidelines ?? [])
+			assert.ok(
+				guideline.includes(tool.name) || guideline.includes("Jev"),
+				`guideline must name its tool: ${guideline.slice(0, 60)}`,
+			);
+	const guidelines = tools.get("jev_run")?.promptGuidelines ?? [];
+	assert.ok(guidelines.some((line) => line.includes("done_unverified")));
+	assert.ok(guidelines.some((line) => line.includes("jev_stop")));
+	assert.ok(
+		(tools.get("jev_stop")?.promptGuidelines ?? []).some((line) =>
+			line.includes("jev_stop"),
+		),
+	);
+	assert.ok(
+		(tools.get("jev_actions")?.promptGuidelines ?? []).some((line) =>
+			line.includes("never use it automatically"),
+		),
+	);
+});
+
+test("tool schemas keep the Jev Browser bounds", () => {
+	const { tools } = loadExtension();
+	const run = tools.get("jev_run")!.parameters;
+	assert.equal(run.type, "object");
+	assert.deepEqual(run.required, ["goal"]);
+	assert.equal(run.properties.goal.maxLength, 12_000);
+	assert.equal(run.properties.maxSteps.maximum, 60);
+	assert.equal(run.properties.minProbability.maximum, 1);
+
+	const actions = tools.get("jev_actions")!.parameters;
+	assert.deepEqual(actions.required, ["actions"]);
+	assert.equal(actions.properties.actions.maxItems, 50);
+	assert.equal(actions.properties.includeScreenshot.type, "boolean");
+
+	// Google rejects unions and literals in tool schemas, so enums stay flat.
+	assert.deepEqual(tools.get("jev_stream")!.parameters.properties.action, {
+		type: "string",
+		enum: ["start", "status", "stop"],
+	});
+	assert.ok(
+		(actions.properties.actions.items.properties.type as { enum: string[] }).enum.includes(
+			"double_click",
+		),
+	);
+	const drag = actions.properties.actions.items.properties.path;
+	assert.match(JSON.stringify(drag), /"maxItems":2/);
+	assert.equal(tools.get("jev_state")!.parameters.type, "object");
+});
+
+test("verification does not depend on the model being able to see images", () => {
+	assert.equal(modelAcceptsImages({ input: ["text", "image"] }), true);
+	assert.equal(modelAcceptsImages({ input: ["text"] }), false);
+	assert.equal(modelAcceptsImages(undefined), false);
+
+	const visual = verificationHint(true);
+	const textual = verificationHint(false);
+	for (const hint of [visual, textual])
+		assert.match(hint, /done_unverified is a claim/, hint.slice(0, 40));
+	// Pi turns blocked images into this exact placeholder, so the model can name the cause.
+	assert.match(visual, /Image reading is disabled/);
+	assert.match(visual, /blockImages/);
+	// Without vision the run still has to be checkable, and status alone is not enough.
+	assert.match(textual, /cannot receive images/);
+	assert.match(textual, /final page text/);
+	assert.match(textual, /Never report success from the status alone/);
+});
+
+test("matches configured origins and blocks unsupported schemes", () => {
+	const patterns = ["https://*.example.com", "http://localhost:*"];
+	assert.equal(isUrlAllowed("https://app.example.com/path", patterns), true);
+	assert.equal(isUrlAllowed("http://localhost:4173", patterns), true);
+	assert.equal(isUrlAllowed("https://example.net", patterns), false);
+	assert.equal(isUrlAllowed("file:///etc/passwd", ["*"]), false);
+});
+
+test("loads bounded config values", () => {
+	const directory = mkdtempSync(join(tmpdir(), "pi-jev-browser-"));
+	const path = join(directory, "config.json");
+	try {
+		writeFileSync(
+			path,
+			JSON.stringify({
+				policy: "typesafe",
+				allowedOrigins: ["https://example.com"],
+				viewport: { width: 99, height: 9999 },
+				stream: { enabled: true, intervalMs: 10 },
+			}),
+		);
+		const config = readConfig(path);
+		assert.equal(config.policy, "typesafe");
+		assert.deepEqual(config.allowedOrigins, ["https://example.com"]);
+		assert.deepEqual(config.viewport, { width: 640, height: 1600 });
+		assert.deepEqual(config.stream, { enabled: true, intervalMs: 250 });
+	} finally {
+		rmSync(directory, { recursive: true, force: true });
+	}
+});
+
+test("uses complete defaults when configuration is omitted", () => {
+	const config = readConfig(join(tmpdir(), "missing-pi-jev-browser.config.json"));
+	// pi's own model by default: no second credential, no extra API quota.
+	assert.equal(config.policy, "pi");
+	assert.deepEqual(config.allowedOrigins, ["http://*", "https://*"]);
+	assert.equal(config.headless, true);
+	assert.equal(config.recordVideo, true);
+	assert.equal(config.showCursor, true);
+	assert.equal(config.showClickIndicators, true);
+	assert.deepEqual(config.viewport, { width: 1280, height: 720 });
+	assert.deepEqual(config.stream, { enabled: false, intervalMs: 1000 });
+	assert.equal(isUrlAllowed("https://openai.com", config.allowedOrigins), true);
+	assert.equal(
+		isUrlAllowed("http://example.test:8080", config.allowedOrigins),
+		true,
+	);
+	assert.equal(isUrlAllowed("file:///etc/passwd", config.allowedOrigins), false);
+	assert.match(config.outputDir, /\.pi\/agent\/data\/jev-browser$/);
+});
+
+test("normalizes Jev Browser key aliases", () => {
+	assert.equal(normalizeKey("CTRL"), "Control");
+	assert.equal(normalizeKey("ARROWDOWN"), "ArrowDown");
+	assert.equal(normalizeKey("a"), "a");
+});