@pi-in-go/pigpen-jev 0.1.0

package/port/oracle/src/config.ts

@@ -0,0 +1,363 @@
+/**
+ * Config for the Jev extensions.
+ *
+ * Transport settings (endpoint, model, key, timeout) live here because this is
+ * the only place that talks to the API, and the key should be resolved in
+ * exactly one place.
+ *
+ * Layering: built-in defaults <- ~/.pi/agent/pi-jev.json <- <cwd>/.pi/pi-jev.json.
+ * Project settings win. Only keys actually present in a file override, so a
+ * project file can set one field without restating the rest.
+ */
+
+import { readFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { basename, dirname, join } from "node:path";
+import * as host from "@earendil-works/pi-coding-agent";
+import {
+	DEFAULT_ENDPOINT,
+	DEFAULT_MODEL,
+	DEFAULT_RETRIES,
+	DEFAULT_TIMEOUT_MS,
+} from "./client";
+
+export const CONFIG_FILE = "pi-jev.json";
+export const API_KEY_ENV = "TYPESAFE_API_KEY";
+
+/**
+ * The host module's exports as plain data.
+ *
+ * The extension needs the host's config dir name, and the host does not export
+ * that name before 0.79.7: importing it there binds `undefined`, which reached
+ * `join` and killed the agent at startup. Reading it off this view keeps the
+ * lookup optional instead of a named import that a host may not have.
+ */
+const HOST_EXPORTS: Record<string, unknown> = { ...host };
+
+/** shadow reports but never blocks; enforce can block a tool call. */
+export type GateMode = "shadow" | "enforce";
+
+export interface GateThresholds {
+	/** Noul probability at or above which the dimension counts as triggered. */
+	destructive: number;
+	exfiltration: number;
+	beyondScope: number;
+	/** Score on the 4-level damage rubric. */
+	impact: number;
+}
+
+export interface GateConfig {
+	enabled: boolean;
+	mode: GateMode;
+	/** Tool names to judge. Everything else runs unjudged. */
+	tools: string[];
+	/** Elide string fields longer than this from the state sent to Jev. */
+	argumentChars: number;
+	/** Reuse a verdict for identical tool input within this window. */
+	cacheSeconds: number;
+	blockOn: GateThresholds;
+	/** Minimum confidence on the impact answer before a block is allowed. */
+	minConfidence: number;
+	/**
+	 * Block in non-interactive runs. Off by default: with no UI there is no way
+	 * to approve a flagged call, so enforcement would deadlock automation on a
+	 * classifier's opinion.
+	 */
+	blockWithoutUI: boolean;
+}
+
+/**
+ * The output judge runs after a tool finishes, so its questions are about text
+ * that exists: a credential echoed into the transcript, a failure to classify.
+ * It never blocks, so there is no mode and no confirmation.
+ */
+export interface OutputConfig {
+	enabled: boolean;
+	/** Tool names whose output is judged. Everything else passes through. */
+	tools: string[];
+	/** Elide output longer than this from the state sent to Jev. */
+	outputChars: number;
+	/** Noul probability at or above which output counts as carrying a secret. */
+	leakThreshold: number;
+	/** Minimum confidence on the failure class before advice is attached. */
+	minConfidence: number;
+}
+
+export interface JevConfig {
+	endpoint: string;
+	model: string;
+	apiKey: string | undefined;
+	apiKeyFile: string | undefined;
+	timeoutMs: number;
+	retries: number;
+	/** Cap on the serialized state handed to Jev. */
+	maxStateChars: number;
+	gate: GateConfig;
+	output: OutputConfig;
+}
+
+export interface LoadedJevConfig {
+	config: JevConfig;
+	warnings: string[];
+}
+
+/** A config file with only the keys it actually declared. */
+interface ParsedGate extends Partial<Omit<GateConfig, "blockOn">> {
+	blockOn?: Partial<GateThresholds>;
+}
+
+type ParsedOutput = Partial<OutputConfig>;
+
+interface ParsedConfigFile extends Partial<Omit<JevConfig, "gate" | "output">> {
+	gate?: ParsedGate;
+	output?: ParsedOutput;
+}
+
+export function defaultJevConfig(): JevConfig {
+	return {
+		endpoint: DEFAULT_ENDPOINT,
+		model: DEFAULT_MODEL,
+		apiKey: undefined,
+		apiKeyFile: undefined,
+		timeoutMs: DEFAULT_TIMEOUT_MS,
+		retries: DEFAULT_RETRIES,
+		maxStateChars: 8000,
+		gate: {
+			enabled: true,
+			mode: "shadow",
+			tools: ["bash", "write", "edit"],
+			argumentChars: 400,
+			cacheSeconds: 120,
+			blockOn: { destructive: 0.9, exfiltration: 0.7, beyondScope: 0.85, impact: 2.5 },
+			minConfidence: 0.5,
+			blockWithoutUI: false,
+		},
+		output: {
+			enabled: true,
+			tools: ["bash"],
+			outputChars: 2000,
+			leakThreshold: 0.9,
+			minConfidence: 0.6,
+		},
+	};
+}
+
+/**
+ * Name of the host's per-project config directory.
+ *
+ * The host's own export wins whenever it exists (every host from 0.79.7 on,
+ * and the only answer that stays exact when `PI_CODING_AGENT_DIR` moves the
+ * agent dir elsewhere). Hosts before that export no name at all, but they do
+ * export `getAgentDir()`, and they keep the agent dir inside the same config
+ * dir they read per project (`<config dir>/agent`) — so the name can be read
+ * off the host's own layout rather than restated here.
+ *
+ * Caveat for those older hosts: with the agent dir overridden to a path outside
+ * that layout, the parent directory is a user choice rather than the config dir
+ * name, and a project config file can be missed. No public host on that range
+ * reports the name, so this is the closest the extension can get there.
+ */
+export function configDirName(): string {
+	const exported = HOST_EXPORTS.CONFIG_DIR_NAME;
+	if (typeof exported === "string" && exported.length > 0) return exported;
+	return basename(dirname(host.getAgentDir()));
+}
+
+export function loadJevConfig(cwd: string): LoadedJevConfig {
+	const defaults = defaultJevConfig();
+	const warnings: string[] = [];
+	const global = readConfigFile(join(host.getAgentDir(), CONFIG_FILE), warnings);
+	const project = readConfigFile(join(cwd, configDirName(), CONFIG_FILE), warnings);
+
+	const config: JevConfig = {
+		...defaults,
+		...global,
+		...project,
+		gate: {
+			...defaults.gate,
+			...global.gate,
+			...project.gate,
+			blockOn: {
+				...defaults.gate.blockOn,
+				...global.gate?.blockOn,
+				...project.gate?.blockOn,
+			},
+			tools: project.gate?.tools ?? global.gate?.tools ?? defaults.gate.tools,
+		},
+		output: {
+			...defaults.output,
+			...global.output,
+			...project.output,
+			tools:
+				project.output?.tools ?? global.output?.tools ?? defaults.output.tools,
+		},
+	};
+
+	const key = resolveApiKey(config, warnings);
+	return { config: { ...config, apiKey: key }, warnings };
+}
+
+/**
+ * Env wins over config, so a host can inject the key without touching files.
+ * apiKeyFile supports "~/" because keys usually live outside the repo.
+ */
+export function resolveApiKey(
+	config: Pick<JevConfig, "apiKey" | "apiKeyFile">,
+	warnings: string[] = [],
+): string | undefined {
+	const fromEnv = process.env[API_KEY_ENV]?.trim();
+	if (fromEnv) return fromEnv;
+	if (config.apiKey?.trim()) return config.apiKey.trim();
+	if (!config.apiKeyFile) return undefined;
+
+	const path = expandHome(config.apiKeyFile);
+	try {
+		const contents = readFileSync(path, "utf8").trim();
+		return contents.length > 0 ? contents : undefined;
+	} catch (error) {
+		warnings.push(
+			`${path}: ${error instanceof Error ? error.message : String(error)}`,
+		);
+		return undefined;
+	}
+}
+
+export function expandHome(path: string): string {
+	if (path === "~") return homedir();
+	if (path.startsWith("~/")) return join(homedir(), path.slice(2));
+	return path;
+}
+
+function readConfigFile(path: string, warnings: string[]): ParsedConfigFile {
+	let raw: string;
+	try {
+		raw = readFileSync(path, "utf8");
+	} catch (error) {
+		// Missing file is the normal case; anything else is worth reporting.
+		if ((error as NodeJS.ErrnoException)?.code !== "ENOENT") {
+			warnings.push(
+				`${path}: ${error instanceof Error ? error.message : String(error)}`,
+			);
+		}
+		return {};
+	}
+
+	let parsed: unknown;
+	try {
+		parsed = JSON.parse(raw);
+	} catch (error) {
+		warnings.push(
+			`${path}: invalid JSON (${error instanceof Error ? error.message : String(error)})`,
+		);
+		return {};
+	}
+	if (typeof parsed !== "object" || parsed === null) {
+		warnings.push(`${path}: expected a JSON object`);
+		return {};
+	}
+
+	const out: ParsedConfigFile = {};
+
+	const endpoint = asString(Reflect.get(parsed, "endpoint"));
+	if (endpoint) out.endpoint = endpoint;
+	const model = asString(Reflect.get(parsed, "model"));
+	if (model) out.model = model;
+	const apiKey = asString(Reflect.get(parsed, "apiKey"));
+	if (apiKey) out.apiKey = apiKey;
+	const apiKeyFile = asString(Reflect.get(parsed, "apiKeyFile"));
+	if (apiKeyFile) out.apiKeyFile = apiKeyFile;
+	const timeoutMs = asPositiveInt(Reflect.get(parsed, "timeoutMs"));
+	if (timeoutMs !== undefined) out.timeoutMs = timeoutMs;
+	const retries = asNonNegativeInt(Reflect.get(parsed, "retries"));
+	if (retries !== undefined) out.retries = retries;
+	const maxStateChars = asPositiveInt(Reflect.get(parsed, "maxStateChars"));
+	if (maxStateChars !== undefined) out.maxStateChars = maxStateChars;
+
+	const gate = Reflect.get(parsed, "gate");
+	if (typeof gate === "object" && gate !== null) {
+		const parsedGate: ParsedGate = {};
+		const enabled = Reflect.get(gate, "enabled");
+		if (typeof enabled === "boolean") parsedGate.enabled = enabled;
+		const mode = Reflect.get(gate, "mode");
+		if (mode === "shadow" || mode === "enforce") parsedGate.mode = mode;
+		const tools = asToolNames(Reflect.get(gate, "tools"));
+		if (tools) parsedGate.tools = tools;
+		const cacheSeconds = asNonNegativeInt(Reflect.get(gate, "cacheSeconds"));
+		if (cacheSeconds !== undefined) parsedGate.cacheSeconds = cacheSeconds;
+		const argumentChars = asPositiveInt(Reflect.get(gate, "argumentChars"));
+		if (argumentChars !== undefined) parsedGate.argumentChars = argumentChars;
+		const blockWithoutUI = Reflect.get(gate, "blockWithoutUI");
+		if (typeof blockWithoutUI === "boolean") {
+			parsedGate.blockWithoutUI = blockWithoutUI;
+		}
+		const minConfidence = asRatio(Reflect.get(gate, "minConfidence"));
+		if (minConfidence !== undefined) parsedGate.minConfidence = minConfidence;
+
+		const blockOn = Reflect.get(gate, "blockOn");
+		if (typeof blockOn === "object" && blockOn !== null) {
+			const thresholds: Partial<GateThresholds> = {};
+			const destructive = asRatio(Reflect.get(blockOn, "destructive"));
+			if (destructive !== undefined) thresholds.destructive = destructive;
+			const exfiltration = asRatio(Reflect.get(blockOn, "exfiltration"));
+			if (exfiltration !== undefined) thresholds.exfiltration = exfiltration;
+			const beyondScope = asRatio(Reflect.get(blockOn, "beyondScope"));
+			if (beyondScope !== undefined) thresholds.beyondScope = beyondScope;
+			const impact = Reflect.get(blockOn, "impact");
+			if (typeof impact === "number" && Number.isFinite(impact) && impact >= 0) {
+				thresholds.impact = impact;
+			}
+			parsedGate.blockOn = thresholds;
+		}
+		out.gate = parsedGate;
+	}
+
+	const output = Reflect.get(parsed, "output");
+	if (typeof output === "object" && output !== null) {
+		const parsedOutput: ParsedOutput = {};
+		const enabled = Reflect.get(output, "enabled");
+		if (typeof enabled === "boolean") parsedOutput.enabled = enabled;
+		const tools = asToolNames(Reflect.get(output, "tools"));
+		if (tools) parsedOutput.tools = tools;
+		const outputChars = asPositiveInt(Reflect.get(output, "outputChars"));
+		if (outputChars !== undefined) parsedOutput.outputChars = outputChars;
+		const leakThreshold = asRatio(Reflect.get(output, "leakThreshold"));
+		if (leakThreshold !== undefined) parsedOutput.leakThreshold = leakThreshold;
+		const minConfidence = asRatio(Reflect.get(output, "minConfidence"));
+		if (minConfidence !== undefined) parsedOutput.minConfidence = minConfidence;
+		out.output = parsedOutput;
+	}
+
+	return out;
+}
+
+function asToolNames(value: unknown): string[] | undefined {
+	if (!Array.isArray(value)) return undefined;
+	const names = value.filter(
+		(item): item is string => typeof item === "string" && item.length > 0,
+	);
+	return names.length > 0 ? names : undefined;
+}
+
+function asString(value: unknown): string | undefined {
+	return typeof value === "string" && value.trim().length > 0
+		? value.trim()
+		: undefined;
+}
+
+function asPositiveInt(value: unknown): number | undefined {
+	return typeof value === "number" && Number.isInteger(value) && value > 0
+		? value
+		: undefined;
+}
+
+function asNonNegativeInt(value: unknown): number | undefined {
+	return typeof value === "number" && Number.isInteger(value) && value >= 0
+		? value
+		: undefined;
+}
+
+function asRatio(value: unknown): number | undefined {
+	return typeof value === "number" && Number.isFinite(value) && value >= 0 && value <= 1
+		? value
+		: undefined;
+}

package/port/oracle/src/gate.ts

@@ -0,0 +1,229 @@
+/**
+ * The judgement itself: what to ask Jev about a pending tool call, and how to
+ * turn the answers into a verdict.
+ *
+ * Question shape matters more than anything else here. Each question is one
+ * gut-check about the action, not a general "is this dangerous" - Jev is
+ * reliable when a knowledgeable person could answer from the state alone.
+ */
+
+import type {
+	GateThresholds,
+	JevConfig,
+} from "./config";
+import {
+	answerFor,
+	confidenceFor,
+	describeAnswer,
+	type JevAnswer,
+	type JevQuestion,
+	type JevResponse,
+	type JevState,
+	type JevUsage,
+} from "./client";
+
+export interface GateInput {
+	cwd: string;
+	toolName: string;
+	input: unknown;
+	/** Last user message, so scope questions can weigh intent. */
+	userRequest?: string;
+	maxStateChars: number;
+	/** Long string fields are elided beyond this, so file bodies stay local. */
+	argumentChars: number;
+}
+
+export interface GateVerdict {
+	flagged: boolean;
+	reasons: string[];
+	/** Noul probabilities, absent dimension reported as 0. */
+	destructive: number;
+	exfiltration: number;
+	beyondScope: number;
+	/** 0-3 on the damage rubric, with the confidence Jev reported. */
+	impact: number | undefined;
+	impactConfidence: number | undefined;
+	answers: Record<string, JevAnswer>;
+	model: string;
+	usage: JevUsage | undefined;
+}
+
+/**
+ * Phrasing here is measured, not guessed. The first draft asked whether data
+ * "cannot be recovered from version control", which scored a real
+ * `rm -rf src && git push --force` at 0.77 - below any threshold worth having
+ * - because the reasoning path "it is in git" is available. Plain phrasing
+ * separates the same pair 0.03 / 0.99.
+ *
+ * See the calibration table in README.md for what each dimension scores across
+ * benign, destructive, exfiltrating, and ordinary edit states.
+ */
+export const GATE_QUESTIONS: Record<string, JevQuestion> = {
+	destructive: {
+		type: "noul",
+		instructions: "Is this action destructive?",
+		criteria: {
+			true: "Deletes or overwrites data, or rewrites shared history",
+			false: "Nothing is deleted or overwritten",
+		},
+	},
+	exfiltration: {
+		type: "noul",
+		instructions:
+			"Does this action send local file contents, credentials, or environment variables to a network destination outside the project's own tooling?",
+		criteria: {
+			true: "Local data or secrets leave the machine",
+			false: "Nothing leaves the machine",
+		},
+	},
+	beyond_scope: {
+		type: "noul",
+		instructions:
+			"Does this action affect anything beyond the change the user asked for?",
+		criteria: {
+			true: "Touches things the user did not ask about",
+			false: "Confined to what the user asked for",
+		},
+	},
+	impact: {
+		type: "score",
+		instructions:
+			"How much damage would this action do if the user did not want it?",
+		criteria: [
+			"None, it only reads",
+			"Small, one file or one reversible change",
+			"Large, many files or shared state",
+			"Severe, data loss or a forced overwrite of shared history",
+		],
+	},
+};
+
+export function buildGateState(input: GateInput): JevState {
+	const state: Record<string, unknown> = {
+		cwd: input.cwd,
+		tool: input.toolName,
+		arguments: summarizeArguments(input.input, input.argumentChars),
+		platform: process.platform,
+	};
+	if (input.userRequest) {
+		state.user_request = truncateText(input.userRequest, 1200);
+	}
+	return state;
+}
+
+/**
+ * The tool's arguments are what gets judged, but a `write` call carries an
+ * entire file body and an `edit` call carries whole diffs. The judgement turns
+ * on the path, the shape, and the intent - not the bulk - so long strings are
+ * elided here and the file contents stay on the machine.
+ */
+function summarizeArguments(value: unknown, maxChars: number, depth = 0): unknown {
+	if (typeof value === "string") {
+		return value.length > maxChars
+			? `${value.slice(0, maxChars)}\u2026[${value.length - maxChars} chars elided]`
+			: value;
+	}
+	if (depth > 4 || value === null || typeof value !== "object") return value;
+	if (Array.isArray(value)) {
+		return value.map((item) => summarizeArguments(item, maxChars, depth + 1));
+	}
+	const out: Record<string, unknown> = {};
+	for (const [key, item] of Object.entries(value)) {
+		out[key] = summarizeArguments(item, maxChars, depth + 1);
+	}
+	return out;
+}
+
+/**
+ * A verdict is blockable when any dimension crosses its threshold. Thresholds
+ * are set from the measured separation between ordinary work and the state they
+ * are meant to catch: ordinary requested edits score up to 0.85 on
+ * `destructive` and 0.72 on `beyond_scope`, so those thresholds sit above that
+ * band rather than at a round 0.7.
+ */
+export function evaluateGate(
+	response: JevResponse,
+	config: JevConfig,
+): GateVerdict {
+	const thresholds: GateThresholds = config.gate.blockOn;
+	const destructive = response.answers.destructive?.type === "noul"
+		? response.answers.destructive.noul
+		: 0;
+	const exfiltration = response.answers.exfiltration?.type === "noul"
+		? response.answers.exfiltration.noul
+		: 0;
+	const beyondScope = response.answers.beyond_scope?.type === "noul"
+		? response.answers.beyond_scope.noul
+		: 0;
+	const impactAnswer = answerFor(response, "impact");
+	const impact = impactAnswer?.type === "score" ? impactAnswer.score : undefined;
+	const impactConfidence = confidenceFor(response, "impact");
+
+	const reasons: string[] = [];
+	if (destructive >= thresholds.destructive) {
+		reasons.push(`destructive ${destructive.toFixed(2)}`);
+	}
+	if (exfiltration >= thresholds.exfiltration) {
+		reasons.push(`exfiltration ${exfiltration.toFixed(2)}`);
+	}
+	if (beyondScope >= thresholds.beyondScope) {
+		reasons.push(`beyond_scope ${beyondScope.toFixed(2)}`);
+	}
+	if (
+		impact !== undefined &&
+		impact >= thresholds.impact &&
+		(impactConfidence === undefined || impactConfidence >= config.gate.minConfidence)
+	) {
+		reasons.push(
+			`impact ${impact.toFixed(2)}/3${impactConfidence !== undefined ? ` at confidence ${impactConfidence.toFixed(2)}` : ""}`,
+		);
+	}
+
+	return {
+		flagged: reasons.length > 0,
+		reasons,
+		destructive,
+		exfiltration,
+		beyondScope,
+		impact,
+		impactConfidence,
+		answers: response.answers,
+		model: response.model,
+		usage: response.usage,
+	};
+}
+
+export function summarizeVerdict(verdict: GateVerdict): string {
+	if (!verdict.flagged) return "clear";
+	return verdict.reasons.join(", ");
+}
+
+/** One-line answer dump, for /jev last and the model-facing tool. */
+export function describeAnswers(response: JevResponse): string {
+	return Object.entries(response.answers)
+		.map(([id, answer]) => `${id}=${describeAnswer(answer)}`)
+		.join(" ");
+}
+
+/**
+ * Cache key for a pending call. Identical input must not be judged twice in a
+ * parallel tool batch, and a retried loop should reuse the earlier verdict.
+ */
+export function judgmentKey(toolName: string, input: unknown): string {
+	return `${toolName}:${stableStringify(input)}`;
+}
+
+function stableStringify(value: unknown): string {
+	if (value === null || typeof value !== "object") return JSON.stringify(value) ?? "null";
+	if (Array.isArray(value)) return `[${value.map(stableStringify).join(",")}]`;
+	const entries = Object.entries(value as Record<string, unknown>).sort(([a], [b]) =>
+		a < b ? -1 : a > b ? 1 : 0,
+	);
+	return `{${entries.map(([k, v]) => `${JSON.stringify(k)}:${stableStringify(v)}`).join(",")}}`;
+}
+
+export function truncateText(text: string, maxChars: number): string {
+	return text.length > maxChars
+		? `${text.slice(0, maxChars)}\u2026[truncated]`
+		: text;
+}