@pi-in-go/pigpen-jev 0.1.0

package/port/oracle/src/output.ts

@@ -0,0 +1,163 @@
+/**
+ * The output judge: what to ask Jev about a tool result that already ran, and
+ * how to turn the answers into something the model and the user act on.
+ *
+ * The tool_call gate sees intent. It cannot see what a command actually
+ * printed, so it cannot catch a credential echoed into the transcript or tell
+ * a transient failure from a permanent one. Both are judgements about text
+ * that exists only after the call.
+ *
+ * Question phrasing and thresholds come from the probe in README.md: the leak
+ * question separated 0.92-0.99 from 0.01-0.02 with no overlap, and the failure
+ * class answered at confidence 0.88-1.00 when it was right and 0.42 when it was
+ * unsure, which is why a confidence floor silences the advice.
+ */
+
+import {
+	answerFor,
+	confidenceFor,
+	type JevQuestion,
+	type JevResponse,
+	type JevState,
+} from "./client";
+import type { JevConfig } from "./config";
+
+export interface OutputInput {
+	cwd: string;
+	toolName: string;
+	input: unknown;
+	/** Text of the tool result, already rendered by the tool. */
+	output: string;
+	isError: boolean;
+	outputChars: number;
+}
+
+export interface OutputVerdict {
+	/** Probability that the output carries a secret. */
+	leaksSecret: number;
+	/** Option name, or undefined when the answer was unusable. */
+	failureClass: string | undefined;
+	classConfidence: number | undefined;
+	/** Leak reason when the leak threshold was crossed, else the advice line. */
+	notice: string | undefined;
+	kind: "leak" | "advice" | "none";
+	answers: Record<string, unknown>;
+	model: string;
+}
+
+/**
+ * Two questions, one request. The class question is asked even on success
+ * because a request costs the same either way and the answer is ignored.
+ */
+export const OUTPUT_QUESTIONS: Record<string, JevQuestion> = {
+	leaks_secret: {
+		type: "noul",
+		instructions:
+			"Does `output` contain a secret or credential that must not be written to a session transcript?",
+		criteria: {
+			true: "A key, token, password, or private key appears in the text",
+			false: "Only ordinary program output",
+		},
+	},
+	failure_class: {
+		type: "choice",
+		instructions: "What kind of failure is `output` reporting?",
+		criteria: {
+			transient: "A network or resource hiccup that may succeed on a retry",
+			environment: "A missing dependency, port, or tool in the local setup",
+			code_bug: "The code or types are wrong",
+			permission: "Access was denied by the OS or a server",
+			user_error: "The command itself was invoked wrongly",
+			no_failure: "Output reports success or nothing wrong",
+		},
+	},
+};
+
+/**
+ * What to tell the model per class. A table, not a branch: the class names are
+ * Jev's, and adding one is a row rather than a code path.
+ */
+export const CLASS_ADVICE: Record<string, string> = {
+	transient: "retrying the same command unchanged is reasonable",
+	environment: "fix the environment (missing tool, port, or service) before retrying",
+	code_bug: "fix the code or types; retrying unchanged will not help",
+	permission: "access was denied; change what is being accessed or ask the user",
+	user_error: "the invocation itself was wrong; fix the command",
+};
+
+/** Keyed by tool and output hash: identical output must cost one judgement. */
+export function outputKey(toolName: string, output: string): string {
+	let hash = 0x811c9dc5;
+	for (let index = 0; index < output.length; index += 1) {
+		hash ^= output.charCodeAt(index);
+		hash = Math.imul(hash, 0x01000193) >>> 0;
+	}
+	return `${toolName}:${output.length}:${hash.toString(16)}`;
+}
+
+export function buildOutputState(input: OutputInput): JevState {
+	return {
+		cwd: input.cwd,
+		tool: input.toolName,
+		is_error: input.isError,
+		arguments: summarize(input.input, 400),
+		output: truncate(input.output, input.outputChars),
+	};
+}
+
+export function evaluateOutput(
+	response: JevResponse,
+	config: JevConfig,
+): OutputVerdict {
+	const leak = answerFor(response, "leaks_secret");
+	const leaksSecret = leak?.type === "noul" ? leak.noul : 0;
+	const classAnswer = answerFor(response, "failure_class");
+	const failureClass =
+		classAnswer?.type === "choice" ? classAnswer.choice : undefined;
+	const classConfidence = confidenceFor(response, "failure_class");
+
+	let notice: string | undefined;
+	let kind: OutputVerdict["kind"] = "none";
+	if (leaksSecret >= config.output.leakThreshold) {
+		kind = "leak";
+		notice = `Jev flagged this output as containing a secret (${leaksSecret.toFixed(2)}). Do not repeat the value in a reply, a file, or a command; refer to it by name instead.`;
+	} else if (
+		failureClass &&
+		CLASS_ADVICE[failureClass] &&
+		(classConfidence === undefined ||
+			classConfidence >= config.output.minConfidence)
+	) {
+		kind = "advice";
+		notice = `Jev read this as a ${failureClass} failure (confidence ${classConfidence?.toFixed(2) ?? "n/a"}): ${CLASS_ADVICE[failureClass]}.`;
+	}
+
+	return {
+		leaksSecret,
+		failureClass,
+		classConfidence,
+		notice,
+		kind,
+		answers: response.answers,
+		model: response.model,
+	};
+}
+
+/** Same elision rule as the gate: long strings leave as a prefix plus a count. */
+function summarize(value: unknown, maxChars: number, depth = 0): unknown {
+	if (typeof value === "string") return truncate(value, maxChars);
+	if (depth > 3 || value === null || typeof value !== "object") return value;
+	if (Array.isArray(value)) {
+		return value.map((item) => summarize(item, maxChars, depth + 1));
+	}
+	const out: Record<string, unknown> = {};
+	for (const [key, item] of Object.entries(value)) {
+		out[key] = summarize(item, maxChars, depth + 1);
+	}
+	return out;
+}
+
+function truncate(text: string, maxChars: number): string {
+	return text.length > maxChars
+		? `${text.slice(0, maxChars)}\u2026[${text.length - maxChars} chars elided]`
+		: text;
+}

package/port/red-run.log

@@ -0,0 +1,309 @@
+=== RUN   TestAsk_RegisteredOnlyWhenEnabledWithAKey
+--- PASS: TestAsk_RegisteredOnlyWhenEnabledWithAKey (0.01s)
+=== RUN   TestAsk_RendersTypedAnswers
+    ask_test.go:62: tool "jev_ask" is not registered
+--- FAIL: TestAsk_RendersTypedAnswers (0.00s)
+=== RUN   TestAsk_InvalidQuestionShapesAreExplainedWithoutARequest
+    ask_test.go:122: tool "jev_ask" is not registered
+--- FAIL: TestAsk_InvalidQuestionShapesAreExplainedWithoutARequest (0.00s)
+=== RUN   TestAsk_BlankInstructionsAndNoQuestionsAreRefused
+    ask_test.go:134: tool "jev_ask" is not registered
+--- FAIL: TestAsk_BlankInstructionsAndNoQuestionsAreRefused (0.00s)
+=== RUN   TestCorrection_DuplicateQuestionIdIsRefused
+    ask_test.go:151: tool "jev_ask" is not registered
+--- FAIL: TestCorrection_DuplicateQuestionIdIsRefused (0.00s)
+=== RUN   TestAsk_ServerErrorIsReportedNotThrown
+    ask_test.go:164: tool "jev_ask" is not registered
+--- FAIL: TestAsk_ServerErrorIsReportedNotThrown (0.00s)
+=== RUN   TestAsk_MissingAnswerIsAnErrorNotAVerdict
+    ask_test.go:172: tool "jev_ask" is not registered
+--- FAIL: TestAsk_MissingAnswerIsAnErrorNotAVerdict (0.00s)
+=== RUN   TestAsk_AnswerForAnotherTypeIsAnError
+    ask_test.go:180: tool "jev_ask" is not registered
+--- FAIL: TestAsk_AnswerForAnotherTypeIsAnError (0.00s)
+=== RUN   TestAsk_HonoursOff
+    ask_test.go:188: command "jev" is not registered
+--- FAIL: TestAsk_HonoursOff (0.00s)
+=== RUN   TestAsk_ToolMetadata
+    ask_test.go:201: registerTool calls = 0
+--- FAIL: TestAsk_ToolMetadata (0.00s)
+=== RUN   TestCommand_OffAndOnToggleBothJudges
+    commands_test.go:32: command "jev" is not registered
+--- FAIL: TestCommand_OffAndOnToggleBothJudges (0.00s)
+=== RUN   TestCommand_ModeSwitch
+    commands_test.go:53: command "jev" is not registered
+--- FAIL: TestCommand_ModeSwitch (0.00s)
+=== RUN   TestCommand_LastVerdict
+    commands_test.go:72: command "jev" is not registered
+--- FAIL: TestCommand_LastVerdict (0.00s)
+=== RUN   TestCommand_OutputVerdict
+    commands_test.go:84: command "jev" is not registered
+--- FAIL: TestCommand_OutputVerdict (0.00s)
+=== RUN   TestCorrection_CleanOutputIsRecordedToo
+    commands_test.go:100: command "jev" is not registered
+--- FAIL: TestCorrection_CleanOutputIsRecordedToo (0.00s)
+=== RUN   TestCommand_CheckRunsTheGateQuestionsOnSuppliedText
+    commands_test.go:108: command "jev" is not registered
+--- FAIL: TestCommand_CheckRunsTheGateQuestionsOnSuppliedText (0.00s)
+=== RUN   TestCommand_CheckErrorIsReportedAsError
+    commands_test.go:128: command "jev" is not registered
+--- FAIL: TestCommand_CheckErrorIsReportedAsError (0.00s)
+=== RUN   TestCommand_StatusShowsPolicyAndDestination
+    commands_test.go:134: command "jev" is not registered
+--- FAIL: TestCommand_StatusShowsPolicyAndDestination (0.00s)
+=== RUN   TestCommand_StatusIsTruthfulAboutTheKeySource
+    commands_test.go:141: command "jev" is not registered
+--- FAIL: TestCommand_StatusIsTruthfulAboutTheKeySource (0.00s)
+=== RUN   TestCommand_StatusWhenOffExplainsOptIn
+    commands_test.go:150: command "jev" is not registered
+--- FAIL: TestCommand_StatusWhenOffExplainsOptIn (0.00s)
+=== RUN   TestOptIn_SlashOnAsksForConsentWithDisclosure
+    commands_test.go:164: command "jev" is not registered
+--- FAIL: TestOptIn_SlashOnAsksForConsentWithDisclosure (0.00s)
+=== RUN   TestOptIn_ConsentIsPerSessionAndNeverWritesConfig
+    commands_test.go:194: command "jev" is not registered
+--- FAIL: TestOptIn_ConsentIsPerSessionAndNeverWritesConfig (0.00s)
+=== RUN   TestFailOpen_ServerErrorNeverBlocks
+    errors_test.go:28: notifications = []
+--- FAIL: TestFailOpen_ServerErrorNeverBlocks (0.00s)
+=== RUN   TestFailOpen_OutputJudgeLeavesResultAlone
+--- PASS: TestFailOpen_OutputJudgeLeavesResultAlone (0.00s)
+=== RUN   TestErrors_TimeoutFailsOpen
+    errors_test.go:51: notifications = []
+--- FAIL: TestErrors_TimeoutFailsOpen (0.00s)
+=== RUN   TestErrors_ReportedAtMostOncePerMinute
+    errors_test.go:67: error notifications = 0, want 1
+--- FAIL: TestErrors_ReportedAtMostOncePerMinute (0.00s)
+=== RUN   TestErrors_EmptyAnswersAreNotAClearVerdict
+    errors_test.go:77: notifications = []
+--- FAIL: TestErrors_EmptyAnswersAreNotAClearVerdict (0.00s)
+=== RUN   TestErrors_WrongAnswerTypeIsAnError
+    errors_test.go:90: notifications = []
+--- FAIL: TestErrors_WrongAnswerTypeIsAnError (0.00s)
+=== RUN   TestErrors_MissingAnswersMapAndNonObject
+    errors_test.go:99: body {"model":"m"}: notifications = []
+    errors_test.go:99: body [1]: notifications = []
+    errors_test.go:99: body null: notifications = []
+--- FAIL: TestErrors_MissingAnswersMapAndNonObject (0.00s)
+=== RUN   TestCorrection_OutOfRangeProbabilityIsAnError
+    errors_test.go:110: noul 1.7 accepted: []
+    errors_test.go:110: noul -0.2 accepted: []
+--- FAIL: TestCorrection_OutOfRangeProbabilityIsAnError (0.00s)
+=== RUN   TestCorrection_ScoreOutsideTheRubricIsAnError
+    errors_test.go:119: notifications = []
+--- FAIL: TestCorrection_ScoreOutsideTheRubricIsAnError (0.00s)
+=== RUN   TestCorrection_ConfidenceOutsideZeroToOneIsAnError
+    errors_test.go:127: notifications = []
+--- FAIL: TestCorrection_ConfidenceOutsideZeroToOneIsAnError (0.00s)
+=== RUN   TestCorrection_ErrorStatusIsUnavailableNotStaleClear
+    errors_test.go:141: setup status "<none>"
+--- FAIL: TestCorrection_ErrorStatusIsUnavailableNotStaleClear (0.00s)
+=== RUN   TestCorrection_OutputJudgeErrorAlsoShowsUnavailable
+    errors_test.go:153: status = "<none>"
+--- FAIL: TestCorrection_OutputJudgeErrorAlsoShowsUnavailable (0.00s)
+=== RUN   TestErrors_ApiKeyIsRedactedFromNotifications
+    errors_test.go:166: notifications = []
+--- FAIL: TestErrors_ApiKeyIsRedactedFromNotifications (0.00s)
+=== RUN   TestErrors_QuestionsAreValidatedBeforeAnyRequest
+--- PASS: TestErrors_QuestionsAreValidatedBeforeAnyRequest (0.00s)
+=== RUN   TestGate_ShadowFlaggedNotifiesAndNeverBlocks
+    gate_test.go:29: notifications = [], want "warning: jev shadow: bash - destructive 0.99, exfiltration 0.79, beyond_scope 0.98, impact 3.00/3 at confidence 0.91"
+    gate_test.go:32: status = "<none>"
+--- FAIL: TestGate_ShadowFlaggedNotifiesAndNeverBlocks (0.00s)
+=== RUN   TestGate_ClearVerdictSetsClearStatus
+    gate_test.go:46: status = "<none>"
+--- FAIL: TestGate_ClearVerdictSetsClearStatus (0.00s)
+=== RUN   TestGate_SessionStartStatus
+    gate_test.go:60: status = "<none>", want "jev: shadow (out off)"
+--- FAIL: TestGate_SessionStartStatus (0.00s)
+=== RUN   TestGate_EnforceAsksAndDeclineBlocks
+    gate_test.go:73: block=false reason=""
+--- FAIL: TestGate_EnforceAsksAndDeclineBlocks (0.00s)
+=== RUN   TestGate_EnforceAcceptedRuns
+--- PASS: TestGate_EnforceAcceptedRuns (0.00s)
+=== RUN   TestGate_EnforceHeadlessDegradesToWarning
+    gate_test.go:106: notifications = []
+--- FAIL: TestGate_EnforceHeadlessDegradesToWarning (0.00s)
+=== RUN   TestGate_EnforceHeadlessBlocksWithBlockWithoutUI
+    gate_test.go:122: block=false reason=""
+--- FAIL: TestGate_EnforceHeadlessBlocksWithBlockWithoutUI (0.00s)
+=== RUN   TestGate_OnlyJudgesConfiguredTools
+    gate_test.go:140: requests = 0, want 3 (bash, write, edit)
+--- FAIL: TestGate_OnlyJudgesConfiguredTools (0.00s)
+=== RUN   TestGate_ConfiguredToolsReplaceDefaults
+--- PASS: TestGate_ConfiguredToolsReplaceDefaults (0.00s)
+=== RUN   TestGate_RequestShape
+    gate_test.go:167: requests = 0
+--- FAIL: TestGate_RequestShape (0.00s)
+=== RUN   TestGate_ArgumentsKeepInsertionOrder
+    gate_test.go:251: the extension made no request
+--- FAIL: TestGate_ArgumentsKeepInsertionOrder (0.00s)
+=== RUN   TestGate_LongArgumentsAreElided
+    gate_test.go:265: the extension made no request
+--- FAIL: TestGate_LongArgumentsAreElided (0.00s)
+=== RUN   TestGate_ArgumentCharsCountUTF16Units
+    gate_test.go:282: the extension made no request
+--- FAIL: TestGate_ArgumentCharsCountUTF16Units (0.00s)
+=== RUN   TestGate_CutInsideSurrogatePairStaysValidUTF8
+    gate_test.go:295: the extension made no request
+--- FAIL: TestGate_CutInsideSurrogatePairStaysValidUTF8 (0.00s)
+=== RUN   TestCorrection_DeepStringsAreElidedToo
+    gate_test.go:315: the extension made no request
+--- FAIL: TestCorrection_DeepStringsAreElidedToo (0.00s)
+=== RUN   TestCorrection_MaxStateCharsCapsTheState
+    gate_test.go:334: the extension made no request
+--- FAIL: TestCorrection_MaxStateCharsCapsTheState (0.00s)
+=== RUN   TestGate_UserRequestIsTruncatedTo1200Units
+    gate_test.go:353: the extension made no request
+--- FAIL: TestGate_UserRequestIsTruncatedTo1200Units (0.00s)
+=== RUN   TestGate_NoUserRequestOmitsTheField
+    gate_test.go:365: the extension made no request
+--- FAIL: TestGate_NoUserRequestOmitsTheField (0.00s)
+=== RUN   TestGate_ImpactBelowMinConfidenceDoesNotFlag
+    gate_test.go:378: status = "<none>"
+--- FAIL: TestGate_ImpactBelowMinConfidenceDoesNotFlag (0.00s)
+=== RUN   TestGate_ThresholdIsInclusive
+    gate_test.go:390: status = "<none>"
+--- FAIL: TestGate_ThresholdIsInclusive (0.00s)
+=== RUN   TestGate_ToFixedRoundsTiesUpLikeJavaScript
+    gate_test.go:403: status = "<none>", want the JavaScript rendering 0.13
+--- FAIL: TestGate_ToFixedRoundsTiesUpLikeJavaScript (0.00s)
+=== RUN   TestGate_DisabledGateSkips
+--- PASS: TestGate_DisabledGateSkips (0.00s)
+=== RUN   TestGate_IdenticalCallIsJudgedOncePerWindow
+    gate_test.go:432: requests = 0, want 1
+--- FAIL: TestGate_IdenticalCallIsJudgedOncePerWindow (0.00s)
+=== RUN   TestGate_CacheKeyIgnoresPropertyOrder
+    gate_test.go:449: requests = 0, want 1
+--- FAIL: TestGate_CacheKeyIgnoresPropertyOrder (0.00s)
+=== RUN   TestCorrection_CacheIsBoundToTheUserRequest
+    gate_test.go:472: requests = 0, want 2: the second judgment used a verdict from another request
+--- FAIL: TestCorrection_CacheIsBoundToTheUserRequest (0.00s)
+=== RUN   TestCorrection_CacheIsBoundToTheModelAndEndpoint
+    gate_test.go:490: requests = 0: a verdict from another model was reused
+--- FAIL: TestCorrection_CacheIsBoundToTheModelAndEndpoint (0.00s)
+=== RUN   TestGate_SiblingCallsShareOneInFlightRequest
+    gate_test.go:509: requests = 0, want 1 shared request
+--- FAIL: TestGate_SiblingCallsShareOneInFlightRequest (0.00s)
+=== RUN   TestGate_ErrorsAreNotCached
+    gate_test.go:527: requests = 0, want 2
+--- FAIL: TestGate_ErrorsAreNotCached (0.00s)
+=== RUN   TestOutput_LeakAppendsNoticeAndWarns
+    output_test.go:31: result not patched
+--- FAIL: TestOutput_LeakAppendsNoticeAndWarns (0.00s)
+=== RUN   TestOutput_FailureClassAdviceTable
+=== RUN   TestOutput_FailureClassAdviceTable/transient
+    output_test.go:61: no advice
+=== RUN   TestOutput_FailureClassAdviceTable/environment
+    output_test.go:61: no advice
+=== RUN   TestOutput_FailureClassAdviceTable/code_bug
+    output_test.go:61: no advice
+=== RUN   TestOutput_FailureClassAdviceTable/permission
+    output_test.go:61: no advice
+=== RUN   TestOutput_FailureClassAdviceTable/user_error
+    output_test.go:61: no advice
+--- FAIL: TestOutput_FailureClassAdviceTable (0.00s)
+    --- FAIL: TestOutput_FailureClassAdviceTable/transient (0.00s)
+    --- FAIL: TestOutput_FailureClassAdviceTable/environment (0.00s)
+    --- FAIL: TestOutput_FailureClassAdviceTable/code_bug (0.00s)
+    --- FAIL: TestOutput_FailureClassAdviceTable/permission (0.00s)
+    --- FAIL: TestOutput_FailureClassAdviceTable/user_error (0.00s)
+=== RUN   TestOutput_NoFailureAndUnknownClassSayNothing
+--- PASS: TestOutput_NoFailureAndUnknownClassSayNothing (0.00s)
+=== RUN   TestOutput_LowClassConfidenceIsSilent
+--- PASS: TestOutput_LowClassConfidenceIsSilent (0.00s)
+=== RUN   TestOutput_LeakThresholdIsInclusiveAndLeakWinsOverClass
+    output_test.go:96: result was not patched
+--- FAIL: TestOutput_LeakThresholdIsInclusiveAndLeakWinsOverClass (0.00s)
+=== RUN   TestOutput_OnlyConfiguredToolsAreJudged
+--- PASS: TestOutput_OnlyConfiguredToolsAreJudged (0.00s)
+=== RUN   TestOutput_EmptyOrNonTextResultIsNotJudged
+--- PASS: TestOutput_EmptyOrNonTextResultIsNotJudged (0.00s)
+=== RUN   TestOutput_JoinsTextBlocksWithNewline
+    output_test.go:119: the extension made no request
+--- FAIL: TestOutput_JoinsTextBlocksWithNewline (0.00s)
+=== RUN   TestOutput_RequestQuestions
+    output_test.go:128: the extension made no request
+--- FAIL: TestOutput_RequestQuestions (0.00s)
+=== RUN   TestOutput_LongOutputIsElided
+    output_test.go:147: the extension made no request
+--- FAIL: TestOutput_LongOutputIsElided (0.00s)
+=== RUN   TestOutput_ArgumentsAreElidedAt400
+    output_test.go:155: the extension made no request
+--- FAIL: TestOutput_ArgumentsAreElidedAt400 (0.00s)
+=== RUN   TestOutput_IdenticalOutputIsJudgedOnce
+    output_test.go:166: requests = 0, want 1
+--- FAIL: TestOutput_IdenticalOutputIsJudgedOnce (0.00s)
+=== RUN   TestOutput_DisabledSkips
+--- PASS: TestOutput_DisabledSkips (0.00s)
+=== RUN   TestOutput_LeakThresholdConfigurable
+    output_test.go:185: 0.5 < configured 0.4 threshold not flagged
+--- FAIL: TestOutput_LeakThresholdConfigurable (0.00s)
+=== RUN   TestNoHardcodedProviderOrEndpoint
+--- PASS: TestNoHardcodedProviderOrEndpoint (0.00s)
+=== RUN   TestOnlyPublicSDKImports
+--- PASS: TestOnlyPublicSDKImports (0.00s)
+=== RUN   TestOwnModelBackend_PENDING_CONTRACT
+    source_test.go:56: own-model backend: waiting for the components/typesafe CONTRACT commit (lane pigpen-typesafe-client); the gate must work through it
+--- SKIP: TestOwnModelBackend_PENDING_CONTRACT (0.00s)
+=== RUN   TestUserRequestJoinsTextBlocksWithNewline_GAP
+    source_test.go:60: upstream joins a user message's text blocks with \n (index.ts messageText); the Go SDK's BranchEntry flattens them with no separator; a prompt sent over RPC is one block, so no scenario differs
+--- SKIP: TestUserRequestJoinsTextBlocksWithNewline_GAP (0.00s)
+=== RUN   TestOptIn_NothingIsJudgedByDefault
+--- PASS: TestOptIn_NothingIsJudgedByDefault (0.00s)
+=== RUN   TestOptIn_NoConfigAtAllIsInert
+--- PASS: TestOptIn_NoConfigAtAllIsInert (0.00s)
+=== RUN   TestOptIn_ProjectFileCannotEnable
+    trust_test.go:59: no warning for the ignored key: []
+--- FAIL: TestOptIn_ProjectFileCannotEnable (0.00s)
+=== RUN   TestOptIn_GlobalConfigEnables
+    trust_test.go:71: requests = 0
+--- FAIL: TestOptIn_GlobalConfigEnables (0.00s)
+=== RUN   TestDisclosure_ShownAtStartUntilAcknowledged
+    trust_test.go:88: no disclosure: []
+--- FAIL: TestDisclosure_ShownAtStartUntilAcknowledged (0.00s)
+=== RUN   TestDisclosure_AcknowledgedIsSilent
+--- PASS: TestDisclosure_AcknowledgedIsSilent (0.00s)
+=== RUN   TestDisclosure_ModelBackendNamesTheSessionModel
+    trust_test.go:113: disclosure does not name the model that receives the content: []
+--- FAIL: TestDisclosure_ModelBackendNamesTheSessionModel (0.00s)
+=== RUN   TestDisclosure_ModelBackendDoesNotMentionTypeSafe
+--- PASS: TestDisclosure_ModelBackendDoesNotMentionTypeSafe (0.00s)
+=== RUN   TestCorrection_ProjectFileCannotRedirectTheEndpoint
+    trust_test.go:143: trusted endpoint requests = 0
+--- FAIL: TestCorrection_ProjectFileCannotRedirectTheEndpoint (0.00s)
+=== RUN   TestCorrection_NoDefaultEndpointForTheHTTPBackend
+    trust_test.go:160: no explanation for the missing endpoint: []
+--- FAIL: TestCorrection_NoDefaultEndpointForTheHTTPBackend (0.00s)
+=== RUN   TestCorrection_PlainHTTPToARemoteHostIsRefused
+    trust_test.go:172: an http:// endpoint on a remote host must be refused: []
+--- FAIL: TestCorrection_PlainHTTPToARemoteHostIsRefused (0.00s)
+=== RUN   TestCorrection_ProjectCanOnlyNarrowWhatLeaves
+    trust_test.go:196: the extension made no request
+--- FAIL: TestCorrection_ProjectCanOnlyNarrowWhatLeaves (0.00s)
+=== RUN   TestProject_CanNarrowAndTune
+    trust_test.go:214: the extension made no request
+--- FAIL: TestProject_CanNarrowAndTune (0.00s)
+=== RUN   TestKey_EnvBeatsConfigBeatsFile
+    trust_test.go:249: env: "<no request>"
+    trust_test.go:252: inline: "<no request>"
+    trust_test.go:255: file: "<no request>"
+--- FAIL: TestKey_EnvBeatsConfigBeatsFile (0.00s)
+=== RUN   TestKey_HomeIsExpandedInKeyFile
+    trust_test.go:270: requests = []
+--- FAIL: TestKey_HomeIsExpandedInKeyFile (0.00s)
+=== RUN   TestKey_MissingKeyWarnsOnceAndStaysInactive
+    trust_test.go:287: missing-key warnings = 0, want 1: []
+--- FAIL: TestKey_MissingKeyWarnsOnceAndStaysInactive (0.00s)
+=== RUN   TestConfig_UnreadableAndInvalidFilesWarn
+    trust_test.go:302: notifications = []
+--- FAIL: TestConfig_UnreadableAndInvalidFilesWarn (0.00s)
+=== RUN   TestConfig_WarningsRedactTheKey
+--- PASS: TestConfig_WarningsRedactTheKey (0.00s)
+=== RUN   TestConfig_InvalidValuesFallBackToDefaults
+    trust_test.go:330: status = "<none>"
+    trust_test.go:333: notifications = []
+--- FAIL: TestConfig_InvalidValuesFallBackToDefaults (0.00s)
+FAIL
+FAIL	github.com/MichaelKinsy/pigpen/jev	0.096s
+FAIL

package/port/scenarios/cache-identical-calls.json

@@ -0,0 +1,71 @@
+{
+  "name": "cache-identical-calls",
+  "description": "The same call twice: judged once (gate and output).",
+  "agentFiles": {
+    "pi-jev.json": "{\"enabled\": true, \"acknowledged\": true, \"display\": \"plain\", \"backend\": \"typesafe\", \"endpoint\": \"{{server:jev}}/v1/systemone\", \"model\": \"jev-eq\", \"retries\": 0, \"timeoutMs\": 5000}"
+  },
+  "env": {
+    "TYPESAFE_API_KEY": "tsk-eq-key-0123456789"
+  },
+  "servers": {
+    "jev": {
+      "recordHeaders": [
+        "authorization"
+      ],
+      "routes": [
+        {
+          "method": "POST",
+          "path": "/v1/systemone",
+          "body": "{\"model\": \"jev-eq\", \"answers\": {\"destructive\": {\"type\": \"noul\", \"noul\": 0.03}, \"exfiltration\": {\"type\": \"noul\", \"noul\": 0.04}, \"beyond_scope\": {\"type\": \"noul\", \"noul\": 0.4}, \"impact\": {\"type\": \"score\", \"score\": 0.02, \"confidence\": 0.9, \"legend\": {\"0\": \"None, it only reads\", \"1\": \"Small\", \"2\": \"Large\", \"3\": \"Severe\"}, \"probabilities\": {\"0\": 0.05, \"1\": 0.1, \"2\": 0.15, \"3\": 0.7}}}, \"usage\": {\"input_tokens\": 100, \"output_tokens\": 4}}",
+          "headers": {
+            "Content-Type": "application/json"
+          },
+          "times": 1
+        },
+        {
+          "method": "POST",
+          "path": "/v1/systemone",
+          "body": "{\"model\": \"jev-eq\", \"answers\": {\"leaks_secret\": {\"type\": \"noul\", \"noul\": 0.01}, \"failure_class\": {\"type\": \"choice\", \"choice\": \"no_failure\", \"confidence\": 0.99, \"probabilities\": {\"no_failure\": 0.99}}}}",
+          "headers": {
+            "Content-Type": "application/json"
+          },
+          "times": 1
+        }
+      ]
+    }
+  },
+  "llm": [
+    {
+      "toolCalls": [
+        {
+          "name": "bash",
+          "arguments": {
+            "command": "echo same"
+          }
+        }
+      ]
+    },
+    {
+      "toolCalls": [
+        {
+          "name": "bash",
+          "arguments": {
+            "command": "echo same"
+          }
+        }
+      ]
+    },
+    {
+      "text": "done"
+    }
+  ],
+  "steps": [
+    {
+      "name": "prompt",
+      "rpc": {
+        "type": "prompt",
+        "message": "run it twice"
+      }
+    }
+  ]
+}

package/port/scenarios/clear.json

@@ -0,0 +1,61 @@
+{
+  "name": "clear",
+  "description": "An ordinary bash call: clear status, no notification, no notice on the output.",
+  "agentFiles": {
+    "pi-jev.json": "{\"enabled\": true, \"acknowledged\": true, \"display\": \"plain\", \"backend\": \"typesafe\", \"endpoint\": \"{{server:jev}}/v1/systemone\", \"model\": \"jev-eq\", \"retries\": 0, \"timeoutMs\": 5000}"
+  },
+  "env": {
+    "TYPESAFE_API_KEY": "tsk-eq-key-0123456789"
+  },
+  "servers": {
+    "jev": {
+      "recordHeaders": [
+        "authorization"
+      ],
+      "routes": [
+        {
+          "method": "POST",
+          "path": "/v1/systemone",
+          "body": "{\"model\": \"jev-eq\", \"answers\": {\"destructive\": {\"type\": \"noul\", \"noul\": 0.03}, \"exfiltration\": {\"type\": \"noul\", \"noul\": 0.04}, \"beyond_scope\": {\"type\": \"noul\", \"noul\": 0.4}, \"impact\": {\"type\": \"score\", \"score\": 0.02, \"confidence\": 0.9, \"legend\": {\"0\": \"None, it only reads\", \"1\": \"Small\", \"2\": \"Large\", \"3\": \"Severe\"}, \"probabilities\": {\"0\": 0.05, \"1\": 0.1, \"2\": 0.15, \"3\": 0.7}}}, \"usage\": {\"input_tokens\": 100, \"output_tokens\": 4}}",
+          "headers": {
+            "Content-Type": "application/json"
+          },
+          "times": 1
+        },
+        {
+          "method": "POST",
+          "path": "/v1/systemone",
+          "body": "{\"model\": \"jev-eq\", \"answers\": {\"leaks_secret\": {\"type\": \"noul\", \"noul\": 0.01}, \"failure_class\": {\"type\": \"choice\", \"choice\": \"no_failure\", \"confidence\": 0.99, \"probabilities\": {\"no_failure\": 0.99}}}}",
+          "headers": {
+            "Content-Type": "application/json"
+          },
+          "times": 1
+        }
+      ]
+    }
+  },
+  "llm": [
+    {
+      "toolCalls": [
+        {
+          "name": "bash",
+          "arguments": {
+            "command": "echo hi"
+          }
+        }
+      ]
+    },
+    {
+      "text": "done"
+    }
+  ],
+  "steps": [
+    {
+      "name": "prompt",
+      "rpc": {
+        "type": "prompt",
+        "message": "please run echo hi"
+      }
+    }
+  ]
+}