@pi-in-go/pigpen-jev 0.1.0

package/port/golden/commands.jsonl

@@ -0,0 +1,43 @@
+{"kind":"header","scenario":"commands","lane":"pi-ts","host":"pi 1.0.1","extension":"sha256:7fecf89390f53ec308d08d182796b1340e4c9553ac5e6cbdc2df19d9ad1b63e0","normalizer":"v3"}
+{"step":"00-startup","ch":"ui","data":{"method":"setStatus","statusKey":"jev","statusText":"jev: shadow"}}
+{"step":"01-prompt","ch":"response","data":{"command":"prompt","data":{"disposition":"started"},"success":true}}
+{"step":"01-prompt","ch":"host","data":{"type":"agent_start"}}
+{"step":"01-prompt","ch":"host","data":{"type":"turn_start"}}
+{"step":"01-prompt","ch":"host","data":{"message":{"content":"","role":"system"},"type":"message_end"}}
+{"step":"01-prompt","ch":"host","data":{"message":{"content":[{"text":"please run echo hi","type":"text"}],"role":"user"},"type":"message_end"}}
+{"step":"01-prompt","ch":"llm","data":{"messages":[{"role":"user","text":"please run echo hi"}],"system":{"inserted":"- jev_ask: Ask Jev typed questions (yes/no, choice, rubric) about text and get calibrated answers\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n</tools>\n\n<rules>\n- Use bash for file operations like ls, rg, find\n- Use read to examine files instead of cat or sed.\n- You can inspect PI_* environment variables for current model and session details.\n- Use edit for precise changes (edits[].oldText must match exactly)\n- When changing multiple separate locations in one file, use one edit call with multiple entries in edits[] instead of multiple edit calls\n- Each edits[].oldText is matched against the original file, not after earlier edits are applied. Do not emit overlapping or nested edits. Merge nearby changes into one edit.\n- Keep edits[].oldText as small as possible while still being unique in the file. Do not pad with large unchanged regions.\n- Use write only for new files or complete rewrites.\n- Use jev_ask when a judgement must be typed and calibrated rather than written: classification, relevance, yes/no checks, rubric scores.\n- Ask one specific question per entry in jev_ask; split multi-factor judgements into separate questions and combine the answers yourself","removed":"\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n</tools>\n\n<rules>\n- Use bash for file operations like ls, rg, find\n- Use read to examine files instead of cat or sed.\n- You can inspect PI_* environment variables for current model and session details.\n- Use edit for precise changes (edits[].oldText must match exactly)\n- When changing multiple separate locations in one file, use one edit call with multiple entries in edits[] instead of multiple edit calls\n- Each edits[].oldText is matched against the original file, not after earlier edits are applied. Do not emit overlapping or nested edits. Merge nearby changes into one edit.\n- Keep edits[].oldText as small as possible while still being unique in the file. Do not pad with large unchanged regions.\n- Use write only for new files or complete rewrites"},"tools":[{"description":"Read the contents of a file. Supports text files and images (jpg, png, gif, webp, bmp). Images are sent as attachments. For text files, output is truncated to 2000 lines or 50KB (whichever is hit first). Use offset/limit for large files. When you need the full file, continue with offset until complete.","name":"read","parameters":{"properties":{"limit":{"description":"Maximum number of lines to read","type":"number"},"offset":{"description":"Line number to start reading from (1-indexed)","type":"number"},"path":{"description":"Path to the file to read (relative or absolute)","type":"string"}},"required":["path"],"type":"object"}},{"description":"Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.","name":"bash","parameters":{"properties":{"command":{"description":"Shell command to execute","type":"string"},"timeout":{"description":"Timeout in seconds (optional, no default timeout)","type":"number"}},"required":["command"],"type":"object"}},{"description":"Edit a single file using exact text replacement. Every edits[].oldText must match a unique, non-overlapping region of the original file. If two changes affect the same block or nearby lines, merge them into one edit instead of emitting overlapping edits. Do not include large unchanged regions just to connect distant changes.","name":"edit","parameters":{"properties":{"edits":{"description":"One or more targeted replacements. Each edit is matched against the original file, not incrementally. Do not include overlapping or nested edits. If two changes touch the same block or nearby lines, merge them into one edit instead.","items":{"properties":{"newText":{"description":"Replacement text for this targeted edit.","type":"string"},"oldText":{"description":"Exact text for one targeted replacement. It must be unique in the original file and must not overlap with any other edits[].oldText in the same call.","type":"string"}},"required":["oldText","newText"],"type":"object"},"type":"array"},"path":{"description":"Path to the file to edit (relative or absolute)","type":"string"}},"required":["path","edits"],"type":"object"}},{"description":"Write content to a file. Creates the file if it doesn't exist, overwrites if it does. Automatically creates parent directories.","name":"write","parameters":{"properties":{"content":{"description":"Content to write to the file","type":"string"},"path":{"description":"Path to the file to write (relative or absolute)","type":"string"}},"required":["path","content"],"type":"object"}},{"description":"Ask TypeSafe Jev typed questions about a piece of text and get calibrated answers (probabilities, a chosen option, a rubric score) instead of prose.","name":"jev_ask","parameters":{"properties":{"questions":{"description":"One or more questions. All are evaluated in parallel against the same state.","items":{"properties":{"id":{"description":"Short key for this question. The answer comes back under it.","type":"string"},"instructions":{"description":"The one thing to judge. One specific, well-scoped gut-check per question.","type":"string"},"levels":{"description":"score only: ordered rubric levels, lowest first, at least two","items":{"type":"string"},"type":"array"},"options":{"description":"choice only: the options to choose between","items":{"properties":{"description":{"description":"When this option applies","type":"string"},"name":{"description":"Option key","type":"string"}},"required":["name"],"type":"object"},"type":"array"},"type":{"description":"noul = yes/no probability, choice = pick one option, score = value on a rubric","enum":["noul","choice","score"],"type":"string"}},"required":["id","type","instructions"],"type":"object"},"type":"array"},"state":{"description":"The text to judge: tool output, a diff, a message, a document excerpt.","type":"string"}},"required":["state","questions"],"type":"object"}}]}}
+{"step":"01-prompt","ch":"host","data":{"message":{"content":[{"arguments":{"command":"echo hi"},"name":"bash","type":"toolCall"}],"role":"assistant","stopReason":"toolUse"},"type":"message_end"}}
+{"step":"01-prompt","ch":"host","data":{"args":{"command":"echo hi"},"toolName":"bash","type":"tool_execution_start"}}
+{"step":"01-prompt","ch":"http","data":{"body":{"model":"jev-eq","questions":{"beyond_scope":{"criteria":{"false":"Confined to what the user asked for","true":"Touches things the user did not ask about"},"instructions":"Does this action affect anything beyond the change the user asked for?","type":"noul"},"destructive":{"criteria":{"false":"Nothing is deleted or overwritten","true":"Deletes or overwrites data, or rewrites shared history"},"instructions":"Is this action destructive?","type":"noul"},"exfiltration":{"criteria":{"false":"Nothing leaves the machine","true":"Local data or secrets leave the machine"},"instructions":"Does this action send local file contents, credentials, or environment variables to a network destination outside the project's own tooling?","type":"noul"},"impact":{"criteria":["None, it only reads","Small, one file or one reversible change","Large, many files or shared state","Severe, data loss or a forced overwrite of shared history"],"instructions":"How much damage would this action do if the user did not want it?","type":"score"}},"state":{"arguments":{"command":"echo hi"},"cwd":"<cwd>","platform":"linux","tool":"bash","user_request":"please run echo hi"}},"headers":{"authorization":"Bearer tsk-eq-key-0123456789","content-type":"application/json"},"matched":true,"method":"POST","path":"/v1/systemone","query":{},"server":"jev"}}
+{"step":"01-prompt","ch":"ui","data":{"method":"setStatus","statusKey":"jev","statusText":"jev: destructive 0.99, exfiltration 0.79, beyond_scope 0.98, impact 3.00/3 at confidence 0.91"}}
+{"step":"01-prompt","ch":"ui","data":{"message":"jev shadow: bash - destructive 0.99, exfiltration 0.79, beyond_scope 0.98, impact 3.00/3 at confidence 0.91","method":"notify","notifyType":"warning"}}
+{"step":"01-prompt","ch":"http","data":{"body":{"model":"jev-eq","questions":{"failure_class":{"criteria":{"code_bug":"The code or types are wrong","environment":"A missing dependency, port, or tool in the local setup","no_failure":"Output reports success or nothing wrong","permission":"Access was denied by the OS or a server","transient":"A network or resource hiccup that may succeed on a retry","user_error":"The command itself was invoked wrongly"},"instructions":"What kind of failure is `output` reporting?","type":"choice"},"leaks_secret":{"criteria":{"false":"Only ordinary program output","true":"A key, token, password, or private key appears in the text"},"instructions":"Does `output` contain a secret or credential that must not be written to a session transcript?","type":"noul"}},"state":{"arguments":{"command":"echo hi"},"cwd":"<cwd>","is_error":false,"output":"hi\n","tool":"bash"}},"headers":{"authorization":"Bearer tsk-eq-key-0123456789","content-type":"application/json"},"matched":true,"method":"POST","path":"/v1/systemone","query":{},"server":"jev"}}
+{"step":"01-prompt","ch":"ui","data":{"method":"setStatus","statusKey":"jev","statusText":"jev: leak (bash)"}}
+{"step":"01-prompt","ch":"ui","data":{"message":"jev: bash output may carry a secret (0.94)","method":"notify","notifyType":"warning"}}
+{"step":"01-prompt","ch":"host","data":{"isError":false,"result":{"content":[{"text":"hi\n","type":"text"},{"text":"[pi-jev] Jev flagged this output as containing a secret (0.94). Do not repeat the value in a reply, a file, or a command; refer to it by name instead.","type":"text"}]},"toolName":"bash","type":"tool_execution_end"}}
+{"step":"01-prompt","ch":"host","data":{"message":{"content":[{"text":"hi\n","type":"text"},{"text":"[pi-jev] Jev flagged this output as containing a secret (0.94). Do not repeat the value in a reply, a file, or a command; refer to it by name instead.","type":"text"}],"isError":false,"role":"toolResult","toolName":"bash"},"type":"message_end"}}
+{"step":"01-prompt","ch":"host","data":{"type":"turn_end"}}
+{"step":"01-prompt","ch":"host","data":{"type":"turn_start"}}
+{"step":"01-prompt","ch":"llm","data":{"messages":[{"role":"user","text":"please run echo hi"},{"role":"assistant","text":"","toolCalls":[{"arguments":"{\"command\":\"echo hi\"}","name":"bash"}]},{"role":"tool","text":"hi\n\n[pi-jev] Jev flagged this output as containing a secret (0.94). Do not repeat the value in a reply, a file, or a command; refer to it by name instead."}],"system":{"inserted":"- jev_ask: Ask Jev typed questions (yes/no, choice, rubric) about text and get calibrated answers\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n</tools>\n\n<rules>\n- Use bash for file operations like ls, rg, find\n- Use read to examine files instead of cat or sed.\n- You can inspect PI_* environment variables for current model and session details.\n- Use edit for precise changes (edits[].oldText must match exactly)\n- When changing multiple separate locations in one file, use one edit call with multiple entries in edits[] instead of multiple edit calls\n- Each edits[].oldText is matched against the original file, not after earlier edits are applied. Do not emit overlapping or nested edits. Merge nearby changes into one edit.\n- Keep edits[].oldText as small as possible while still being unique in the file. Do not pad with large unchanged regions.\n- Use write only for new files or complete rewrites.\n- Use jev_ask when a judgement must be typed and calibrated rather than written: classification, relevance, yes/no checks, rubric scores.\n- Ask one specific question per entry in jev_ask; split multi-factor judgements into separate questions and combine the answers yourself","removed":"\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n</tools>\n\n<rules>\n- Use bash for file operations like ls, rg, find\n- Use read to examine files instead of cat or sed.\n- You can inspect PI_* environment variables for current model and session details.\n- Use edit for precise changes (edits[].oldText must match exactly)\n- When changing multiple separate locations in one file, use one edit call with multiple entries in edits[] instead of multiple edit calls\n- Each edits[].oldText is matched against the original file, not after earlier edits are applied. Do not emit overlapping or nested edits. Merge nearby changes into one edit.\n- Keep edits[].oldText as small as possible while still being unique in the file. Do not pad with large unchanged regions.\n- Use write only for new files or complete rewrites"},"tools":[{"description":"Read the contents of a file. Supports text files and images (jpg, png, gif, webp, bmp). Images are sent as attachments. For text files, output is truncated to 2000 lines or 50KB (whichever is hit first). Use offset/limit for large files. When you need the full file, continue with offset until complete.","name":"read","parameters":{"properties":{"limit":{"description":"Maximum number of lines to read","type":"number"},"offset":{"description":"Line number to start reading from (1-indexed)","type":"number"},"path":{"description":"Path to the file to read (relative or absolute)","type":"string"}},"required":["path"],"type":"object"}},{"description":"Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.","name":"bash","parameters":{"properties":{"command":{"description":"Shell command to execute","type":"string"},"timeout":{"description":"Timeout in seconds (optional, no default timeout)","type":"number"}},"required":["command"],"type":"object"}},{"description":"Edit a single file using exact text replacement. Every edits[].oldText must match a unique, non-overlapping region of the original file. If two changes affect the same block or nearby lines, merge them into one edit instead of emitting overlapping edits. Do not include large unchanged regions just to connect distant changes.","name":"edit","parameters":{"properties":{"edits":{"description":"One or more targeted replacements. Each edit is matched against the original file, not incrementally. Do not include overlapping or nested edits. If two changes touch the same block or nearby lines, merge them into one edit instead.","items":{"properties":{"newText":{"description":"Replacement text for this targeted edit.","type":"string"},"oldText":{"description":"Exact text for one targeted replacement. It must be unique in the original file and must not overlap with any other edits[].oldText in the same call.","type":"string"}},"required":["oldText","newText"],"type":"object"},"type":"array"},"path":{"description":"Path to the file to edit (relative or absolute)","type":"string"}},"required":["path","edits"],"type":"object"}},{"description":"Write content to a file. Creates the file if it doesn't exist, overwrites if it does. Automatically creates parent directories.","name":"write","parameters":{"properties":{"content":{"description":"Content to write to the file","type":"string"},"path":{"description":"Path to the file to write (relative or absolute)","type":"string"}},"required":["path","content"],"type":"object"}},{"description":"Ask TypeSafe Jev typed questions about a piece of text and get calibrated answers (probabilities, a chosen option, a rubric score) instead of prose.","name":"jev_ask","parameters":{"properties":{"questions":{"description":"One or more questions. All are evaluated in parallel against the same state.","items":{"properties":{"id":{"description":"Short key for this question. The answer comes back under it.","type":"string"},"instructions":{"description":"The one thing to judge. One specific, well-scoped gut-check per question.","type":"string"},"levels":{"description":"score only: ordered rubric levels, lowest first, at least two","items":{"type":"string"},"type":"array"},"options":{"description":"choice only: the options to choose between","items":{"properties":{"description":{"description":"When this option applies","type":"string"},"name":{"description":"Option key","type":"string"}},"required":["name"],"type":"object"},"type":"array"},"type":{"description":"noul = yes/no probability, choice = pick one option, score = value on a rubric","enum":["noul","choice","score"],"type":"string"}},"required":["id","type","instructions"],"type":"object"},"type":"array"},"state":{"description":"The text to judge: tool output, a diff, a message, a document excerpt.","type":"string"}},"required":["state","questions"],"type":"object"}}]}}
+{"step":"01-prompt","ch":"host","data":{"message":{"content":[{"text":"done","type":"text"}],"role":"assistant","stopReason":"stop"},"type":"message_end"}}
+{"step":"01-prompt","ch":"host","data":{"type":"turn_end"}}
+{"step":"01-prompt","ch":"host","data":{"type":"agent_end"}}
+{"step":"02-last","ch":"ui","data":{"message":"pi-jev: bash - destructive 0.99, exfiltration 0.79, beyond_scope 0.98, impact 3.00/3 at confidence 0.91 | destructive=yes 0.99 exfiltration=yes 0.79 beyond_scope=yes 0.98 impact=3.00/3 (conf 0.91)","method":"notify","notifyType":"info"}}
+{"step":"02-last","ch":"response","data":{"command":"prompt","data":{"disposition":"handled"},"success":true}}
+{"step":"03-output","ch":"ui","data":{"message":"pi-jev output: bash - leak | leak 0.94 | class transient at 0.80","method":"notify","notifyType":"info"}}
+{"step":"03-output","ch":"response","data":{"command":"prompt","data":{"disposition":"handled"},"success":true}}
+{"step":"04-mode-enforce","ch":"ui","data":{"method":"setStatus","statusKey":"jev","statusText":"jev: enforce"}}
+{"step":"04-mode-enforce","ch":"ui","data":{"message":"pi-jev: enforce (asks before running flagged calls)","method":"notify","notifyType":"info"}}
+{"step":"04-mode-enforce","ch":"response","data":{"command":"prompt","data":{"disposition":"handled"},"success":true}}
+{"step":"05-mode-bogus","ch":"ui","data":{"message":"pi-jev: mode is enforce (usage: /jev mode shadow|enforce)","method":"notify","notifyType":"warning"}}
+{"step":"05-mode-bogus","ch":"response","data":{"command":"prompt","data":{"disposition":"handled"},"success":true}}
+{"step":"06-check","ch":"http","data":{"body":{"model":"jev-eq","questions":{"beyond_scope":{"criteria":{"false":"Confined to what the user asked for","true":"Touches things the user did not ask about"},"instructions":"Does this action affect anything beyond the change the user asked for?","type":"noul"},"destructive":{"criteria":{"false":"Nothing is deleted or overwritten","true":"Deletes or overwrites data, or rewrites shared history"},"instructions":"Is this action destructive?","type":"noul"},"exfiltration":{"criteria":{"false":"Nothing leaves the machine","true":"Local data or secrets leave the machine"},"instructions":"Does this action send local file contents, credentials, or environment variables to a network destination outside the project's own tooling?","type":"noul"},"impact":{"criteria":["None, it only reads","Small, one file or one reversible change","Large, many files or shared state","Severe, data loss or a forced overwrite of shared history"],"instructions":"How much damage would this action do if the user did not want it?","type":"score"}},"state":"Rm -RF /tmp/Build"},"headers":{"authorization":"Bearer tsk-eq-key-0123456789","content-type":"application/json"},"matched":true,"method":"POST","path":"/v1/systemone","query":{},"server":"jev"}}
+{"step":"06-check","ch":"ui","data":{"message":"pi-jev check: destructive 0.99 | destructive=yes 0.99 exfiltration=yes 0.10 beyond_scope=yes 0.10 impact=1.00/3 (conf 0.90)","method":"notify","notifyType":"info"}}
+{"step":"06-check","ch":"response","data":{"command":"prompt","data":{"disposition":"handled"},"success":true}}
+{"step":"07-off","ch":"ui","data":{"method":"setStatus","statusKey":"jev","statusText":"jev: off"}}
+{"step":"07-off","ch":"ui","data":{"message":"pi-jev: gate off","method":"notify","notifyType":"info"}}
+{"step":"07-off","ch":"response","data":{"command":"prompt","data":{"disposition":"handled"},"success":true}}
+{"step":"08-on","ch":"ui","data":{"method":"setStatus","statusKey":"jev","statusText":"jev: enforce"}}
+{"step":"08-on","ch":"ui","data":{"message":"pi-jev: gate on","method":"notify","notifyType":"info"}}
+{"step":"08-on","ch":"response","data":{"command":"prompt","data":{"disposition":"handled"},"success":true}}
+{"step":"99-shutdown","ch":"exit","data":{"code":0}}

package/port/golden/enforce-accept.jsonl

@@ -0,0 +1,23 @@
+{"kind":"header","scenario":"enforce-accept","lane":"pi-ts","host":"pi 1.0.1","extension":"sha256:7fecf89390f53ec308d08d182796b1340e4c9553ac5e6cbdc2df19d9ad1b63e0","normalizer":"v3"}
+{"step":"00-startup","ch":"ui","data":{"method":"setStatus","statusKey":"jev","statusText":"jev: enforce"}}
+{"step":"01-prompt","ch":"response","data":{"command":"prompt","data":{"disposition":"started"},"success":true}}
+{"step":"01-prompt","ch":"host","data":{"type":"agent_start"}}
+{"step":"01-prompt","ch":"host","data":{"type":"turn_start"}}
+{"step":"01-prompt","ch":"host","data":{"message":{"content":"","role":"system"},"type":"message_end"}}
+{"step":"01-prompt","ch":"host","data":{"message":{"content":[{"text":"please run echo hi","type":"text"}],"role":"user"},"type":"message_end"}}
+{"step":"01-prompt","ch":"llm","data":{"messages":[{"role":"user","text":"please run echo hi"}],"system":{"inserted":"- jev_ask: Ask Jev typed questions (yes/no, choice, rubric) about text and get calibrated answers\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n</tools>\n\n<rules>\n- Use bash for file operations like ls, rg, find\n- Use read to examine files instead of cat or sed.\n- You can inspect PI_* environment variables for current model and session details.\n- Use edit for precise changes (edits[].oldText must match exactly)\n- When changing multiple separate locations in one file, use one edit call with multiple entries in edits[] instead of multiple edit calls\n- Each edits[].oldText is matched against the original file, not after earlier edits are applied. Do not emit overlapping or nested edits. Merge nearby changes into one edit.\n- Keep edits[].oldText as small as possible while still being unique in the file. Do not pad with large unchanged regions.\n- Use write only for new files or complete rewrites.\n- Use jev_ask when a judgement must be typed and calibrated rather than written: classification, relevance, yes/no checks, rubric scores.\n- Ask one specific question per entry in jev_ask; split multi-factor judgements into separate questions and combine the answers yourself","removed":"\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n</tools>\n\n<rules>\n- Use bash for file operations like ls, rg, find\n- Use read to examine files instead of cat or sed.\n- You can inspect PI_* environment variables for current model and session details.\n- Use edit for precise changes (edits[].oldText must match exactly)\n- When changing multiple separate locations in one file, use one edit call with multiple entries in edits[] instead of multiple edit calls\n- Each edits[].oldText is matched against the original file, not after earlier edits are applied. Do not emit overlapping or nested edits. Merge nearby changes into one edit.\n- Keep edits[].oldText as small as possible while still being unique in the file. Do not pad with large unchanged regions.\n- Use write only for new files or complete rewrites"},"tools":[{"description":"Read the contents of a file. Supports text files and images (jpg, png, gif, webp, bmp). Images are sent as attachments. For text files, output is truncated to 2000 lines or 50KB (whichever is hit first). Use offset/limit for large files. When you need the full file, continue with offset until complete.","name":"read","parameters":{"properties":{"limit":{"description":"Maximum number of lines to read","type":"number"},"offset":{"description":"Line number to start reading from (1-indexed)","type":"number"},"path":{"description":"Path to the file to read (relative or absolute)","type":"string"}},"required":["path"],"type":"object"}},{"description":"Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.","name":"bash","parameters":{"properties":{"command":{"description":"Shell command to execute","type":"string"},"timeout":{"description":"Timeout in seconds (optional, no default timeout)","type":"number"}},"required":["command"],"type":"object"}},{"description":"Edit a single file using exact text replacement. Every edits[].oldText must match a unique, non-overlapping region of the original file. If two changes affect the same block or nearby lines, merge them into one edit instead of emitting overlapping edits. Do not include large unchanged regions just to connect distant changes.","name":"edit","parameters":{"properties":{"edits":{"description":"One or more targeted replacements. Each edit is matched against the original file, not incrementally. Do not include overlapping or nested edits. If two changes touch the same block or nearby lines, merge them into one edit instead.","items":{"properties":{"newText":{"description":"Replacement text for this targeted edit.","type":"string"},"oldText":{"description":"Exact text for one targeted replacement. It must be unique in the original file and must not overlap with any other edits[].oldText in the same call.","type":"string"}},"required":["oldText","newText"],"type":"object"},"type":"array"},"path":{"description":"Path to the file to edit (relative or absolute)","type":"string"}},"required":["path","edits"],"type":"object"}},{"description":"Write content to a file. Creates the file if it doesn't exist, overwrites if it does. Automatically creates parent directories.","name":"write","parameters":{"properties":{"content":{"description":"Content to write to the file","type":"string"},"path":{"description":"Path to the file to write (relative or absolute)","type":"string"}},"required":["path","content"],"type":"object"}},{"description":"Ask TypeSafe Jev typed questions about a piece of text and get calibrated answers (probabilities, a chosen option, a rubric score) instead of prose.","name":"jev_ask","parameters":{"properties":{"questions":{"description":"One or more questions. All are evaluated in parallel against the same state.","items":{"properties":{"id":{"description":"Short key for this question. The answer comes back under it.","type":"string"},"instructions":{"description":"The one thing to judge. One specific, well-scoped gut-check per question.","type":"string"},"levels":{"description":"score only: ordered rubric levels, lowest first, at least two","items":{"type":"string"},"type":"array"},"options":{"description":"choice only: the options to choose between","items":{"properties":{"description":{"description":"When this option applies","type":"string"},"name":{"description":"Option key","type":"string"}},"required":["name"],"type":"object"},"type":"array"},"type":{"description":"noul = yes/no probability, choice = pick one option, score = value on a rubric","enum":["noul","choice","score"],"type":"string"}},"required":["id","type","instructions"],"type":"object"},"type":"array"},"state":{"description":"The text to judge: tool output, a diff, a message, a document excerpt.","type":"string"}},"required":["state","questions"],"type":"object"}}]}}
+{"step":"01-prompt","ch":"host","data":{"message":{"content":[{"arguments":{"command":"echo hi"},"name":"bash","type":"toolCall"}],"role":"assistant","stopReason":"toolUse"},"type":"message_end"}}
+{"step":"01-prompt","ch":"host","data":{"args":{"command":"echo hi"},"toolName":"bash","type":"tool_execution_start"}}
+{"step":"01-prompt","ch":"http","data":{"body":{"model":"jev-eq","questions":{"beyond_scope":{"criteria":{"false":"Confined to what the user asked for","true":"Touches things the user did not ask about"},"instructions":"Does this action affect anything beyond the change the user asked for?","type":"noul"},"destructive":{"criteria":{"false":"Nothing is deleted or overwritten","true":"Deletes or overwrites data, or rewrites shared history"},"instructions":"Is this action destructive?","type":"noul"},"exfiltration":{"criteria":{"false":"Nothing leaves the machine","true":"Local data or secrets leave the machine"},"instructions":"Does this action send local file contents, credentials, or environment variables to a network destination outside the project's own tooling?","type":"noul"},"impact":{"criteria":["None, it only reads","Small, one file or one reversible change","Large, many files or shared state","Severe, data loss or a forced overwrite of shared history"],"instructions":"How much damage would this action do if the user did not want it?","type":"score"}},"state":{"arguments":{"command":"echo hi"},"cwd":"<cwd>","platform":"linux","tool":"bash","user_request":"please run echo hi"}},"headers":{"authorization":"Bearer tsk-eq-key-0123456789","content-type":"application/json"},"matched":true,"method":"POST","path":"/v1/systemone","query":{},"server":"jev"}}
+{"step":"01-prompt","ch":"ui","data":{"method":"setStatus","statusKey":"jev","statusText":"jev: destructive 0.99, exfiltration 0.79, beyond_scope 0.98, impact 3.00/3 at confidence 0.91"}}
+{"step":"01-prompt","ch":"ui","data":{"message":"bash\ndestructive 0.99, exfiltration 0.79, beyond_scope 0.98, impact 3.00/3 at confidence 0.91\n\nRun it anyway?","method":"confirm","title":"Jev flagged this tool call"}}
+{"step":"01-prompt","ch":"http","data":{"body":{"model":"jev-eq","questions":{"failure_class":{"criteria":{"code_bug":"The code or types are wrong","environment":"A missing dependency, port, or tool in the local setup","no_failure":"Output reports success or nothing wrong","permission":"Access was denied by the OS or a server","transient":"A network or resource hiccup that may succeed on a retry","user_error":"The command itself was invoked wrongly"},"instructions":"What kind of failure is `output` reporting?","type":"choice"},"leaks_secret":{"criteria":{"false":"Only ordinary program output","true":"A key, token, password, or private key appears in the text"},"instructions":"Does `output` contain a secret or credential that must not be written to a session transcript?","type":"noul"}},"state":{"arguments":{"command":"echo hi"},"cwd":"<cwd>","is_error":false,"output":"hi\n","tool":"bash"}},"headers":{"authorization":"Bearer tsk-eq-key-0123456789","content-type":"application/json"},"matched":true,"method":"POST","path":"/v1/systemone","query":{},"server":"jev"}}
+{"step":"01-prompt","ch":"host","data":{"isError":false,"result":{"content":[{"text":"hi\n","type":"text"}],"structuredContent":{"exit_code":0,"output":"hi\n","truncated":false,"wall_time_seconds":"<seconds>"}},"toolName":"bash","type":"tool_execution_end"}}
+{"step":"01-prompt","ch":"host","data":{"message":{"content":[{"text":"hi\n","type":"text"}],"isError":false,"role":"toolResult","toolName":"bash"},"type":"message_end"}}
+{"step":"01-prompt","ch":"host","data":{"type":"turn_end"}}
+{"step":"01-prompt","ch":"host","data":{"type":"turn_start"}}
+{"step":"01-prompt","ch":"llm","data":{"messages":[{"role":"user","text":"please run echo hi"},{"role":"assistant","text":"","toolCalls":[{"arguments":"{\"command\":\"echo hi\"}","name":"bash"}]},{"role":"tool","text":"hi\n"}],"system":{"inserted":"- jev_ask: Ask Jev typed questions (yes/no, choice, rubric) about text and get calibrated answers\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n</tools>\n\n<rules>\n- Use bash for file operations like ls, rg, find\n- Use read to examine files instead of cat or sed.\n- You can inspect PI_* environment variables for current model and session details.\n- Use edit for precise changes (edits[].oldText must match exactly)\n- When changing multiple separate locations in one file, use one edit call with multiple entries in edits[] instead of multiple edit calls\n- Each edits[].oldText is matched against the original file, not after earlier edits are applied. Do not emit overlapping or nested edits. Merge nearby changes into one edit.\n- Keep edits[].oldText as small as possible while still being unique in the file. Do not pad with large unchanged regions.\n- Use write only for new files or complete rewrites.\n- Use jev_ask when a judgement must be typed and calibrated rather than written: classification, relevance, yes/no checks, rubric scores.\n- Ask one specific question per entry in jev_ask; split multi-factor judgements into separate questions and combine the answers yourself","removed":"\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n</tools>\n\n<rules>\n- Use bash for file operations like ls, rg, find\n- Use read to examine files instead of cat or sed.\n- You can inspect PI_* environment variables for current model and session details.\n- Use edit for precise changes (edits[].oldText must match exactly)\n- When changing multiple separate locations in one file, use one edit call with multiple entries in edits[] instead of multiple edit calls\n- Each edits[].oldText is matched against the original file, not after earlier edits are applied. Do not emit overlapping or nested edits. Merge nearby changes into one edit.\n- Keep edits[].oldText as small as possible while still being unique in the file. Do not pad with large unchanged regions.\n- Use write only for new files or complete rewrites"},"tools":[{"description":"Read the contents of a file. Supports text files and images (jpg, png, gif, webp, bmp). Images are sent as attachments. For text files, output is truncated to 2000 lines or 50KB (whichever is hit first). Use offset/limit for large files. When you need the full file, continue with offset until complete.","name":"read","parameters":{"properties":{"limit":{"description":"Maximum number of lines to read","type":"number"},"offset":{"description":"Line number to start reading from (1-indexed)","type":"number"},"path":{"description":"Path to the file to read (relative or absolute)","type":"string"}},"required":["path"],"type":"object"}},{"description":"Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.","name":"bash","parameters":{"properties":{"command":{"description":"Shell command to execute","type":"string"},"timeout":{"description":"Timeout in seconds (optional, no default timeout)","type":"number"}},"required":["command"],"type":"object"}},{"description":"Edit a single file using exact text replacement. Every edits[].oldText must match a unique, non-overlapping region of the original file. If two changes affect the same block or nearby lines, merge them into one edit instead of emitting overlapping edits. Do not include large unchanged regions just to connect distant changes.","name":"edit","parameters":{"properties":{"edits":{"description":"One or more targeted replacements. Each edit is matched against the original file, not incrementally. Do not include overlapping or nested edits. If two changes touch the same block or nearby lines, merge them into one edit instead.","items":{"properties":{"newText":{"description":"Replacement text for this targeted edit.","type":"string"},"oldText":{"description":"Exact text for one targeted replacement. It must be unique in the original file and must not overlap with any other edits[].oldText in the same call.","type":"string"}},"required":["oldText","newText"],"type":"object"},"type":"array"},"path":{"description":"Path to the file to edit (relative or absolute)","type":"string"}},"required":["path","edits"],"type":"object"}},{"description":"Write content to a file. Creates the file if it doesn't exist, overwrites if it does. Automatically creates parent directories.","name":"write","parameters":{"properties":{"content":{"description":"Content to write to the file","type":"string"},"path":{"description":"Path to the file to write (relative or absolute)","type":"string"}},"required":["path","content"],"type":"object"}},{"description":"Ask TypeSafe Jev typed questions about a piece of text and get calibrated answers (probabilities, a chosen option, a rubric score) instead of prose.","name":"jev_ask","parameters":{"properties":{"questions":{"description":"One or more questions. All are evaluated in parallel against the same state.","items":{"properties":{"id":{"description":"Short key for this question. The answer comes back under it.","type":"string"},"instructions":{"description":"The one thing to judge. One specific, well-scoped gut-check per question.","type":"string"},"levels":{"description":"score only: ordered rubric levels, lowest first, at least two","items":{"type":"string"},"type":"array"},"options":{"description":"choice only: the options to choose between","items":{"properties":{"description":{"description":"When this option applies","type":"string"},"name":{"description":"Option key","type":"string"}},"required":["name"],"type":"object"},"type":"array"},"type":{"description":"noul = yes/no probability, choice = pick one option, score = value on a rubric","enum":["noul","choice","score"],"type":"string"}},"required":["id","type","instructions"],"type":"object"},"type":"array"},"state":{"description":"The text to judge: tool output, a diff, a message, a document excerpt.","type":"string"}},"required":["state","questions"],"type":"object"}}]}}
+{"step":"01-prompt","ch":"host","data":{"message":{"content":[{"text":"done","type":"text"}],"role":"assistant","stopReason":"stop"},"type":"message_end"}}
+{"step":"01-prompt","ch":"host","data":{"type":"turn_end"}}
+{"step":"01-prompt","ch":"host","data":{"type":"agent_end"}}
+{"step":"99-shutdown","ch":"exit","data":{"code":0}}

package/port/golden/enforce-decline.jsonl

@@ -0,0 +1,22 @@
+{"kind":"header","scenario":"enforce-decline","lane":"pi-ts","host":"pi 1.0.1","extension":"sha256:7fecf89390f53ec308d08d182796b1340e4c9553ac5e6cbdc2df19d9ad1b63e0","normalizer":"v3"}
+{"step":"00-startup","ch":"ui","data":{"method":"setStatus","statusKey":"jev","statusText":"jev: enforce"}}
+{"step":"01-prompt","ch":"response","data":{"command":"prompt","data":{"disposition":"started"},"success":true}}
+{"step":"01-prompt","ch":"host","data":{"type":"agent_start"}}
+{"step":"01-prompt","ch":"host","data":{"type":"turn_start"}}
+{"step":"01-prompt","ch":"host","data":{"message":{"content":"","role":"system"},"type":"message_end"}}
+{"step":"01-prompt","ch":"host","data":{"message":{"content":[{"text":"please run echo hi","type":"text"}],"role":"user"},"type":"message_end"}}
+{"step":"01-prompt","ch":"llm","data":{"messages":[{"role":"user","text":"please run echo hi"}],"system":{"inserted":"- jev_ask: Ask Jev typed questions (yes/no, choice, rubric) about text and get calibrated answers\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n</tools>\n\n<rules>\n- Use bash for file operations like ls, rg, find\n- Use read to examine files instead of cat or sed.\n- You can inspect PI_* environment variables for current model and session details.\n- Use edit for precise changes (edits[].oldText must match exactly)\n- When changing multiple separate locations in one file, use one edit call with multiple entries in edits[] instead of multiple edit calls\n- Each edits[].oldText is matched against the original file, not after earlier edits are applied. Do not emit overlapping or nested edits. Merge nearby changes into one edit.\n- Keep edits[].oldText as small as possible while still being unique in the file. Do not pad with large unchanged regions.\n- Use write only for new files or complete rewrites.\n- Use jev_ask when a judgement must be typed and calibrated rather than written: classification, relevance, yes/no checks, rubric scores.\n- Ask one specific question per entry in jev_ask; split multi-factor judgements into separate questions and combine the answers yourself","removed":"\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n</tools>\n\n<rules>\n- Use bash for file operations like ls, rg, find\n- Use read to examine files instead of cat or sed.\n- You can inspect PI_* environment variables for current model and session details.\n- Use edit for precise changes (edits[].oldText must match exactly)\n- When changing multiple separate locations in one file, use one edit call with multiple entries in edits[] instead of multiple edit calls\n- Each edits[].oldText is matched against the original file, not after earlier edits are applied. Do not emit overlapping or nested edits. Merge nearby changes into one edit.\n- Keep edits[].oldText as small as possible while still being unique in the file. Do not pad with large unchanged regions.\n- Use write only for new files or complete rewrites"},"tools":[{"description":"Read the contents of a file. Supports text files and images (jpg, png, gif, webp, bmp). Images are sent as attachments. For text files, output is truncated to 2000 lines or 50KB (whichever is hit first). Use offset/limit for large files. When you need the full file, continue with offset until complete.","name":"read","parameters":{"properties":{"limit":{"description":"Maximum number of lines to read","type":"number"},"offset":{"description":"Line number to start reading from (1-indexed)","type":"number"},"path":{"description":"Path to the file to read (relative or absolute)","type":"string"}},"required":["path"],"type":"object"}},{"description":"Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.","name":"bash","parameters":{"properties":{"command":{"description":"Shell command to execute","type":"string"},"timeout":{"description":"Timeout in seconds (optional, no default timeout)","type":"number"}},"required":["command"],"type":"object"}},{"description":"Edit a single file using exact text replacement. Every edits[].oldText must match a unique, non-overlapping region of the original file. If two changes affect the same block or nearby lines, merge them into one edit instead of emitting overlapping edits. Do not include large unchanged regions just to connect distant changes.","name":"edit","parameters":{"properties":{"edits":{"description":"One or more targeted replacements. Each edit is matched against the original file, not incrementally. Do not include overlapping or nested edits. If two changes touch the same block or nearby lines, merge them into one edit instead.","items":{"properties":{"newText":{"description":"Replacement text for this targeted edit.","type":"string"},"oldText":{"description":"Exact text for one targeted replacement. It must be unique in the original file and must not overlap with any other edits[].oldText in the same call.","type":"string"}},"required":["oldText","newText"],"type":"object"},"type":"array"},"path":{"description":"Path to the file to edit (relative or absolute)","type":"string"}},"required":["path","edits"],"type":"object"}},{"description":"Write content to a file. Creates the file if it doesn't exist, overwrites if it does. Automatically creates parent directories.","name":"write","parameters":{"properties":{"content":{"description":"Content to write to the file","type":"string"},"path":{"description":"Path to the file to write (relative or absolute)","type":"string"}},"required":["path","content"],"type":"object"}},{"description":"Ask TypeSafe Jev typed questions about a piece of text and get calibrated answers (probabilities, a chosen option, a rubric score) instead of prose.","name":"jev_ask","parameters":{"properties":{"questions":{"description":"One or more questions. All are evaluated in parallel against the same state.","items":{"properties":{"id":{"description":"Short key for this question. The answer comes back under it.","type":"string"},"instructions":{"description":"The one thing to judge. One specific, well-scoped gut-check per question.","type":"string"},"levels":{"description":"score only: ordered rubric levels, lowest first, at least two","items":{"type":"string"},"type":"array"},"options":{"description":"choice only: the options to choose between","items":{"properties":{"description":{"description":"When this option applies","type":"string"},"name":{"description":"Option key","type":"string"}},"required":["name"],"type":"object"},"type":"array"},"type":{"description":"noul = yes/no probability, choice = pick one option, score = value on a rubric","enum":["noul","choice","score"],"type":"string"}},"required":["id","type","instructions"],"type":"object"},"type":"array"},"state":{"description":"The text to judge: tool output, a diff, a message, a document excerpt.","type":"string"}},"required":["state","questions"],"type":"object"}}]}}
+{"step":"01-prompt","ch":"host","data":{"message":{"content":[{"arguments":{"command":"echo hi"},"name":"bash","type":"toolCall"}],"role":"assistant","stopReason":"toolUse"},"type":"message_end"}}
+{"step":"01-prompt","ch":"host","data":{"args":{"command":"echo hi"},"toolName":"bash","type":"tool_execution_start"}}
+{"step":"01-prompt","ch":"http","data":{"body":{"model":"jev-eq","questions":{"beyond_scope":{"criteria":{"false":"Confined to what the user asked for","true":"Touches things the user did not ask about"},"instructions":"Does this action affect anything beyond the change the user asked for?","type":"noul"},"destructive":{"criteria":{"false":"Nothing is deleted or overwritten","true":"Deletes or overwrites data, or rewrites shared history"},"instructions":"Is this action destructive?","type":"noul"},"exfiltration":{"criteria":{"false":"Nothing leaves the machine","true":"Local data or secrets leave the machine"},"instructions":"Does this action send local file contents, credentials, or environment variables to a network destination outside the project's own tooling?","type":"noul"},"impact":{"criteria":["None, it only reads","Small, one file or one reversible change","Large, many files or shared state","Severe, data loss or a forced overwrite of shared history"],"instructions":"How much damage would this action do if the user did not want it?","type":"score"}},"state":{"arguments":{"command":"echo hi"},"cwd":"<cwd>","platform":"linux","tool":"bash","user_request":"please run echo hi"}},"headers":{"authorization":"Bearer tsk-eq-key-0123456789","content-type":"application/json"},"matched":true,"method":"POST","path":"/v1/systemone","query":{},"server":"jev"}}
+{"step":"01-prompt","ch":"ui","data":{"method":"setStatus","statusKey":"jev","statusText":"jev: destructive 0.99, exfiltration 0.79, beyond_scope 0.98, impact 3.00/3 at confidence 0.91"}}
+{"step":"01-prompt","ch":"ui","data":{"message":"bash\ndestructive 0.99, exfiltration 0.79, beyond_scope 0.98, impact 3.00/3 at confidence 0.91\n\nRun it anyway?","method":"confirm","title":"Jev flagged this tool call"}}
+{"step":"01-prompt","ch":"host","data":{"isError":true,"result":{"content":[{"text":"pi-jev: destructive 0.99, exfiltration 0.79, beyond_scope 0.98, impact 3.00/3 at confidence 0.91 (declined)","type":"text"}],"details":{}},"toolName":"bash","type":"tool_execution_end"}}
+{"step":"01-prompt","ch":"host","data":{"message":{"content":[{"text":"pi-jev: destructive 0.99, exfiltration 0.79, beyond_scope 0.98, impact 3.00/3 at confidence 0.91 (declined)","type":"text"}],"details":{},"isError":true,"role":"toolResult","toolName":"bash"},"type":"message_end"}}
+{"step":"01-prompt","ch":"host","data":{"type":"turn_end"}}
+{"step":"01-prompt","ch":"host","data":{"type":"turn_start"}}
+{"step":"01-prompt","ch":"llm","data":{"messages":[{"role":"user","text":"please run echo hi"},{"role":"assistant","text":"","toolCalls":[{"arguments":"{\"command\":\"echo hi\"}","name":"bash"}]},{"role":"tool","text":"pi-jev: destructive 0.99, exfiltration 0.79, beyond_scope 0.98, impact 3.00/3 at confidence 0.91 (declined)"}],"system":{"inserted":"- jev_ask: Ask Jev typed questions (yes/no, choice, rubric) about text and get calibrated answers\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n</tools>\n\n<rules>\n- Use bash for file operations like ls, rg, find\n- Use read to examine files instead of cat or sed.\n- You can inspect PI_* environment variables for current model and session details.\n- Use edit for precise changes (edits[].oldText must match exactly)\n- When changing multiple separate locations in one file, use one edit call with multiple entries in edits[] instead of multiple edit calls\n- Each edits[].oldText is matched against the original file, not after earlier edits are applied. Do not emit overlapping or nested edits. Merge nearby changes into one edit.\n- Keep edits[].oldText as small as possible while still being unique in the file. Do not pad with large unchanged regions.\n- Use write only for new files or complete rewrites.\n- Use jev_ask when a judgement must be typed and calibrated rather than written: classification, relevance, yes/no checks, rubric scores.\n- Ask one specific question per entry in jev_ask; split multi-factor judgements into separate questions and combine the answers yourself","removed":"\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n</tools>\n\n<rules>\n- Use bash for file operations like ls, rg, find\n- Use read to examine files instead of cat or sed.\n- You can inspect PI_* environment variables for current model and session details.\n- Use edit for precise changes (edits[].oldText must match exactly)\n- When changing multiple separate locations in one file, use one edit call with multiple entries in edits[] instead of multiple edit calls\n- Each edits[].oldText is matched against the original file, not after earlier edits are applied. Do not emit overlapping or nested edits. Merge nearby changes into one edit.\n- Keep edits[].oldText as small as possible while still being unique in the file. Do not pad with large unchanged regions.\n- Use write only for new files or complete rewrites"},"tools":[{"description":"Read the contents of a file. Supports text files and images (jpg, png, gif, webp, bmp). Images are sent as attachments. For text files, output is truncated to 2000 lines or 50KB (whichever is hit first). Use offset/limit for large files. When you need the full file, continue with offset until complete.","name":"read","parameters":{"properties":{"limit":{"description":"Maximum number of lines to read","type":"number"},"offset":{"description":"Line number to start reading from (1-indexed)","type":"number"},"path":{"description":"Path to the file to read (relative or absolute)","type":"string"}},"required":["path"],"type":"object"}},{"description":"Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.","name":"bash","parameters":{"properties":{"command":{"description":"Shell command to execute","type":"string"},"timeout":{"description":"Timeout in seconds (optional, no default timeout)","type":"number"}},"required":["command"],"type":"object"}},{"description":"Edit a single file using exact text replacement. Every edits[].oldText must match a unique, non-overlapping region of the original file. If two changes affect the same block or nearby lines, merge them into one edit instead of emitting overlapping edits. Do not include large unchanged regions just to connect distant changes.","name":"edit","parameters":{"properties":{"edits":{"description":"One or more targeted replacements. Each edit is matched against the original file, not incrementally. Do not include overlapping or nested edits. If two changes touch the same block or nearby lines, merge them into one edit instead.","items":{"properties":{"newText":{"description":"Replacement text for this targeted edit.","type":"string"},"oldText":{"description":"Exact text for one targeted replacement. It must be unique in the original file and must not overlap with any other edits[].oldText in the same call.","type":"string"}},"required":["oldText","newText"],"type":"object"},"type":"array"},"path":{"description":"Path to the file to edit (relative or absolute)","type":"string"}},"required":["path","edits"],"type":"object"}},{"description":"Write content to a file. Creates the file if it doesn't exist, overwrites if it does. Automatically creates parent directories.","name":"write","parameters":{"properties":{"content":{"description":"Content to write to the file","type":"string"},"path":{"description":"Path to the file to write (relative or absolute)","type":"string"}},"required":["path","content"],"type":"object"}},{"description":"Ask TypeSafe Jev typed questions about a piece of text and get calibrated answers (probabilities, a chosen option, a rubric score) instead of prose.","name":"jev_ask","parameters":{"properties":{"questions":{"description":"One or more questions. All are evaluated in parallel against the same state.","items":{"properties":{"id":{"description":"Short key for this question. The answer comes back under it.","type":"string"},"instructions":{"description":"The one thing to judge. One specific, well-scoped gut-check per question.","type":"string"},"levels":{"description":"score only: ordered rubric levels, lowest first, at least two","items":{"type":"string"},"type":"array"},"options":{"description":"choice only: the options to choose between","items":{"properties":{"description":{"description":"When this option applies","type":"string"},"name":{"description":"Option key","type":"string"}},"required":["name"],"type":"object"},"type":"array"},"type":{"description":"noul = yes/no probability, choice = pick one option, score = value on a rubric","enum":["noul","choice","score"],"type":"string"}},"required":["id","type","instructions"],"type":"object"},"type":"array"},"state":{"description":"The text to judge: tool output, a diff, a message, a document excerpt.","type":"string"}},"required":["state","questions"],"type":"object"}}]}}
+{"step":"01-prompt","ch":"host","data":{"message":{"content":[{"text":"blocked","type":"text"}],"role":"assistant","stopReason":"stop"},"type":"message_end"}}
+{"step":"01-prompt","ch":"host","data":{"type":"turn_end"}}
+{"step":"01-prompt","ch":"host","data":{"type":"agent_end"}}
+{"step":"99-shutdown","ch":"exit","data":{"code":0}}

package/port/golden/jev-ask.jsonl

@@ -0,0 +1,20 @@
+{"kind":"header","scenario":"jev-ask","lane":"pi-ts","host":"pi 1.0.1","extension":"sha256:7fecf89390f53ec308d08d182796b1340e4c9553ac5e6cbdc2df19d9ad1b63e0","normalizer":"v3"}
+{"step":"00-startup","ch":"ui","data":{"method":"setStatus","statusKey":"jev","statusText":"jev: shadow"}}
+{"step":"01-prompt","ch":"response","data":{"command":"prompt","data":{"disposition":"started"},"success":true}}
+{"step":"01-prompt","ch":"host","data":{"type":"agent_start"}}
+{"step":"01-prompt","ch":"host","data":{"type":"turn_start"}}
+{"step":"01-prompt","ch":"host","data":{"message":{"content":"","role":"system"},"type":"message_end"}}
+{"step":"01-prompt","ch":"host","data":{"message":{"content":[{"text":"ask jev about the diff","type":"text"}],"role":"user"},"type":"message_end"}}
+{"step":"01-prompt","ch":"llm","data":{"messages":[{"role":"user","text":"ask jev about the diff"}],"system":{"inserted":"- jev_ask: Ask Jev typed questions (yes/no, choice, rubric) about text and get calibrated answers\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n</tools>\n\n<rules>\n- Use bash for file operations like ls, rg, find\n- Use read to examine files instead of cat or sed.\n- You can inspect PI_* environment variables for current model and session details.\n- Use edit for precise changes (edits[].oldText must match exactly)\n- When changing multiple separate locations in one file, use one edit call with multiple entries in edits[] instead of multiple edit calls\n- Each edits[].oldText is matched against the original file, not after earlier edits are applied. Do not emit overlapping or nested edits. Merge nearby changes into one edit.\n- Keep edits[].oldText as small as possible while still being unique in the file. Do not pad with large unchanged regions.\n- Use write only for new files or complete rewrites.\n- Use jev_ask when a judgement must be typed and calibrated rather than written: classification, relevance, yes/no checks, rubric scores.\n- Ask one specific question per entry in jev_ask; split multi-factor judgements into separate questions and combine the answers yourself","removed":"\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n</tools>\n\n<rules>\n- Use bash for file operations like ls, rg, find\n- Use read to examine files instead of cat or sed.\n- You can inspect PI_* environment variables for current model and session details.\n- Use edit for precise changes (edits[].oldText must match exactly)\n- When changing multiple separate locations in one file, use one edit call with multiple entries in edits[] instead of multiple edit calls\n- Each edits[].oldText is matched against the original file, not after earlier edits are applied. Do not emit overlapping or nested edits. Merge nearby changes into one edit.\n- Keep edits[].oldText as small as possible while still being unique in the file. Do not pad with large unchanged regions.\n- Use write only for new files or complete rewrites"},"tools":[{"description":"Read the contents of a file. Supports text files and images (jpg, png, gif, webp, bmp). Images are sent as attachments. For text files, output is truncated to 2000 lines or 50KB (whichever is hit first). Use offset/limit for large files. When you need the full file, continue with offset until complete.","name":"read","parameters":{"properties":{"limit":{"description":"Maximum number of lines to read","type":"number"},"offset":{"description":"Line number to start reading from (1-indexed)","type":"number"},"path":{"description":"Path to the file to read (relative or absolute)","type":"string"}},"required":["path"],"type":"object"}},{"description":"Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.","name":"bash","parameters":{"properties":{"command":{"description":"Shell command to execute","type":"string"},"timeout":{"description":"Timeout in seconds (optional, no default timeout)","type":"number"}},"required":["command"],"type":"object"}},{"description":"Edit a single file using exact text replacement. Every edits[].oldText must match a unique, non-overlapping region of the original file. If two changes affect the same block or nearby lines, merge them into one edit instead of emitting overlapping edits. Do not include large unchanged regions just to connect distant changes.","name":"edit","parameters":{"properties":{"edits":{"description":"One or more targeted replacements. Each edit is matched against the original file, not incrementally. Do not include overlapping or nested edits. If two changes touch the same block or nearby lines, merge them into one edit instead.","items":{"properties":{"newText":{"description":"Replacement text for this targeted edit.","type":"string"},"oldText":{"description":"Exact text for one targeted replacement. It must be unique in the original file and must not overlap with any other edits[].oldText in the same call.","type":"string"}},"required":["oldText","newText"],"type":"object"},"type":"array"},"path":{"description":"Path to the file to edit (relative or absolute)","type":"string"}},"required":["path","edits"],"type":"object"}},{"description":"Write content to a file. Creates the file if it doesn't exist, overwrites if it does. Automatically creates parent directories.","name":"write","parameters":{"properties":{"content":{"description":"Content to write to the file","type":"string"},"path":{"description":"Path to the file to write (relative or absolute)","type":"string"}},"required":["path","content"],"type":"object"}},{"description":"Ask TypeSafe Jev typed questions about a piece of text and get calibrated answers (probabilities, a chosen option, a rubric score) instead of prose.","name":"jev_ask","parameters":{"properties":{"questions":{"description":"One or more questions. All are evaluated in parallel against the same state.","items":{"properties":{"id":{"description":"Short key for this question. The answer comes back under it.","type":"string"},"instructions":{"description":"The one thing to judge. One specific, well-scoped gut-check per question.","type":"string"},"levels":{"description":"score only: ordered rubric levels, lowest first, at least two","items":{"type":"string"},"type":"array"},"options":{"description":"choice only: the options to choose between","items":{"properties":{"description":{"description":"When this option applies","type":"string"},"name":{"description":"Option key","type":"string"}},"required":["name"],"type":"object"},"type":"array"},"type":{"description":"noul = yes/no probability, choice = pick one option, score = value on a rubric","enum":["noul","choice","score"],"type":"string"}},"required":["id","type","instructions"],"type":"object"},"type":"array"},"state":{"description":"The text to judge: tool output, a diff, a message, a document excerpt.","type":"string"}},"required":["state","questions"],"type":"object"}}]}}
+{"step":"01-prompt","ch":"host","data":{"message":{"content":[{"arguments":{"questions":[{"id":"relevant","instructions":"Is this relevant?","type":"noul"},{"id":"label","instructions":"Which bucket?","options":[{"description":"Defect","name":"bug"},{"name":"feature"}],"type":"choice"},{"id":"quality","instructions":"How thorough?","levels":["Superficial","Adequate","Thorough"],"type":"score"}],"state":"the diff"},"name":"jev_ask","type":"toolCall"}],"role":"assistant","stopReason":"toolUse"},"type":"message_end"}}
+{"step":"01-prompt","ch":"host","data":{"args":{"questions":[{"id":"relevant","instructions":"Is this relevant?","type":"noul"},{"id":"label","instructions":"Which bucket?","options":[{"description":"Defect","name":"bug"},{"name":"feature"}],"type":"choice"},{"id":"quality","instructions":"How thorough?","levels":["Superficial","Adequate","Thorough"],"type":"score"}],"state":"the diff"},"toolName":"jev_ask","type":"tool_execution_start"}}
+{"step":"01-prompt","ch":"http","data":{"body":{"model":"jev-eq","questions":{"label":{"criteria":{"bug":"Defect","feature":null},"instructions":"Which bucket?","type":"choice"},"quality":{"criteria":["Superficial","Adequate","Thorough"],"instructions":"How thorough?","type":"score"},"relevant":{"instructions":"Is this relevant?","type":"noul"}},"state":"the diff"},"headers":{"authorization":"Bearer tsk-eq-key-0123456789","content-type":"application/json"},"matched":true,"method":"POST","path":"/v1/systemone","query":{},"server":"jev"}}
+{"step":"01-prompt","ch":"host","data":{"isError":false,"result":{"content":[{"text":"model jev-eq\nrelevant: yes 0.93  <- Is this relevant?\nlabel: bug (conf 0.90) [bug 0.90, feature 0.10]  <- Which bucket?\nquality: 1.75/3 (conf 0.80)  <- How thorough?\ntokens 12 in / 3 out","type":"text"}],"details":{"answers":{"label":{"choice":"bug","confidence":0.9,"probabilities":{"bug":0.9,"feature":0.1},"type":"choice"},"quality":{"confidence":0.8,"legend":{"0":"None, it only reads","1":"Small","2":"Large","3":"Severe"},"probabilities":{"0":0.05,"1":0.1,"2":0.15,"3":0.7},"score":1.75,"type":"score"},"relevant":{"noul":0.93,"type":"noul"}},"model":"jev-eq","ok":true,"usage":{"input_tokens":12,"output_tokens":3}}},"toolName":"jev_ask","type":"tool_execution_end"}}
+{"step":"01-prompt","ch":"host","data":{"message":{"content":[{"text":"model jev-eq\nrelevant: yes 0.93  <- Is this relevant?\nlabel: bug (conf 0.90) [bug 0.90, feature 0.10]  <- Which bucket?\nquality: 1.75/3 (conf 0.80)  <- How thorough?\ntokens 12 in / 3 out","type":"text"}],"details":{"answers":{"label":{"choice":"bug","confidence":0.9,"probabilities":{"bug":0.9,"feature":0.1},"type":"choice"},"quality":{"confidence":0.8,"legend":{"0":"None, it only reads","1":"Small","2":"Large","3":"Severe"},"probabilities":{"0":0.05,"1":0.1,"2":0.15,"3":0.7},"score":1.75,"type":"score"},"relevant":{"noul":0.93,"type":"noul"}},"model":"jev-eq","ok":true,"usage":{"input_tokens":12,"output_tokens":3}},"isError":false,"role":"toolResult","toolName":"jev_ask"},"type":"message_end"}}
+{"step":"01-prompt","ch":"host","data":{"type":"turn_end"}}
+{"step":"01-prompt","ch":"host","data":{"type":"turn_start"}}
+{"step":"01-prompt","ch":"llm","data":{"messages":[{"role":"user","text":"ask jev about the diff"},{"role":"assistant","text":"","toolCalls":[{"arguments":"{\"questions\":[{\"id\":\"relevant\",\"instructions\":\"Is this relevant?\",\"type\":\"noul\"},{\"id\":\"label\",\"instructions\":\"Which bucket?\",\"options\":[{\"description\":\"Defect\",\"name\":\"bug\"},{\"name\":\"feature\"}],\"type\":\"choice\"},{\"id\":\"quality\",\"instructions\":\"How thorough?\",\"levels\":[\"Superficial\",\"Adequate\",\"Thorough\"],\"type\":\"score\"}],\"state\":\"the diff\"}","name":"jev_ask"}]},{"role":"tool","text":"model jev-eq\nrelevant: yes 0.93  <- Is this relevant?\nlabel: bug (conf 0.90) [bug 0.90, feature 0.10]  <- Which bucket?\nquality: 1.75/3 (conf 0.80)  <- How thorough?\ntokens 12 in / 3 out"}],"system":{"inserted":"- jev_ask: Ask Jev typed questions (yes/no, choice, rubric) about text and get calibrated answers\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n</tools>\n\n<rules>\n- Use bash for file operations like ls, rg, find\n- Use read to examine files instead of cat or sed.\n- You can inspect PI_* environment variables for current model and session details.\n- Use edit for precise changes (edits[].oldText must match exactly)\n- When changing multiple separate locations in one file, use one edit call with multiple entries in edits[] instead of multiple edit calls\n- Each edits[].oldText is matched against the original file, not after earlier edits are applied. Do not emit overlapping or nested edits. Merge nearby changes into one edit.\n- Keep edits[].oldText as small as possible while still being unique in the file. Do not pad with large unchanged regions.\n- Use write only for new files or complete rewrites.\n- Use jev_ask when a judgement must be typed and calibrated rather than written: classification, relevance, yes/no checks, rubric scores.\n- Ask one specific question per entry in jev_ask; split multi-factor judgements into separate questions and combine the answers yourself","removed":"\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n</tools>\n\n<rules>\n- Use bash for file operations like ls, rg, find\n- Use read to examine files instead of cat or sed.\n- You can inspect PI_* environment variables for current model and session details.\n- Use edit for precise changes (edits[].oldText must match exactly)\n- When changing multiple separate locations in one file, use one edit call with multiple entries in edits[] instead of multiple edit calls\n- Each edits[].oldText is matched against the original file, not after earlier edits are applied. Do not emit overlapping or nested edits. Merge nearby changes into one edit.\n- Keep edits[].oldText as small as possible while still being unique in the file. Do not pad with large unchanged regions.\n- Use write only for new files or complete rewrites"},"tools":[{"description":"Read the contents of a file. Supports text files and images (jpg, png, gif, webp, bmp). Images are sent as attachments. For text files, output is truncated to 2000 lines or 50KB (whichever is hit first). Use offset/limit for large files. When you need the full file, continue with offset until complete.","name":"read","parameters":{"properties":{"limit":{"description":"Maximum number of lines to read","type":"number"},"offset":{"description":"Line number to start reading from (1-indexed)","type":"number"},"path":{"description":"Path to the file to read (relative or absolute)","type":"string"}},"required":["path"],"type":"object"}},{"description":"Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.","name":"bash","parameters":{"properties":{"command":{"description":"Shell command to execute","type":"string"},"timeout":{"description":"Timeout in seconds (optional, no default timeout)","type":"number"}},"required":["command"],"type":"object"}},{"description":"Edit a single file using exact text replacement. Every edits[].oldText must match a unique, non-overlapping region of the original file. If two changes affect the same block or nearby lines, merge them into one edit instead of emitting overlapping edits. Do not include large unchanged regions just to connect distant changes.","name":"edit","parameters":{"properties":{"edits":{"description":"One or more targeted replacements. Each edit is matched against the original file, not incrementally. Do not include overlapping or nested edits. If two changes touch the same block or nearby lines, merge them into one edit instead.","items":{"properties":{"newText":{"description":"Replacement text for this targeted edit.","type":"string"},"oldText":{"description":"Exact text for one targeted replacement. It must be unique in the original file and must not overlap with any other edits[].oldText in the same call.","type":"string"}},"required":["oldText","newText"],"type":"object"},"type":"array"},"path":{"description":"Path to the file to edit (relative or absolute)","type":"string"}},"required":["path","edits"],"type":"object"}},{"description":"Write content to a file. Creates the file if it doesn't exist, overwrites if it does. Automatically creates parent directories.","name":"write","parameters":{"properties":{"content":{"description":"Content to write to the file","type":"string"},"path":{"description":"Path to the file to write (relative or absolute)","type":"string"}},"required":["path","content"],"type":"object"}},{"description":"Ask TypeSafe Jev typed questions about a piece of text and get calibrated answers (probabilities, a chosen option, a rubric score) instead of prose.","name":"jev_ask","parameters":{"properties":{"questions":{"description":"One or more questions. All are evaluated in parallel against the same state.","items":{"properties":{"id":{"description":"Short key for this question. The answer comes back under it.","type":"string"},"instructions":{"description":"The one thing to judge. One specific, well-scoped gut-check per question.","type":"string"},"levels":{"description":"score only: ordered rubric levels, lowest first, at least two","items":{"type":"string"},"type":"array"},"options":{"description":"choice only: the options to choose between","items":{"properties":{"description":{"description":"When this option applies","type":"string"},"name":{"description":"Option key","type":"string"}},"required":["name"],"type":"object"},"type":"array"},"type":{"description":"noul = yes/no probability, choice = pick one option, score = value on a rubric","enum":["noul","choice","score"],"type":"string"}},"required":["id","type","instructions"],"type":"object"},"type":"array"},"state":{"description":"The text to judge: tool output, a diff, a message, a document excerpt.","type":"string"}},"required":["state","questions"],"type":"object"}}]}}
+{"step":"01-prompt","ch":"host","data":{"message":{"content":[{"text":"asked","type":"text"}],"role":"assistant","stopReason":"stop"},"type":"message_end"}}
+{"step":"01-prompt","ch":"host","data":{"type":"turn_end"}}
+{"step":"01-prompt","ch":"host","data":{"type":"agent_end"}}
+{"step":"99-shutdown","ch":"exit","data":{"code":0}}

package/port/golden/output-advice.jsonl

@@ -0,0 +1,23 @@
+{"kind":"header","scenario":"output-advice","lane":"pi-ts","host":"pi 1.0.1","extension":"sha256:7fecf89390f53ec308d08d182796b1340e4c9553ac5e6cbdc2df19d9ad1b63e0","normalizer":"v3"}
+{"step":"00-startup","ch":"ui","data":{"method":"setStatus","statusKey":"jev","statusText":"jev: shadow"}}
+{"step":"01-prompt","ch":"response","data":{"command":"prompt","data":{"disposition":"started"},"success":true}}
+{"step":"01-prompt","ch":"host","data":{"type":"agent_start"}}
+{"step":"01-prompt","ch":"host","data":{"type":"turn_start"}}
+{"step":"01-prompt","ch":"host","data":{"message":{"content":"","role":"system"},"type":"message_end"}}
+{"step":"01-prompt","ch":"host","data":{"message":{"content":[{"text":"run the flaky command","type":"text"}],"role":"user"},"type":"message_end"}}
+{"step":"01-prompt","ch":"llm","data":{"messages":[{"role":"user","text":"run the flaky command"}],"system":{"inserted":"- jev_ask: Ask Jev typed questions (yes/no, choice, rubric) about text and get calibrated answers\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n</tools>\n\n<rules>\n- Use bash for file operations like ls, rg, find\n- Use read to examine files instead of cat or sed.\n- You can inspect PI_* environment variables for current model and session details.\n- Use edit for precise changes (edits[].oldText must match exactly)\n- When changing multiple separate locations in one file, use one edit call with multiple entries in edits[] instead of multiple edit calls\n- Each edits[].oldText is matched against the original file, not after earlier edits are applied. Do not emit overlapping or nested edits. Merge nearby changes into one edit.\n- Keep edits[].oldText as small as possible while still being unique in the file. Do not pad with large unchanged regions.\n- Use write only for new files or complete rewrites.\n- Use jev_ask when a judgement must be typed and calibrated rather than written: classification, relevance, yes/no checks, rubric scores.\n- Ask one specific question per entry in jev_ask; split multi-factor judgements into separate questions and combine the answers yourself","removed":"\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n</tools>\n\n<rules>\n- Use bash for file operations like ls, rg, find\n- Use read to examine files instead of cat or sed.\n- You can inspect PI_* environment variables for current model and session details.\n- Use edit for precise changes (edits[].oldText must match exactly)\n- When changing multiple separate locations in one file, use one edit call with multiple entries in edits[] instead of multiple edit calls\n- Each edits[].oldText is matched against the original file, not after earlier edits are applied. Do not emit overlapping or nested edits. Merge nearby changes into one edit.\n- Keep edits[].oldText as small as possible while still being unique in the file. Do not pad with large unchanged regions.\n- Use write only for new files or complete rewrites"},"tools":[{"description":"Read the contents of a file. Supports text files and images (jpg, png, gif, webp, bmp). Images are sent as attachments. For text files, output is truncated to 2000 lines or 50KB (whichever is hit first). Use offset/limit for large files. When you need the full file, continue with offset until complete.","name":"read","parameters":{"properties":{"limit":{"description":"Maximum number of lines to read","type":"number"},"offset":{"description":"Line number to start reading from (1-indexed)","type":"number"},"path":{"description":"Path to the file to read (relative or absolute)","type":"string"}},"required":["path"],"type":"object"}},{"description":"Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.","name":"bash","parameters":{"properties":{"command":{"description":"Shell command to execute","type":"string"},"timeout":{"description":"Timeout in seconds (optional, no default timeout)","type":"number"}},"required":["command"],"type":"object"}},{"description":"Edit a single file using exact text replacement. Every edits[].oldText must match a unique, non-overlapping region of the original file. If two changes affect the same block or nearby lines, merge them into one edit instead of emitting overlapping edits. Do not include large unchanged regions just to connect distant changes.","name":"edit","parameters":{"properties":{"edits":{"description":"One or more targeted replacements. Each edit is matched against the original file, not incrementally. Do not include overlapping or nested edits. If two changes touch the same block or nearby lines, merge them into one edit instead.","items":{"properties":{"newText":{"description":"Replacement text for this targeted edit.","type":"string"},"oldText":{"description":"Exact text for one targeted replacement. It must be unique in the original file and must not overlap with any other edits[].oldText in the same call.","type":"string"}},"required":["oldText","newText"],"type":"object"},"type":"array"},"path":{"description":"Path to the file to edit (relative or absolute)","type":"string"}},"required":["path","edits"],"type":"object"}},{"description":"Write content to a file. Creates the file if it doesn't exist, overwrites if it does. Automatically creates parent directories.","name":"write","parameters":{"properties":{"content":{"description":"Content to write to the file","type":"string"},"path":{"description":"Path to the file to write (relative or absolute)","type":"string"}},"required":["path","content"],"type":"object"}},{"description":"Ask TypeSafe Jev typed questions about a piece of text and get calibrated answers (probabilities, a chosen option, a rubric score) instead of prose.","name":"jev_ask","parameters":{"properties":{"questions":{"description":"One or more questions. All are evaluated in parallel against the same state.","items":{"properties":{"id":{"description":"Short key for this question. The answer comes back under it.","type":"string"},"instructions":{"description":"The one thing to judge. One specific, well-scoped gut-check per question.","type":"string"},"levels":{"description":"score only: ordered rubric levels, lowest first, at least two","items":{"type":"string"},"type":"array"},"options":{"description":"choice only: the options to choose between","items":{"properties":{"description":{"description":"When this option applies","type":"string"},"name":{"description":"Option key","type":"string"}},"required":["name"],"type":"object"},"type":"array"},"type":{"description":"noul = yes/no probability, choice = pick one option, score = value on a rubric","enum":["noul","choice","score"],"type":"string"}},"required":["id","type","instructions"],"type":"object"},"type":"array"},"state":{"description":"The text to judge: tool output, a diff, a message, a document excerpt.","type":"string"}},"required":["state","questions"],"type":"object"}}]}}
+{"step":"01-prompt","ch":"host","data":{"message":{"content":[{"arguments":{"command":"echo timeout; exit 3"},"name":"bash","type":"toolCall"}],"role":"assistant","stopReason":"toolUse"},"type":"message_end"}}
+{"step":"01-prompt","ch":"host","data":{"args":{"command":"echo timeout; exit 3"},"toolName":"bash","type":"tool_execution_start"}}
+{"step":"01-prompt","ch":"http","data":{"body":{"model":"jev-eq","questions":{"beyond_scope":{"criteria":{"false":"Confined to what the user asked for","true":"Touches things the user did not ask about"},"instructions":"Does this action affect anything beyond the change the user asked for?","type":"noul"},"destructive":{"criteria":{"false":"Nothing is deleted or overwritten","true":"Deletes or overwrites data, or rewrites shared history"},"instructions":"Is this action destructive?","type":"noul"},"exfiltration":{"criteria":{"false":"Nothing leaves the machine","true":"Local data or secrets leave the machine"},"instructions":"Does this action send local file contents, credentials, or environment variables to a network destination outside the project's own tooling?","type":"noul"},"impact":{"criteria":["None, it only reads","Small, one file or one reversible change","Large, many files or shared state","Severe, data loss or a forced overwrite of shared history"],"instructions":"How much damage would this action do if the user did not want it?","type":"score"}},"state":{"arguments":{"command":"echo timeout; exit 3"},"cwd":"<cwd>","platform":"linux","tool":"bash","user_request":"run the flaky command"}},"headers":{"authorization":"Bearer tsk-eq-key-0123456789","content-type":"application/json"},"matched":true,"method":"POST","path":"/v1/systemone","query":{},"server":"jev"}}
+{"step":"01-prompt","ch":"ui","data":{"method":"setStatus","statusKey":"jev","statusText":"jev: clear (shadow)"}}
+{"step":"01-prompt","ch":"http","data":{"body":{"model":"jev-eq","questions":{"failure_class":{"criteria":{"code_bug":"The code or types are wrong","environment":"A missing dependency, port, or tool in the local setup","no_failure":"Output reports success or nothing wrong","permission":"Access was denied by the OS or a server","transient":"A network or resource hiccup that may succeed on a retry","user_error":"The command itself was invoked wrongly"},"instructions":"What kind of failure is `output` reporting?","type":"choice"},"leaks_secret":{"criteria":{"false":"Only ordinary program output","true":"A key, token, password, or private key appears in the text"},"instructions":"Does `output` contain a secret or credential that must not be written to a session transcript?","type":"noul"}},"state":{"arguments":{"command":"echo timeout; exit 3"},"cwd":"<cwd>","is_error":true,"output":"timeout\n\n\nCommand exited with code 3","tool":"bash"}},"headers":{"authorization":"Bearer tsk-eq-key-0123456789","content-type":"application/json"},"matched":true,"method":"POST","path":"/v1/systemone","query":{},"server":"jev"}}
+{"step":"01-prompt","ch":"ui","data":{"method":"setStatus","statusKey":"jev","statusText":"jev: advice (bash)"}}
+{"step":"01-prompt","ch":"host","data":{"isError":true,"result":{"content":[{"text":"timeout\n\n\nCommand exited with code 3","type":"text"},{"text":"[pi-jev] Jev read this as a transient failure (confidence 1.00): retrying the same command unchanged is reasonable.","type":"text"}],"isError":true},"toolName":"bash","type":"tool_execution_end"}}
+{"step":"01-prompt","ch":"host","data":{"message":{"content":[{"text":"timeout\n\n\nCommand exited with code 3","type":"text"},{"text":"[pi-jev] Jev read this as a transient failure (confidence 1.00): retrying the same command unchanged is reasonable.","type":"text"}],"isError":true,"role":"toolResult","toolName":"bash"},"type":"message_end"}}
+{"step":"01-prompt","ch":"host","data":{"type":"turn_end"}}
+{"step":"01-prompt","ch":"host","data":{"type":"turn_start"}}
+{"step":"01-prompt","ch":"llm","data":{"messages":[{"role":"user","text":"run the flaky command"},{"role":"assistant","text":"","toolCalls":[{"arguments":"{\"command\":\"echo timeout; exit 3\"}","name":"bash"}]},{"role":"tool","text":"timeout\n\n\nCommand exited with code 3\n[pi-jev] Jev read this as a transient failure (confidence 1.00): retrying the same command unchanged is reasonable."}],"system":{"inserted":"- jev_ask: Ask Jev typed questions (yes/no, choice, rubric) about text and get calibrated answers\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n</tools>\n\n<rules>\n- Use bash for file operations like ls, rg, find\n- Use read to examine files instead of cat or sed.\n- You can inspect PI_* environment variables for current model and session details.\n- Use edit for precise changes (edits[].oldText must match exactly)\n- When changing multiple separate locations in one file, use one edit call with multiple entries in edits[] instead of multiple edit calls\n- Each edits[].oldText is matched against the original file, not after earlier edits are applied. Do not emit overlapping or nested edits. Merge nearby changes into one edit.\n- Keep edits[].oldText as small as possible while still being unique in the file. Do not pad with large unchanged regions.\n- Use write only for new files or complete rewrites.\n- Use jev_ask when a judgement must be typed and calibrated rather than written: classification, relevance, yes/no checks, rubric scores.\n- Ask one specific question per entry in jev_ask; split multi-factor judgements into separate questions and combine the answers yourself","removed":"\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n</tools>\n\n<rules>\n- Use bash for file operations like ls, rg, find\n- Use read to examine files instead of cat or sed.\n- You can inspect PI_* environment variables for current model and session details.\n- Use edit for precise changes (edits[].oldText must match exactly)\n- When changing multiple separate locations in one file, use one edit call with multiple entries in edits[] instead of multiple edit calls\n- Each edits[].oldText is matched against the original file, not after earlier edits are applied. Do not emit overlapping or nested edits. Merge nearby changes into one edit.\n- Keep edits[].oldText as small as possible while still being unique in the file. Do not pad with large unchanged regions.\n- Use write only for new files or complete rewrites"},"tools":[{"description":"Read the contents of a file. Supports text files and images (jpg, png, gif, webp, bmp). Images are sent as attachments. For text files, output is truncated to 2000 lines or 50KB (whichever is hit first). Use offset/limit for large files. When you need the full file, continue with offset until complete.","name":"read","parameters":{"properties":{"limit":{"description":"Maximum number of lines to read","type":"number"},"offset":{"description":"Line number to start reading from (1-indexed)","type":"number"},"path":{"description":"Path to the file to read (relative or absolute)","type":"string"}},"required":["path"],"type":"object"}},{"description":"Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.","name":"bash","parameters":{"properties":{"command":{"description":"Shell command to execute","type":"string"},"timeout":{"description":"Timeout in seconds (optional, no default timeout)","type":"number"}},"required":["command"],"type":"object"}},{"description":"Edit a single file using exact text replacement. Every edits[].oldText must match a unique, non-overlapping region of the original file. If two changes affect the same block or nearby lines, merge them into one edit instead of emitting overlapping edits. Do not include large unchanged regions just to connect distant changes.","name":"edit","parameters":{"properties":{"edits":{"description":"One or more targeted replacements. Each edit is matched against the original file, not incrementally. Do not include overlapping or nested edits. If two changes touch the same block or nearby lines, merge them into one edit instead.","items":{"properties":{"newText":{"description":"Replacement text for this targeted edit.","type":"string"},"oldText":{"description":"Exact text for one targeted replacement. It must be unique in the original file and must not overlap with any other edits[].oldText in the same call.","type":"string"}},"required":["oldText","newText"],"type":"object"},"type":"array"},"path":{"description":"Path to the file to edit (relative or absolute)","type":"string"}},"required":["path","edits"],"type":"object"}},{"description":"Write content to a file. Creates the file if it doesn't exist, overwrites if it does. Automatically creates parent directories.","name":"write","parameters":{"properties":{"content":{"description":"Content to write to the file","type":"string"},"path":{"description":"Path to the file to write (relative or absolute)","type":"string"}},"required":["path","content"],"type":"object"}},{"description":"Ask TypeSafe Jev typed questions about a piece of text and get calibrated answers (probabilities, a chosen option, a rubric score) instead of prose.","name":"jev_ask","parameters":{"properties":{"questions":{"description":"One or more questions. All are evaluated in parallel against the same state.","items":{"properties":{"id":{"description":"Short key for this question. The answer comes back under it.","type":"string"},"instructions":{"description":"The one thing to judge. One specific, well-scoped gut-check per question.","type":"string"},"levels":{"description":"score only: ordered rubric levels, lowest first, at least two","items":{"type":"string"},"type":"array"},"options":{"description":"choice only: the options to choose between","items":{"properties":{"description":{"description":"When this option applies","type":"string"},"name":{"description":"Option key","type":"string"}},"required":["name"],"type":"object"},"type":"array"},"type":{"description":"noul = yes/no probability, choice = pick one option, score = value on a rubric","enum":["noul","choice","score"],"type":"string"}},"required":["id","type","instructions"],"type":"object"},"type":"array"},"state":{"description":"The text to judge: tool output, a diff, a message, a document excerpt.","type":"string"}},"required":["state","questions"],"type":"object"}}]}}
+{"step":"01-prompt","ch":"host","data":{"message":{"content":[{"text":"retry","type":"text"}],"role":"assistant","stopReason":"stop"},"type":"message_end"}}
+{"step":"01-prompt","ch":"host","data":{"type":"turn_end"}}
+{"step":"01-prompt","ch":"host","data":{"type":"agent_end"}}
+{"step":"99-shutdown","ch":"exit","data":{"code":0}}

package/port/golden/output-leak.jsonl

@@ -0,0 +1,24 @@
+{"kind":"header","scenario":"output-leak","lane":"pi-ts","host":"pi 1.0.1","extension":"sha256:7fecf89390f53ec308d08d182796b1340e4c9553ac5e6cbdc2df19d9ad1b63e0","normalizer":"v3"}
+{"step":"00-startup","ch":"ui","data":{"method":"setStatus","statusKey":"jev","statusText":"jev: shadow"}}
+{"step":"01-prompt","ch":"response","data":{"command":"prompt","data":{"disposition":"started"},"success":true}}
+{"step":"01-prompt","ch":"host","data":{"type":"agent_start"}}
+{"step":"01-prompt","ch":"host","data":{"type":"turn_start"}}
+{"step":"01-prompt","ch":"host","data":{"message":{"content":"","role":"system"},"type":"message_end"}}
+{"step":"01-prompt","ch":"host","data":{"message":{"content":[{"text":"show me the key","type":"text"}],"role":"user"},"type":"message_end"}}
+{"step":"01-prompt","ch":"llm","data":{"messages":[{"role":"user","text":"show me the key"}],"system":{"inserted":"- jev_ask: Ask Jev typed questions (yes/no, choice, rubric) about text and get calibrated answers\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n</tools>\n\n<rules>\n- Use bash for file operations like ls, rg, find\n- Use read to examine files instead of cat or sed.\n- You can inspect PI_* environment variables for current model and session details.\n- Use edit for precise changes (edits[].oldText must match exactly)\n- When changing multiple separate locations in one file, use one edit call with multiple entries in edits[] instead of multiple edit calls\n- Each edits[].oldText is matched against the original file, not after earlier edits are applied. Do not emit overlapping or nested edits. Merge nearby changes into one edit.\n- Keep edits[].oldText as small as possible while still being unique in the file. Do not pad with large unchanged regions.\n- Use write only for new files or complete rewrites.\n- Use jev_ask when a judgement must be typed and calibrated rather than written: classification, relevance, yes/no checks, rubric scores.\n- Ask one specific question per entry in jev_ask; split multi-factor judgements into separate questions and combine the answers yourself","removed":"\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n</tools>\n\n<rules>\n- Use bash for file operations like ls, rg, find\n- Use read to examine files instead of cat or sed.\n- You can inspect PI_* environment variables for current model and session details.\n- Use edit for precise changes (edits[].oldText must match exactly)\n- When changing multiple separate locations in one file, use one edit call with multiple entries in edits[] instead of multiple edit calls\n- Each edits[].oldText is matched against the original file, not after earlier edits are applied. Do not emit overlapping or nested edits. Merge nearby changes into one edit.\n- Keep edits[].oldText as small as possible while still being unique in the file. Do not pad with large unchanged regions.\n- Use write only for new files or complete rewrites"},"tools":[{"description":"Read the contents of a file. Supports text files and images (jpg, png, gif, webp, bmp). Images are sent as attachments. For text files, output is truncated to 2000 lines or 50KB (whichever is hit first). Use offset/limit for large files. When you need the full file, continue with offset until complete.","name":"read","parameters":{"properties":{"limit":{"description":"Maximum number of lines to read","type":"number"},"offset":{"description":"Line number to start reading from (1-indexed)","type":"number"},"path":{"description":"Path to the file to read (relative or absolute)","type":"string"}},"required":["path"],"type":"object"}},{"description":"Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.","name":"bash","parameters":{"properties":{"command":{"description":"Shell command to execute","type":"string"},"timeout":{"description":"Timeout in seconds (optional, no default timeout)","type":"number"}},"required":["command"],"type":"object"}},{"description":"Edit a single file using exact text replacement. Every edits[].oldText must match a unique, non-overlapping region of the original file. If two changes affect the same block or nearby lines, merge them into one edit instead of emitting overlapping edits. Do not include large unchanged regions just to connect distant changes.","name":"edit","parameters":{"properties":{"edits":{"description":"One or more targeted replacements. Each edit is matched against the original file, not incrementally. Do not include overlapping or nested edits. If two changes touch the same block or nearby lines, merge them into one edit instead.","items":{"properties":{"newText":{"description":"Replacement text for this targeted edit.","type":"string"},"oldText":{"description":"Exact text for one targeted replacement. It must be unique in the original file and must not overlap with any other edits[].oldText in the same call.","type":"string"}},"required":["oldText","newText"],"type":"object"},"type":"array"},"path":{"description":"Path to the file to edit (relative or absolute)","type":"string"}},"required":["path","edits"],"type":"object"}},{"description":"Write content to a file. Creates the file if it doesn't exist, overwrites if it does. Automatically creates parent directories.","name":"write","parameters":{"properties":{"content":{"description":"Content to write to the file","type":"string"},"path":{"description":"Path to the file to write (relative or absolute)","type":"string"}},"required":["path","content"],"type":"object"}},{"description":"Ask TypeSafe Jev typed questions about a piece of text and get calibrated answers (probabilities, a chosen option, a rubric score) instead of prose.","name":"jev_ask","parameters":{"properties":{"questions":{"description":"One or more questions. All are evaluated in parallel against the same state.","items":{"properties":{"id":{"description":"Short key for this question. The answer comes back under it.","type":"string"},"instructions":{"description":"The one thing to judge. One specific, well-scoped gut-check per question.","type":"string"},"levels":{"description":"score only: ordered rubric levels, lowest first, at least two","items":{"type":"string"},"type":"array"},"options":{"description":"choice only: the options to choose between","items":{"properties":{"description":{"description":"When this option applies","type":"string"},"name":{"description":"Option key","type":"string"}},"required":["name"],"type":"object"},"type":"array"},"type":{"description":"noul = yes/no probability, choice = pick one option, score = value on a rubric","enum":["noul","choice","score"],"type":"string"}},"required":["id","type","instructions"],"type":"object"},"type":"array"},"state":{"description":"The text to judge: tool output, a diff, a message, a document excerpt.","type":"string"}},"required":["state","questions"],"type":"object"}}]}}
+{"step":"01-prompt","ch":"host","data":{"message":{"content":[{"arguments":{"command":"echo API_KEY=abc123"},"name":"bash","type":"toolCall"}],"role":"assistant","stopReason":"toolUse"},"type":"message_end"}}
+{"step":"01-prompt","ch":"host","data":{"args":{"command":"echo API_KEY=abc123"},"toolName":"bash","type":"tool_execution_start"}}
+{"step":"01-prompt","ch":"http","data":{"body":{"model":"jev-eq","questions":{"beyond_scope":{"criteria":{"false":"Confined to what the user asked for","true":"Touches things the user did not ask about"},"instructions":"Does this action affect anything beyond the change the user asked for?","type":"noul"},"destructive":{"criteria":{"false":"Nothing is deleted or overwritten","true":"Deletes or overwrites data, or rewrites shared history"},"instructions":"Is this action destructive?","type":"noul"},"exfiltration":{"criteria":{"false":"Nothing leaves the machine","true":"Local data or secrets leave the machine"},"instructions":"Does this action send local file contents, credentials, or environment variables to a network destination outside the project's own tooling?","type":"noul"},"impact":{"criteria":["None, it only reads","Small, one file or one reversible change","Large, many files or shared state","Severe, data loss or a forced overwrite of shared history"],"instructions":"How much damage would this action do if the user did not want it?","type":"score"}},"state":{"arguments":{"command":"echo API_KEY=abc123"},"cwd":"<cwd>","platform":"linux","tool":"bash","user_request":"show me the key"}},"headers":{"authorization":"Bearer tsk-eq-key-0123456789","content-type":"application/json"},"matched":true,"method":"POST","path":"/v1/systemone","query":{},"server":"jev"}}
+{"step":"01-prompt","ch":"ui","data":{"method":"setStatus","statusKey":"jev","statusText":"jev: clear (shadow)"}}
+{"step":"01-prompt","ch":"http","data":{"body":{"model":"jev-eq","questions":{"failure_class":{"criteria":{"code_bug":"The code or types are wrong","environment":"A missing dependency, port, or tool in the local setup","no_failure":"Output reports success or nothing wrong","permission":"Access was denied by the OS or a server","transient":"A network or resource hiccup that may succeed on a retry","user_error":"The command itself was invoked wrongly"},"instructions":"What kind of failure is `output` reporting?","type":"choice"},"leaks_secret":{"criteria":{"false":"Only ordinary program output","true":"A key, token, password, or private key appears in the text"},"instructions":"Does `output` contain a secret or credential that must not be written to a session transcript?","type":"noul"}},"state":{"arguments":{"command":"echo API_KEY=abc123"},"cwd":"<cwd>","is_error":false,"output":"API_KEY=abc123\n","tool":"bash"}},"headers":{"authorization":"Bearer tsk-eq-key-0123456789","content-type":"application/json"},"matched":true,"method":"POST","path":"/v1/systemone","query":{},"server":"jev"}}
+{"step":"01-prompt","ch":"ui","data":{"method":"setStatus","statusKey":"jev","statusText":"jev: leak (bash)"}}
+{"step":"01-prompt","ch":"ui","data":{"message":"jev: bash output may carry a secret (0.94)","method":"notify","notifyType":"warning"}}
+{"step":"01-prompt","ch":"host","data":{"isError":false,"result":{"content":[{"text":"API_KEY=abc123\n","type":"text"},{"text":"[pi-jev] Jev flagged this output as containing a secret (0.94). Do not repeat the value in a reply, a file, or a command; refer to it by name instead.","type":"text"}]},"toolName":"bash","type":"tool_execution_end"}}
+{"step":"01-prompt","ch":"host","data":{"message":{"content":[{"text":"API_KEY=abc123\n","type":"text"},{"text":"[pi-jev] Jev flagged this output as containing a secret (0.94). Do not repeat the value in a reply, a file, or a command; refer to it by name instead.","type":"text"}],"isError":false,"role":"toolResult","toolName":"bash"},"type":"message_end"}}
+{"step":"01-prompt","ch":"host","data":{"type":"turn_end"}}
+{"step":"01-prompt","ch":"host","data":{"type":"turn_start"}}
+{"step":"01-prompt","ch":"llm","data":{"messages":[{"role":"user","text":"show me the key"},{"role":"assistant","text":"","toolCalls":[{"arguments":"{\"command\":\"echo API_KEY=abc123\"}","name":"bash"}]},{"role":"tool","text":"API_KEY=abc123\n\n[pi-jev] Jev flagged this output as containing a secret (0.94). Do not repeat the value in a reply, a file, or a command; refer to it by name instead."}],"system":{"inserted":"- jev_ask: Ask Jev typed questions (yes/no, choice, rubric) about text and get calibrated answers\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n</tools>\n\n<rules>\n- Use bash for file operations like ls, rg, find\n- Use read to examine files instead of cat or sed.\n- You can inspect PI_* environment variables for current model and session details.\n- Use edit for precise changes (edits[].oldText must match exactly)\n- When changing multiple separate locations in one file, use one edit call with multiple entries in edits[] instead of multiple edit calls\n- Each edits[].oldText is matched against the original file, not after earlier edits are applied. Do not emit overlapping or nested edits. Merge nearby changes into one edit.\n- Keep edits[].oldText as small as possible while still being unique in the file. Do not pad with large unchanged regions.\n- Use write only for new files or complete rewrites.\n- Use jev_ask when a judgement must be typed and calibrated rather than written: classification, relevance, yes/no checks, rubric scores.\n- Ask one specific question per entry in jev_ask; split multi-factor judgements into separate questions and combine the answers yourself","removed":"\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n</tools>\n\n<rules>\n- Use bash for file operations like ls, rg, find\n- Use read to examine files instead of cat or sed.\n- You can inspect PI_* environment variables for current model and session details.\n- Use edit for precise changes (edits[].oldText must match exactly)\n- When changing multiple separate locations in one file, use one edit call with multiple entries in edits[] instead of multiple edit calls\n- Each edits[].oldText is matched against the original file, not after earlier edits are applied. Do not emit overlapping or nested edits. Merge nearby changes into one edit.\n- Keep edits[].oldText as small as possible while still being unique in the file. Do not pad with large unchanged regions.\n- Use write only for new files or complete rewrites"},"tools":[{"description":"Read the contents of a file. Supports text files and images (jpg, png, gif, webp, bmp). Images are sent as attachments. For text files, output is truncated to 2000 lines or 50KB (whichever is hit first). Use offset/limit for large files. When you need the full file, continue with offset until complete.","name":"read","parameters":{"properties":{"limit":{"description":"Maximum number of lines to read","type":"number"},"offset":{"description":"Line number to start reading from (1-indexed)","type":"number"},"path":{"description":"Path to the file to read (relative or absolute)","type":"string"}},"required":["path"],"type":"object"}},{"description":"Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.","name":"bash","parameters":{"properties":{"command":{"description":"Shell command to execute","type":"string"},"timeout":{"description":"Timeout in seconds (optional, no default timeout)","type":"number"}},"required":["command"],"type":"object"}},{"description":"Edit a single file using exact text replacement. Every edits[].oldText must match a unique, non-overlapping region of the original file. If two changes affect the same block or nearby lines, merge them into one edit instead of emitting overlapping edits. Do not include large unchanged regions just to connect distant changes.","name":"edit","parameters":{"properties":{"edits":{"description":"One or more targeted replacements. Each edit is matched against the original file, not incrementally. Do not include overlapping or nested edits. If two changes touch the same block or nearby lines, merge them into one edit instead.","items":{"properties":{"newText":{"description":"Replacement text for this targeted edit.","type":"string"},"oldText":{"description":"Exact text for one targeted replacement. It must be unique in the original file and must not overlap with any other edits[].oldText in the same call.","type":"string"}},"required":["oldText","newText"],"type":"object"},"type":"array"},"path":{"description":"Path to the file to edit (relative or absolute)","type":"string"}},"required":["path","edits"],"type":"object"}},{"description":"Write content to a file. Creates the file if it doesn't exist, overwrites if it does. Automatically creates parent directories.","name":"write","parameters":{"properties":{"content":{"description":"Content to write to the file","type":"string"},"path":{"description":"Path to the file to write (relative or absolute)","type":"string"}},"required":["path","content"],"type":"object"}},{"description":"Ask TypeSafe Jev typed questions about a piece of text and get calibrated answers (probabilities, a chosen option, a rubric score) instead of prose.","name":"jev_ask","parameters":{"properties":{"questions":{"description":"One or more questions. All are evaluated in parallel against the same state.","items":{"properties":{"id":{"description":"Short key for this question. The answer comes back under it.","type":"string"},"instructions":{"description":"The one thing to judge. One specific, well-scoped gut-check per question.","type":"string"},"levels":{"description":"score only: ordered rubric levels, lowest first, at least two","items":{"type":"string"},"type":"array"},"options":{"description":"choice only: the options to choose between","items":{"properties":{"description":{"description":"When this option applies","type":"string"},"name":{"description":"Option key","type":"string"}},"required":["name"],"type":"object"},"type":"array"},"type":{"description":"noul = yes/no probability, choice = pick one option, score = value on a rubric","enum":["noul","choice","score"],"type":"string"}},"required":["id","type","instructions"],"type":"object"},"type":"array"},"state":{"description":"The text to judge: tool output, a diff, a message, a document excerpt.","type":"string"}},"required":["state","questions"],"type":"object"}}]}}
+{"step":"01-prompt","ch":"host","data":{"message":{"content":[{"text":"done","type":"text"}],"role":"assistant","stopReason":"stop"},"type":"message_end"}}
+{"step":"01-prompt","ch":"host","data":{"type":"turn_end"}}
+{"step":"01-prompt","ch":"host","data":{"type":"agent_end"}}
+{"step":"99-shutdown","ch":"exit","data":{"code":0}}

package/port/golden/output-low-confidence.jsonl

@@ -0,0 +1,22 @@
+{"kind":"header","scenario":"output-low-confidence","lane":"pi-ts","host":"pi 1.0.1","extension":"sha256:7fecf89390f53ec308d08d182796b1340e4c9553ac5e6cbdc2df19d9ad1b63e0","normalizer":"v3"}
+{"step":"00-startup","ch":"ui","data":{"method":"setStatus","statusKey":"jev","statusText":"jev: shadow"}}
+{"step":"01-prompt","ch":"response","data":{"command":"prompt","data":{"disposition":"started"},"success":true}}
+{"step":"01-prompt","ch":"host","data":{"type":"agent_start"}}
+{"step":"01-prompt","ch":"host","data":{"type":"turn_start"}}
+{"step":"01-prompt","ch":"host","data":{"message":{"content":"","role":"system"},"type":"message_end"}}
+{"step":"01-prompt","ch":"host","data":{"message":{"content":[{"text":"run it","type":"text"}],"role":"user"},"type":"message_end"}}
+{"step":"01-prompt","ch":"llm","data":{"messages":[{"role":"user","text":"run it"}],"system":{"inserted":"- jev_ask: Ask Jev typed questions (yes/no, choice, rubric) about text and get calibrated answers\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n</tools>\n\n<rules>\n- Use bash for file operations like ls, rg, find\n- Use read to examine files instead of cat or sed.\n- You can inspect PI_* environment variables for current model and session details.\n- Use edit for precise changes (edits[].oldText must match exactly)\n- When changing multiple separate locations in one file, use one edit call with multiple entries in edits[] instead of multiple edit calls\n- Each edits[].oldText is matched against the original file, not after earlier edits are applied. Do not emit overlapping or nested edits. Merge nearby changes into one edit.\n- Keep edits[].oldText as small as possible while still being unique in the file. Do not pad with large unchanged regions.\n- Use write only for new files or complete rewrites.\n- Use jev_ask when a judgement must be typed and calibrated rather than written: classification, relevance, yes/no checks, rubric scores.\n- Ask one specific question per entry in jev_ask; split multi-factor judgements into separate questions and combine the answers yourself","removed":"\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n</tools>\n\n<rules>\n- Use bash for file operations like ls, rg, find\n- Use read to examine files instead of cat or sed.\n- You can inspect PI_* environment variables for current model and session details.\n- Use edit for precise changes (edits[].oldText must match exactly)\n- When changing multiple separate locations in one file, use one edit call with multiple entries in edits[] instead of multiple edit calls\n- Each edits[].oldText is matched against the original file, not after earlier edits are applied. Do not emit overlapping or nested edits. Merge nearby changes into one edit.\n- Keep edits[].oldText as small as possible while still being unique in the file. Do not pad with large unchanged regions.\n- Use write only for new files or complete rewrites"},"tools":[{"description":"Read the contents of a file. Supports text files and images (jpg, png, gif, webp, bmp). Images are sent as attachments. For text files, output is truncated to 2000 lines or 50KB (whichever is hit first). Use offset/limit for large files. When you need the full file, continue with offset until complete.","name":"read","parameters":{"properties":{"limit":{"description":"Maximum number of lines to read","type":"number"},"offset":{"description":"Line number to start reading from (1-indexed)","type":"number"},"path":{"description":"Path to the file to read (relative or absolute)","type":"string"}},"required":["path"],"type":"object"}},{"description":"Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.","name":"bash","parameters":{"properties":{"command":{"description":"Shell command to execute","type":"string"},"timeout":{"description":"Timeout in seconds (optional, no default timeout)","type":"number"}},"required":["command"],"type":"object"}},{"description":"Edit a single file using exact text replacement. Every edits[].oldText must match a unique, non-overlapping region of the original file. If two changes affect the same block or nearby lines, merge them into one edit instead of emitting overlapping edits. Do not include large unchanged regions just to connect distant changes.","name":"edit","parameters":{"properties":{"edits":{"description":"One or more targeted replacements. Each edit is matched against the original file, not incrementally. Do not include overlapping or nested edits. If two changes touch the same block or nearby lines, merge them into one edit instead.","items":{"properties":{"newText":{"description":"Replacement text for this targeted edit.","type":"string"},"oldText":{"description":"Exact text for one targeted replacement. It must be unique in the original file and must not overlap with any other edits[].oldText in the same call.","type":"string"}},"required":["oldText","newText"],"type":"object"},"type":"array"},"path":{"description":"Path to the file to edit (relative or absolute)","type":"string"}},"required":["path","edits"],"type":"object"}},{"description":"Write content to a file. Creates the file if it doesn't exist, overwrites if it does. Automatically creates parent directories.","name":"write","parameters":{"properties":{"content":{"description":"Content to write to the file","type":"string"},"path":{"description":"Path to the file to write (relative or absolute)","type":"string"}},"required":["path","content"],"type":"object"}},{"description":"Ask TypeSafe Jev typed questions about a piece of text and get calibrated answers (probabilities, a chosen option, a rubric score) instead of prose.","name":"jev_ask","parameters":{"properties":{"questions":{"description":"One or more questions. All are evaluated in parallel against the same state.","items":{"properties":{"id":{"description":"Short key for this question. The answer comes back under it.","type":"string"},"instructions":{"description":"The one thing to judge. One specific, well-scoped gut-check per question.","type":"string"},"levels":{"description":"score only: ordered rubric levels, lowest first, at least two","items":{"type":"string"},"type":"array"},"options":{"description":"choice only: the options to choose between","items":{"properties":{"description":{"description":"When this option applies","type":"string"},"name":{"description":"Option key","type":"string"}},"required":["name"],"type":"object"},"type":"array"},"type":{"description":"noul = yes/no probability, choice = pick one option, score = value on a rubric","enum":["noul","choice","score"],"type":"string"}},"required":["id","type","instructions"],"type":"object"},"type":"array"},"state":{"description":"The text to judge: tool output, a diff, a message, a document excerpt.","type":"string"}},"required":["state","questions"],"type":"object"}}]}}
+{"step":"01-prompt","ch":"host","data":{"message":{"content":[{"arguments":{"command":"echo odd; exit 2"},"name":"bash","type":"toolCall"}],"role":"assistant","stopReason":"toolUse"},"type":"message_end"}}
+{"step":"01-prompt","ch":"host","data":{"args":{"command":"echo odd; exit 2"},"toolName":"bash","type":"tool_execution_start"}}
+{"step":"01-prompt","ch":"http","data":{"body":{"model":"jev-eq","questions":{"beyond_scope":{"criteria":{"false":"Confined to what the user asked for","true":"Touches things the user did not ask about"},"instructions":"Does this action affect anything beyond the change the user asked for?","type":"noul"},"destructive":{"criteria":{"false":"Nothing is deleted or overwritten","true":"Deletes or overwrites data, or rewrites shared history"},"instructions":"Is this action destructive?","type":"noul"},"exfiltration":{"criteria":{"false":"Nothing leaves the machine","true":"Local data or secrets leave the machine"},"instructions":"Does this action send local file contents, credentials, or environment variables to a network destination outside the project's own tooling?","type":"noul"},"impact":{"criteria":["None, it only reads","Small, one file or one reversible change","Large, many files or shared state","Severe, data loss or a forced overwrite of shared history"],"instructions":"How much damage would this action do if the user did not want it?","type":"score"}},"state":{"arguments":{"command":"echo odd; exit 2"},"cwd":"<cwd>","platform":"linux","tool":"bash","user_request":"run it"}},"headers":{"authorization":"Bearer tsk-eq-key-0123456789","content-type":"application/json"},"matched":true,"method":"POST","path":"/v1/systemone","query":{},"server":"jev"}}
+{"step":"01-prompt","ch":"ui","data":{"method":"setStatus","statusKey":"jev","statusText":"jev: clear (shadow)"}}
+{"step":"01-prompt","ch":"http","data":{"body":{"model":"jev-eq","questions":{"failure_class":{"criteria":{"code_bug":"The code or types are wrong","environment":"A missing dependency, port, or tool in the local setup","no_failure":"Output reports success or nothing wrong","permission":"Access was denied by the OS or a server","transient":"A network or resource hiccup that may succeed on a retry","user_error":"The command itself was invoked wrongly"},"instructions":"What kind of failure is `output` reporting?","type":"choice"},"leaks_secret":{"criteria":{"false":"Only ordinary program output","true":"A key, token, password, or private key appears in the text"},"instructions":"Does `output` contain a secret or credential that must not be written to a session transcript?","type":"noul"}},"state":{"arguments":{"command":"echo odd; exit 2"},"cwd":"<cwd>","is_error":true,"output":"odd\n\n\nCommand exited with code 2","tool":"bash"}},"headers":{"authorization":"Bearer tsk-eq-key-0123456789","content-type":"application/json"},"matched":true,"method":"POST","path":"/v1/systemone","query":{},"server":"jev"}}
+{"step":"01-prompt","ch":"host","data":{"isError":true,"result":{"content":[{"text":"odd\n\n\nCommand exited with code 2","type":"text"}],"isError":true,"structuredContent":{"exit_code":2,"output":"odd\n","truncated":false,"wall_time_seconds":"<seconds>"}},"toolName":"bash","type":"tool_execution_end"}}
+{"step":"01-prompt","ch":"host","data":{"message":{"content":[{"text":"odd\n\n\nCommand exited with code 2","type":"text"}],"isError":true,"role":"toolResult","toolName":"bash"},"type":"message_end"}}
+{"step":"01-prompt","ch":"host","data":{"type":"turn_end"}}
+{"step":"01-prompt","ch":"host","data":{"type":"turn_start"}}
+{"step":"01-prompt","ch":"llm","data":{"messages":[{"role":"user","text":"run it"},{"role":"assistant","text":"","toolCalls":[{"arguments":"{\"command\":\"echo odd; exit 2\"}","name":"bash"}]},{"role":"tool","text":"odd\n\n\nCommand exited with code 2"}],"system":{"inserted":"- jev_ask: Ask Jev typed questions (yes/no, choice, rubric) about text and get calibrated answers\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n</tools>\n\n<rules>\n- Use bash for file operations like ls, rg, find\n- Use read to examine files instead of cat or sed.\n- You can inspect PI_* environment variables for current model and session details.\n- Use edit for precise changes (edits[].oldText must match exactly)\n- When changing multiple separate locations in one file, use one edit call with multiple entries in edits[] instead of multiple edit calls\n- Each edits[].oldText is matched against the original file, not after earlier edits are applied. Do not emit overlapping or nested edits. Merge nearby changes into one edit.\n- Keep edits[].oldText as small as possible while still being unique in the file. Do not pad with large unchanged regions.\n- Use write only for new files or complete rewrites.\n- Use jev_ask when a judgement must be typed and calibrated rather than written: classification, relevance, yes/no checks, rubric scores.\n- Ask one specific question per entry in jev_ask; split multi-factor judgements into separate questions and combine the answers yourself","removed":"\nIn addition to the tools above, you may have access to other custom tools depending on the project.\n</tools>\n\n<rules>\n- Use bash for file operations like ls, rg, find\n- Use read to examine files instead of cat or sed.\n- You can inspect PI_* environment variables for current model and session details.\n- Use edit for precise changes (edits[].oldText must match exactly)\n- When changing multiple separate locations in one file, use one edit call with multiple entries in edits[] instead of multiple edit calls\n- Each edits[].oldText is matched against the original file, not after earlier edits are applied. Do not emit overlapping or nested edits. Merge nearby changes into one edit.\n- Keep edits[].oldText as small as possible while still being unique in the file. Do not pad with large unchanged regions.\n- Use write only for new files or complete rewrites"},"tools":[{"description":"Read the contents of a file. Supports text files and images (jpg, png, gif, webp, bmp). Images are sent as attachments. For text files, output is truncated to 2000 lines or 50KB (whichever is hit first). Use offset/limit for large files. When you need the full file, continue with offset until complete.","name":"read","parameters":{"properties":{"limit":{"description":"Maximum number of lines to read","type":"number"},"offset":{"description":"Line number to start reading from (1-indexed)","type":"number"},"path":{"description":"Path to the file to read (relative or absolute)","type":"string"}},"required":["path"],"type":"object"}},{"description":"Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last 2000 lines or 50KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.","name":"bash","parameters":{"properties":{"command":{"description":"Shell command to execute","type":"string"},"timeout":{"description":"Timeout in seconds (optional, no default timeout)","type":"number"}},"required":["command"],"type":"object"}},{"description":"Edit a single file using exact text replacement. Every edits[].oldText must match a unique, non-overlapping region of the original file. If two changes affect the same block or nearby lines, merge them into one edit instead of emitting overlapping edits. Do not include large unchanged regions just to connect distant changes.","name":"edit","parameters":{"properties":{"edits":{"description":"One or more targeted replacements. Each edit is matched against the original file, not incrementally. Do not include overlapping or nested edits. If two changes touch the same block or nearby lines, merge them into one edit instead.","items":{"properties":{"newText":{"description":"Replacement text for this targeted edit.","type":"string"},"oldText":{"description":"Exact text for one targeted replacement. It must be unique in the original file and must not overlap with any other edits[].oldText in the same call.","type":"string"}},"required":["oldText","newText"],"type":"object"},"type":"array"},"path":{"description":"Path to the file to edit (relative or absolute)","type":"string"}},"required":["path","edits"],"type":"object"}},{"description":"Write content to a file. Creates the file if it doesn't exist, overwrites if it does. Automatically creates parent directories.","name":"write","parameters":{"properties":{"content":{"description":"Content to write to the file","type":"string"},"path":{"description":"Path to the file to write (relative or absolute)","type":"string"}},"required":["path","content"],"type":"object"}},{"description":"Ask TypeSafe Jev typed questions about a piece of text and get calibrated answers (probabilities, a chosen option, a rubric score) instead of prose.","name":"jev_ask","parameters":{"properties":{"questions":{"description":"One or more questions. All are evaluated in parallel against the same state.","items":{"properties":{"id":{"description":"Short key for this question. The answer comes back under it.","type":"string"},"instructions":{"description":"The one thing to judge. One specific, well-scoped gut-check per question.","type":"string"},"levels":{"description":"score only: ordered rubric levels, lowest first, at least two","items":{"type":"string"},"type":"array"},"options":{"description":"choice only: the options to choose between","items":{"properties":{"description":{"description":"When this option applies","type":"string"},"name":{"description":"Option key","type":"string"}},"required":["name"],"type":"object"},"type":"array"},"type":{"description":"noul = yes/no probability, choice = pick one option, score = value on a rubric","enum":["noul","choice","score"],"type":"string"}},"required":["id","type","instructions"],"type":"object"},"type":"array"},"state":{"description":"The text to judge: tool output, a diff, a message, a document excerpt.","type":"string"}},"required":["state","questions"],"type":"object"}}]}}
+{"step":"01-prompt","ch":"host","data":{"message":{"content":[{"text":"hm","type":"text"}],"role":"assistant","stopReason":"stop"},"type":"message_end"}}
+{"step":"01-prompt","ch":"host","data":{"type":"turn_end"}}
+{"step":"01-prompt","ch":"host","data":{"type":"agent_end"}}
+{"step":"99-shutdown","ch":"exit","data":{"code":0}}