@pi-in-go/pigpen-jev 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CREDITS.md +22 -0
- package/LICENSE +22 -0
- package/README.md +237 -0
- package/extensions/jev/ask.go +166 -0
- package/extensions/jev/ask_test.go +218 -0
- package/extensions/jev/backend.go +128 -0
- package/extensions/jev/bench_test.go +64 -0
- package/extensions/jev/boundaries_test.go +159 -0
- package/extensions/jev/command.go +224 -0
- package/extensions/jev/commands_test.go +214 -0
- package/extensions/jev/config.go +450 -0
- package/extensions/jev/errors_test.go +191 -0
- package/extensions/jev/extension.go +391 -0
- package/extensions/jev/fakehost_test.go +548 -0
- package/extensions/jev/gate.go +125 -0
- package/extensions/jev/gate_test.go +610 -0
- package/extensions/jev/gatekey_test.go +24 -0
- package/extensions/jev/go.mod +9 -0
- package/extensions/jev/go.sum +2 -0
- package/extensions/jev/go.work +10 -0
- package/extensions/jev/helpers_test.go +404 -0
- package/extensions/jev/memo.go +88 -0
- package/extensions/jev/output.go +89 -0
- package/extensions/jev/output_test.go +187 -0
- package/extensions/jev/ownmodel_test.go +118 -0
- package/extensions/jev/render.go +136 -0
- package/extensions/jev/review_test.go +310 -0
- package/extensions/jev/source_test.go +57 -0
- package/extensions/jev/text.go +174 -0
- package/extensions/jev/trust_test.go +335 -0
- package/extensions/jev/types.go +227 -0
- package/libs/typesafe/CONTRACT.md +125 -0
- package/libs/typesafe/CREDITS.md +37 -0
- package/libs/typesafe/LICENSE +23 -0
- package/libs/typesafe/README.md +19 -0
- package/libs/typesafe/go.mod +3 -0
- package/libs/typesafe/libraries/ownmodel/backend_test.go +496 -0
- package/libs/typesafe/libraries/ownmodel/canon.go +190 -0
- package/libs/typesafe/libraries/ownmodel/convert.go +199 -0
- package/libs/typesafe/libraries/ownmodel/doc.go +15 -0
- package/libs/typesafe/libraries/ownmodel/equivalence_test.go +199 -0
- package/libs/typesafe/libraries/ownmodel/helpers_test.go +155 -0
- package/libs/typesafe/libraries/ownmodel/mutation_test.go +31 -0
- package/libs/typesafe/libraries/ownmodel/ownmodel.go +225 -0
- package/libs/typesafe/libraries/ownmodel/plan.go +442 -0
- package/libs/typesafe/libraries/ownmodel/run.go +288 -0
- package/libs/typesafe/libraries/ownmodel/schema_test.go +254 -0
- package/libs/typesafe/libraries/ownmodel/twins_test.go +169 -0
- package/libs/typesafe/libraries/ownmodel/utils_test.go +125 -0
- package/libs/typesafe/libraries/pigmodel/pigmodel.go +264 -0
- package/libs/typesafe/libraries/pigmodel/pigmodel_test.go +410 -0
- package/libs/typesafe/libraries/typesafe/answers.go +268 -0
- package/libs/typesafe/libraries/typesafe/api_response_test.go +113 -0
- package/libs/typesafe/libraries/typesafe/batch.go +80 -0
- package/libs/typesafe/libraries/typesafe/batch_test.go +133 -0
- package/libs/typesafe/libraries/typesafe/bench_test.go +71 -0
- package/libs/typesafe/libraries/typesafe/client.go +561 -0
- package/libs/typesafe/libraries/typesafe/client_test.go +495 -0
- package/libs/typesafe/libraries/typesafe/crosscheck_test.go +464 -0
- package/libs/typesafe/libraries/typesafe/crosscheck_workflowevals_test.go +219 -0
- package/libs/typesafe/libraries/typesafe/doc.go +27 -0
- package/libs/typesafe/libraries/typesafe/entry.go +142 -0
- package/libs/typesafe/libraries/typesafe/env.go +11 -0
- package/libs/typesafe/libraries/typesafe/errors.go +310 -0
- package/libs/typesafe/libraries/typesafe/errors_test.go +175 -0
- package/libs/typesafe/libraries/typesafe/helpers_test.go +294 -0
- package/libs/typesafe/libraries/typesafe/live_test.go +96 -0
- package/libs/typesafe/libraries/typesafe/logging.go +160 -0
- package/libs/typesafe/libraries/typesafe/logging_test.go +259 -0
- package/libs/typesafe/libraries/typesafe/marshal_test.go +112 -0
- package/libs/typesafe/libraries/typesafe/mutation_test.go +39 -0
- package/libs/typesafe/libraries/typesafe/questions.go +490 -0
- package/libs/typesafe/libraries/typesafe/questions_test.go +166 -0
- package/libs/typesafe/libraries/typesafe/regressions_test.go +159 -0
- package/libs/typesafe/libraries/typesafe/reliability_test.go +649 -0
- package/libs/typesafe/libraries/typesafe/retry.go +350 -0
- package/libs/typesafe/libraries/typesafe/retry_test.go +297 -0
- package/libs/typesafe/libraries/typesafe/runtime_test.go +26 -0
- package/libs/typesafe/libraries/typesafe/transport_test.go +163 -0
- package/libs/typesafe/libraries/typesafe/twins_test.go +127 -0
- package/libs/typesafe/libraries/typesafe/types_test.go +165 -0
- package/libs/typesafe/libraries/typesafe/version.go +10 -0
- package/libs/typesafe/package.json +37 -0
- package/libs/typesafe/provenance.json +49 -0
- package/package.json +42 -0
- package/port/PORT.md +107 -0
- package/port/e2e/gate-and-output.py +35 -0
- package/port/e2e/jev-ask.py +36 -0
- package/port/e2e/model-switch.py +44 -0
- package/port/e2e/off-by-default.py +34 -0
- package/port/gen-scenarios.py +103 -0
- package/port/golden/cache-identical-calls.jsonl +30 -0
- package/port/golden/clear.jsonl +22 -0
- package/port/golden/commands.jsonl +43 -0
- package/port/golden/enforce-accept.jsonl +23 -0
- package/port/golden/enforce-decline.jsonl +22 -0
- package/port/golden/jev-ask.jsonl +20 -0
- package/port/golden/output-advice.jsonl +23 -0
- package/port/golden/output-leak.jsonl +24 -0
- package/port/golden/output-low-confidence.jsonl +22 -0
- package/port/golden/shadow-flagged.jsonl +23 -0
- package/port/golden/unjudged-tools.jsonl +19 -0
- package/port/golden/write-elision.jsonl +21 -0
- package/port/mutate-unit.py +63 -0
- package/port/mutations.json +578 -0
- package/port/oracle/LICENSE +21 -0
- package/port/oracle/README.md +181 -0
- package/port/oracle/SHA256SUMS +8 -0
- package/port/oracle/package.json +43 -0
- package/port/oracle/src/client.ts +409 -0
- package/port/oracle/src/config.ts +363 -0
- package/port/oracle/src/gate.ts +229 -0
- package/port/oracle/src/index.ts +649 -0
- package/port/oracle/src/output.ts +163 -0
- package/port/red-run.log +309 -0
- package/port/scenarios/cache-identical-calls.json +71 -0
- package/port/scenarios/clear.json +61 -0
- package/port/scenarios/commands.json +119 -0
- package/port/scenarios/enforce-accept.json +66 -0
- package/port/scenarios/enforce-decline.json +57 -0
- package/port/scenarios/jev-ask.json +83 -0
- package/port/scenarios/output-advice.json +61 -0
- package/port/scenarios/output-leak.json +61 -0
- package/port/scenarios/output-low-confidence.json +61 -0
- package/port/scenarios/shadow-flagged.json +61 -0
- package/port/scenarios/unjudged-tools.json +55 -0
- package/port/scenarios/write-elision.json +53 -0
- package/provenance.json +18 -0
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "commands",
|
|
3
|
+
"description": "/jev mode, last, output, check, off and on.",
|
|
4
|
+
"agentFiles": {
|
|
5
|
+
"pi-jev.json": "{\"enabled\": true, \"acknowledged\": true, \"display\": \"plain\", \"backend\": \"typesafe\", \"endpoint\": \"{{server:jev}}/v1/systemone\", \"model\": \"jev-eq\", \"retries\": 0, \"timeoutMs\": 5000}"
|
|
6
|
+
},
|
|
7
|
+
"env": {
|
|
8
|
+
"TYPESAFE_API_KEY": "tsk-eq-key-0123456789"
|
|
9
|
+
},
|
|
10
|
+
"servers": {
|
|
11
|
+
"jev": {
|
|
12
|
+
"recordHeaders": [
|
|
13
|
+
"authorization"
|
|
14
|
+
],
|
|
15
|
+
"routes": [
|
|
16
|
+
{
|
|
17
|
+
"method": "POST",
|
|
18
|
+
"path": "/v1/systemone",
|
|
19
|
+
"body": "{\"model\": \"jev-eq\", \"answers\": {\"destructive\": {\"type\": \"noul\", \"noul\": 0.99}, \"exfiltration\": {\"type\": \"noul\", \"noul\": 0.79}, \"beyond_scope\": {\"type\": \"noul\", \"noul\": 0.98}, \"impact\": {\"type\": \"score\", \"score\": 3.0, \"confidence\": 0.91, \"legend\": {\"0\": \"None, it only reads\", \"1\": \"Small\", \"2\": \"Large\", \"3\": \"Severe\"}, \"probabilities\": {\"0\": 0.05, \"1\": 0.1, \"2\": 0.15, \"3\": 0.7}}}, \"usage\": {\"input_tokens\": 100, \"output_tokens\": 4}}",
|
|
20
|
+
"headers": {
|
|
21
|
+
"Content-Type": "application/json"
|
|
22
|
+
},
|
|
23
|
+
"times": 1
|
|
24
|
+
},
|
|
25
|
+
{
|
|
26
|
+
"method": "POST",
|
|
27
|
+
"path": "/v1/systemone",
|
|
28
|
+
"body": "{\"model\": \"jev-eq\", \"answers\": {\"leaks_secret\": {\"type\": \"noul\", \"noul\": 0.94}, \"failure_class\": {\"type\": \"choice\", \"choice\": \"transient\", \"confidence\": 0.8, \"probabilities\": {\"transient\": 0.8}}}}",
|
|
29
|
+
"headers": {
|
|
30
|
+
"Content-Type": "application/json"
|
|
31
|
+
},
|
|
32
|
+
"times": 1
|
|
33
|
+
},
|
|
34
|
+
{
|
|
35
|
+
"method": "POST",
|
|
36
|
+
"path": "/v1/systemone",
|
|
37
|
+
"body": "{\"model\": \"jev-eq\", \"answers\": {\"destructive\": {\"type\": \"noul\", \"noul\": 0.99}, \"exfiltration\": {\"type\": \"noul\", \"noul\": 0.1}, \"beyond_scope\": {\"type\": \"noul\", \"noul\": 0.1}, \"impact\": {\"type\": \"score\", \"score\": 1.0, \"confidence\": 0.9, \"legend\": {\"0\": \"None, it only reads\", \"1\": \"Small\", \"2\": \"Large\", \"3\": \"Severe\"}, \"probabilities\": {\"0\": 0.05, \"1\": 0.1, \"2\": 0.15, \"3\": 0.7}}}, \"usage\": {\"input_tokens\": 100, \"output_tokens\": 4}}",
|
|
38
|
+
"headers": {
|
|
39
|
+
"Content-Type": "application/json"
|
|
40
|
+
},
|
|
41
|
+
"times": 1
|
|
42
|
+
}
|
|
43
|
+
]
|
|
44
|
+
}
|
|
45
|
+
},
|
|
46
|
+
"llm": [
|
|
47
|
+
{
|
|
48
|
+
"toolCalls": [
|
|
49
|
+
{
|
|
50
|
+
"name": "bash",
|
|
51
|
+
"arguments": {
|
|
52
|
+
"command": "echo hi"
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
]
|
|
56
|
+
},
|
|
57
|
+
{
|
|
58
|
+
"text": "done"
|
|
59
|
+
}
|
|
60
|
+
],
|
|
61
|
+
"steps": [
|
|
62
|
+
{
|
|
63
|
+
"name": "prompt",
|
|
64
|
+
"rpc": {
|
|
65
|
+
"type": "prompt",
|
|
66
|
+
"message": "please run echo hi"
|
|
67
|
+
}
|
|
68
|
+
},
|
|
69
|
+
{
|
|
70
|
+
"name": "last",
|
|
71
|
+
"rpc": {
|
|
72
|
+
"type": "prompt",
|
|
73
|
+
"message": "/jev last"
|
|
74
|
+
}
|
|
75
|
+
},
|
|
76
|
+
{
|
|
77
|
+
"name": "output",
|
|
78
|
+
"rpc": {
|
|
79
|
+
"type": "prompt",
|
|
80
|
+
"message": "/jev output"
|
|
81
|
+
}
|
|
82
|
+
},
|
|
83
|
+
{
|
|
84
|
+
"name": "mode-enforce",
|
|
85
|
+
"rpc": {
|
|
86
|
+
"type": "prompt",
|
|
87
|
+
"message": "/jev mode enforce"
|
|
88
|
+
}
|
|
89
|
+
},
|
|
90
|
+
{
|
|
91
|
+
"name": "mode-bogus",
|
|
92
|
+
"rpc": {
|
|
93
|
+
"type": "prompt",
|
|
94
|
+
"message": "/jev mode bogus"
|
|
95
|
+
}
|
|
96
|
+
},
|
|
97
|
+
{
|
|
98
|
+
"name": "check",
|
|
99
|
+
"rpc": {
|
|
100
|
+
"type": "prompt",
|
|
101
|
+
"message": "/jev check Rm -RF /tmp/Build"
|
|
102
|
+
}
|
|
103
|
+
},
|
|
104
|
+
{
|
|
105
|
+
"name": "off",
|
|
106
|
+
"rpc": {
|
|
107
|
+
"type": "prompt",
|
|
108
|
+
"message": "/jev off"
|
|
109
|
+
}
|
|
110
|
+
},
|
|
111
|
+
{
|
|
112
|
+
"name": "on",
|
|
113
|
+
"rpc": {
|
|
114
|
+
"type": "prompt",
|
|
115
|
+
"message": "/jev on"
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
]
|
|
119
|
+
}
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "enforce-accept",
|
|
3
|
+
"description": "Enforce mode, the user allows the call: it runs, the output is judged.",
|
|
4
|
+
"agentFiles": {
|
|
5
|
+
"pi-jev.json": "{\"enabled\": true, \"acknowledged\": true, \"display\": \"plain\", \"backend\": \"typesafe\", \"endpoint\": \"{{server:jev}}/v1/systemone\", \"model\": \"jev-eq\", \"retries\": 0, \"timeoutMs\": 5000, \"gate\": {\"mode\": \"enforce\"}}"
|
|
6
|
+
},
|
|
7
|
+
"env": {
|
|
8
|
+
"TYPESAFE_API_KEY": "tsk-eq-key-0123456789"
|
|
9
|
+
},
|
|
10
|
+
"servers": {
|
|
11
|
+
"jev": {
|
|
12
|
+
"recordHeaders": [
|
|
13
|
+
"authorization"
|
|
14
|
+
],
|
|
15
|
+
"routes": [
|
|
16
|
+
{
|
|
17
|
+
"method": "POST",
|
|
18
|
+
"path": "/v1/systemone",
|
|
19
|
+
"body": "{\"model\": \"jev-eq\", \"answers\": {\"destructive\": {\"type\": \"noul\", \"noul\": 0.99}, \"exfiltration\": {\"type\": \"noul\", \"noul\": 0.79}, \"beyond_scope\": {\"type\": \"noul\", \"noul\": 0.98}, \"impact\": {\"type\": \"score\", \"score\": 3.0, \"confidence\": 0.91, \"legend\": {\"0\": \"None, it only reads\", \"1\": \"Small\", \"2\": \"Large\", \"3\": \"Severe\"}, \"probabilities\": {\"0\": 0.05, \"1\": 0.1, \"2\": 0.15, \"3\": 0.7}}}, \"usage\": {\"input_tokens\": 100, \"output_tokens\": 4}}",
|
|
20
|
+
"headers": {
|
|
21
|
+
"Content-Type": "application/json"
|
|
22
|
+
},
|
|
23
|
+
"times": 1
|
|
24
|
+
},
|
|
25
|
+
{
|
|
26
|
+
"method": "POST",
|
|
27
|
+
"path": "/v1/systemone",
|
|
28
|
+
"body": "{\"model\": \"jev-eq\", \"answers\": {\"leaks_secret\": {\"type\": \"noul\", \"noul\": 0.01}, \"failure_class\": {\"type\": \"choice\", \"choice\": \"no_failure\", \"confidence\": 0.99, \"probabilities\": {\"no_failure\": 0.99}}}}",
|
|
29
|
+
"headers": {
|
|
30
|
+
"Content-Type": "application/json"
|
|
31
|
+
},
|
|
32
|
+
"times": 1
|
|
33
|
+
}
|
|
34
|
+
]
|
|
35
|
+
}
|
|
36
|
+
},
|
|
37
|
+
"llm": [
|
|
38
|
+
{
|
|
39
|
+
"toolCalls": [
|
|
40
|
+
{
|
|
41
|
+
"name": "bash",
|
|
42
|
+
"arguments": {
|
|
43
|
+
"command": "echo hi"
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
]
|
|
47
|
+
},
|
|
48
|
+
{
|
|
49
|
+
"text": "done"
|
|
50
|
+
}
|
|
51
|
+
],
|
|
52
|
+
"steps": [
|
|
53
|
+
{
|
|
54
|
+
"name": "prompt",
|
|
55
|
+
"rpc": {
|
|
56
|
+
"type": "prompt",
|
|
57
|
+
"message": "please run echo hi"
|
|
58
|
+
},
|
|
59
|
+
"ui": [
|
|
60
|
+
{
|
|
61
|
+
"confirmed": true
|
|
62
|
+
}
|
|
63
|
+
]
|
|
64
|
+
}
|
|
65
|
+
]
|
|
66
|
+
}
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "enforce-decline",
|
|
3
|
+
"description": "Enforce mode, the user declines the confirmation: the call is blocked with the gate's reason.",
|
|
4
|
+
"agentFiles": {
|
|
5
|
+
"pi-jev.json": "{\"enabled\": true, \"acknowledged\": true, \"display\": \"plain\", \"backend\": \"typesafe\", \"endpoint\": \"{{server:jev}}/v1/systemone\", \"model\": \"jev-eq\", \"retries\": 0, \"timeoutMs\": 5000, \"gate\": {\"mode\": \"enforce\"}}"
|
|
6
|
+
},
|
|
7
|
+
"env": {
|
|
8
|
+
"TYPESAFE_API_KEY": "tsk-eq-key-0123456789"
|
|
9
|
+
},
|
|
10
|
+
"servers": {
|
|
11
|
+
"jev": {
|
|
12
|
+
"recordHeaders": [
|
|
13
|
+
"authorization"
|
|
14
|
+
],
|
|
15
|
+
"routes": [
|
|
16
|
+
{
|
|
17
|
+
"method": "POST",
|
|
18
|
+
"path": "/v1/systemone",
|
|
19
|
+
"body": "{\"model\": \"jev-eq\", \"answers\": {\"destructive\": {\"type\": \"noul\", \"noul\": 0.99}, \"exfiltration\": {\"type\": \"noul\", \"noul\": 0.79}, \"beyond_scope\": {\"type\": \"noul\", \"noul\": 0.98}, \"impact\": {\"type\": \"score\", \"score\": 3.0, \"confidence\": 0.91, \"legend\": {\"0\": \"None, it only reads\", \"1\": \"Small\", \"2\": \"Large\", \"3\": \"Severe\"}, \"probabilities\": {\"0\": 0.05, \"1\": 0.1, \"2\": 0.15, \"3\": 0.7}}}, \"usage\": {\"input_tokens\": 100, \"output_tokens\": 4}}",
|
|
20
|
+
"headers": {
|
|
21
|
+
"Content-Type": "application/json"
|
|
22
|
+
},
|
|
23
|
+
"times": 1
|
|
24
|
+
}
|
|
25
|
+
]
|
|
26
|
+
}
|
|
27
|
+
},
|
|
28
|
+
"llm": [
|
|
29
|
+
{
|
|
30
|
+
"toolCalls": [
|
|
31
|
+
{
|
|
32
|
+
"name": "bash",
|
|
33
|
+
"arguments": {
|
|
34
|
+
"command": "echo hi"
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
]
|
|
38
|
+
},
|
|
39
|
+
{
|
|
40
|
+
"text": "blocked"
|
|
41
|
+
}
|
|
42
|
+
],
|
|
43
|
+
"steps": [
|
|
44
|
+
{
|
|
45
|
+
"name": "prompt",
|
|
46
|
+
"rpc": {
|
|
47
|
+
"type": "prompt",
|
|
48
|
+
"message": "please run echo hi"
|
|
49
|
+
},
|
|
50
|
+
"ui": [
|
|
51
|
+
{
|
|
52
|
+
"confirmed": false
|
|
53
|
+
}
|
|
54
|
+
]
|
|
55
|
+
}
|
|
56
|
+
]
|
|
57
|
+
}
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "jev-ask",
|
|
3
|
+
"description": "The model calls jev_ask with a noul, a choice and a score question.",
|
|
4
|
+
"agentFiles": {
|
|
5
|
+
"pi-jev.json": "{\"enabled\": true, \"acknowledged\": true, \"display\": \"plain\", \"backend\": \"typesafe\", \"endpoint\": \"{{server:jev}}/v1/systemone\", \"model\": \"jev-eq\", \"retries\": 0, \"timeoutMs\": 5000}"
|
|
6
|
+
},
|
|
7
|
+
"env": {
|
|
8
|
+
"TYPESAFE_API_KEY": "tsk-eq-key-0123456789"
|
|
9
|
+
},
|
|
10
|
+
"servers": {
|
|
11
|
+
"jev": {
|
|
12
|
+
"recordHeaders": [
|
|
13
|
+
"authorization"
|
|
14
|
+
],
|
|
15
|
+
"routes": [
|
|
16
|
+
{
|
|
17
|
+
"method": "POST",
|
|
18
|
+
"path": "/v1/systemone",
|
|
19
|
+
"body": "{\"model\": \"jev-eq\", \"usage\": {\"input_tokens\": 12, \"output_tokens\": 3}, \"answers\": {\"relevant\": {\"type\": \"noul\", \"noul\": 0.93}, \"label\": {\"type\": \"choice\", \"choice\": \"bug\", \"confidence\": 0.9, \"probabilities\": {\"feature\": 0.1, \"bug\": 0.9}}, \"quality\": {\"type\": \"score\", \"score\": 1.75, \"confidence\": 0.8, \"legend\": {\"0\": \"None, it only reads\", \"1\": \"Small\", \"2\": \"Large\", \"3\": \"Severe\"}, \"probabilities\": {\"0\": 0.05, \"1\": 0.1, \"2\": 0.15, \"3\": 0.7}}}}",
|
|
20
|
+
"headers": {
|
|
21
|
+
"Content-Type": "application/json"
|
|
22
|
+
},
|
|
23
|
+
"times": 1
|
|
24
|
+
}
|
|
25
|
+
]
|
|
26
|
+
}
|
|
27
|
+
},
|
|
28
|
+
"llm": [
|
|
29
|
+
{
|
|
30
|
+
"toolCalls": [
|
|
31
|
+
{
|
|
32
|
+
"name": "jev_ask",
|
|
33
|
+
"arguments": {
|
|
34
|
+
"state": "the diff",
|
|
35
|
+
"questions": [
|
|
36
|
+
{
|
|
37
|
+
"id": "relevant",
|
|
38
|
+
"type": "noul",
|
|
39
|
+
"instructions": "Is this relevant?"
|
|
40
|
+
},
|
|
41
|
+
{
|
|
42
|
+
"id": "label",
|
|
43
|
+
"type": "choice",
|
|
44
|
+
"instructions": "Which bucket?",
|
|
45
|
+
"options": [
|
|
46
|
+
{
|
|
47
|
+
"name": "bug",
|
|
48
|
+
"description": "Defect"
|
|
49
|
+
},
|
|
50
|
+
{
|
|
51
|
+
"name": "feature"
|
|
52
|
+
}
|
|
53
|
+
]
|
|
54
|
+
},
|
|
55
|
+
{
|
|
56
|
+
"id": "quality",
|
|
57
|
+
"type": "score",
|
|
58
|
+
"instructions": "How thorough?",
|
|
59
|
+
"levels": [
|
|
60
|
+
"Superficial",
|
|
61
|
+
"Adequate",
|
|
62
|
+
"Thorough"
|
|
63
|
+
]
|
|
64
|
+
}
|
|
65
|
+
]
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
]
|
|
69
|
+
},
|
|
70
|
+
{
|
|
71
|
+
"text": "asked"
|
|
72
|
+
}
|
|
73
|
+
],
|
|
74
|
+
"steps": [
|
|
75
|
+
{
|
|
76
|
+
"name": "prompt",
|
|
77
|
+
"rpc": {
|
|
78
|
+
"type": "prompt",
|
|
79
|
+
"message": "ask jev about the diff"
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
]
|
|
83
|
+
}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "output-advice",
|
|
3
|
+
"description": "The output judge classifies a failure: advice appended to the result, no notification.",
|
|
4
|
+
"agentFiles": {
|
|
5
|
+
"pi-jev.json": "{\"enabled\": true, \"acknowledged\": true, \"display\": \"plain\", \"backend\": \"typesafe\", \"endpoint\": \"{{server:jev}}/v1/systemone\", \"model\": \"jev-eq\", \"retries\": 0, \"timeoutMs\": 5000}"
|
|
6
|
+
},
|
|
7
|
+
"env": {
|
|
8
|
+
"TYPESAFE_API_KEY": "tsk-eq-key-0123456789"
|
|
9
|
+
},
|
|
10
|
+
"servers": {
|
|
11
|
+
"jev": {
|
|
12
|
+
"recordHeaders": [
|
|
13
|
+
"authorization"
|
|
14
|
+
],
|
|
15
|
+
"routes": [
|
|
16
|
+
{
|
|
17
|
+
"method": "POST",
|
|
18
|
+
"path": "/v1/systemone",
|
|
19
|
+
"body": "{\"model\": \"jev-eq\", \"answers\": {\"destructive\": {\"type\": \"noul\", \"noul\": 0.03}, \"exfiltration\": {\"type\": \"noul\", \"noul\": 0.04}, \"beyond_scope\": {\"type\": \"noul\", \"noul\": 0.4}, \"impact\": {\"type\": \"score\", \"score\": 0.02, \"confidence\": 0.9, \"legend\": {\"0\": \"None, it only reads\", \"1\": \"Small\", \"2\": \"Large\", \"3\": \"Severe\"}, \"probabilities\": {\"0\": 0.05, \"1\": 0.1, \"2\": 0.15, \"3\": 0.7}}}, \"usage\": {\"input_tokens\": 100, \"output_tokens\": 4}}",
|
|
20
|
+
"headers": {
|
|
21
|
+
"Content-Type": "application/json"
|
|
22
|
+
},
|
|
23
|
+
"times": 1
|
|
24
|
+
},
|
|
25
|
+
{
|
|
26
|
+
"method": "POST",
|
|
27
|
+
"path": "/v1/systemone",
|
|
28
|
+
"body": "{\"model\": \"jev-eq\", \"answers\": {\"leaks_secret\": {\"type\": \"noul\", \"noul\": 0.01}, \"failure_class\": {\"type\": \"choice\", \"choice\": \"transient\", \"confidence\": 1.0, \"probabilities\": {\"transient\": 1.0}}}}",
|
|
29
|
+
"headers": {
|
|
30
|
+
"Content-Type": "application/json"
|
|
31
|
+
},
|
|
32
|
+
"times": 1
|
|
33
|
+
}
|
|
34
|
+
]
|
|
35
|
+
}
|
|
36
|
+
},
|
|
37
|
+
"llm": [
|
|
38
|
+
{
|
|
39
|
+
"toolCalls": [
|
|
40
|
+
{
|
|
41
|
+
"name": "bash",
|
|
42
|
+
"arguments": {
|
|
43
|
+
"command": "echo timeout; exit 3"
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
]
|
|
47
|
+
},
|
|
48
|
+
{
|
|
49
|
+
"text": "retry"
|
|
50
|
+
}
|
|
51
|
+
],
|
|
52
|
+
"steps": [
|
|
53
|
+
{
|
|
54
|
+
"name": "prompt",
|
|
55
|
+
"rpc": {
|
|
56
|
+
"type": "prompt",
|
|
57
|
+
"message": "run the flaky command"
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
]
|
|
61
|
+
}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "output-leak",
|
|
3
|
+
"description": "The output judge flags a secret: a notification, a status, and a notice appended to the result the model reads.",
|
|
4
|
+
"agentFiles": {
|
|
5
|
+
"pi-jev.json": "{\"enabled\": true, \"acknowledged\": true, \"display\": \"plain\", \"backend\": \"typesafe\", \"endpoint\": \"{{server:jev}}/v1/systemone\", \"model\": \"jev-eq\", \"retries\": 0, \"timeoutMs\": 5000}"
|
|
6
|
+
},
|
|
7
|
+
"env": {
|
|
8
|
+
"TYPESAFE_API_KEY": "tsk-eq-key-0123456789"
|
|
9
|
+
},
|
|
10
|
+
"servers": {
|
|
11
|
+
"jev": {
|
|
12
|
+
"recordHeaders": [
|
|
13
|
+
"authorization"
|
|
14
|
+
],
|
|
15
|
+
"routes": [
|
|
16
|
+
{
|
|
17
|
+
"method": "POST",
|
|
18
|
+
"path": "/v1/systemone",
|
|
19
|
+
"body": "{\"model\": \"jev-eq\", \"answers\": {\"destructive\": {\"type\": \"noul\", \"noul\": 0.03}, \"exfiltration\": {\"type\": \"noul\", \"noul\": 0.04}, \"beyond_scope\": {\"type\": \"noul\", \"noul\": 0.4}, \"impact\": {\"type\": \"score\", \"score\": 0.02, \"confidence\": 0.9, \"legend\": {\"0\": \"None, it only reads\", \"1\": \"Small\", \"2\": \"Large\", \"3\": \"Severe\"}, \"probabilities\": {\"0\": 0.05, \"1\": 0.1, \"2\": 0.15, \"3\": 0.7}}}, \"usage\": {\"input_tokens\": 100, \"output_tokens\": 4}}",
|
|
20
|
+
"headers": {
|
|
21
|
+
"Content-Type": "application/json"
|
|
22
|
+
},
|
|
23
|
+
"times": 1
|
|
24
|
+
},
|
|
25
|
+
{
|
|
26
|
+
"method": "POST",
|
|
27
|
+
"path": "/v1/systemone",
|
|
28
|
+
"body": "{\"model\": \"jev-eq\", \"answers\": {\"leaks_secret\": {\"type\": \"noul\", \"noul\": 0.94}, \"failure_class\": {\"type\": \"choice\", \"choice\": \"no_failure\", \"confidence\": 0.9, \"probabilities\": {\"no_failure\": 0.9}}}}",
|
|
29
|
+
"headers": {
|
|
30
|
+
"Content-Type": "application/json"
|
|
31
|
+
},
|
|
32
|
+
"times": 1
|
|
33
|
+
}
|
|
34
|
+
]
|
|
35
|
+
}
|
|
36
|
+
},
|
|
37
|
+
"llm": [
|
|
38
|
+
{
|
|
39
|
+
"toolCalls": [
|
|
40
|
+
{
|
|
41
|
+
"name": "bash",
|
|
42
|
+
"arguments": {
|
|
43
|
+
"command": "echo API_KEY=abc123"
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
]
|
|
47
|
+
},
|
|
48
|
+
{
|
|
49
|
+
"text": "done"
|
|
50
|
+
}
|
|
51
|
+
],
|
|
52
|
+
"steps": [
|
|
53
|
+
{
|
|
54
|
+
"name": "prompt",
|
|
55
|
+
"rpc": {
|
|
56
|
+
"type": "prompt",
|
|
57
|
+
"message": "show me the key"
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
]
|
|
61
|
+
}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "output-low-confidence",
|
|
3
|
+
"description": "The failure class is below output.minConfidence: the result is left alone.",
|
|
4
|
+
"agentFiles": {
|
|
5
|
+
"pi-jev.json": "{\"enabled\": true, \"acknowledged\": true, \"display\": \"plain\", \"backend\": \"typesafe\", \"endpoint\": \"{{server:jev}}/v1/systemone\", \"model\": \"jev-eq\", \"retries\": 0, \"timeoutMs\": 5000}"
|
|
6
|
+
},
|
|
7
|
+
"env": {
|
|
8
|
+
"TYPESAFE_API_KEY": "tsk-eq-key-0123456789"
|
|
9
|
+
},
|
|
10
|
+
"servers": {
|
|
11
|
+
"jev": {
|
|
12
|
+
"recordHeaders": [
|
|
13
|
+
"authorization"
|
|
14
|
+
],
|
|
15
|
+
"routes": [
|
|
16
|
+
{
|
|
17
|
+
"method": "POST",
|
|
18
|
+
"path": "/v1/systemone",
|
|
19
|
+
"body": "{\"model\": \"jev-eq\", \"answers\": {\"destructive\": {\"type\": \"noul\", \"noul\": 0.03}, \"exfiltration\": {\"type\": \"noul\", \"noul\": 0.04}, \"beyond_scope\": {\"type\": \"noul\", \"noul\": 0.4}, \"impact\": {\"type\": \"score\", \"score\": 0.02, \"confidence\": 0.9, \"legend\": {\"0\": \"None, it only reads\", \"1\": \"Small\", \"2\": \"Large\", \"3\": \"Severe\"}, \"probabilities\": {\"0\": 0.05, \"1\": 0.1, \"2\": 0.15, \"3\": 0.7}}}, \"usage\": {\"input_tokens\": 100, \"output_tokens\": 4}}",
|
|
20
|
+
"headers": {
|
|
21
|
+
"Content-Type": "application/json"
|
|
22
|
+
},
|
|
23
|
+
"times": 1
|
|
24
|
+
},
|
|
25
|
+
{
|
|
26
|
+
"method": "POST",
|
|
27
|
+
"path": "/v1/systemone",
|
|
28
|
+
"body": "{\"model\": \"jev-eq\", \"answers\": {\"leaks_secret\": {\"type\": \"noul\", \"noul\": 0.01}, \"failure_class\": {\"type\": \"choice\", \"choice\": \"environment\", \"confidence\": 0.42, \"probabilities\": {\"environment\": 0.42}}}}",
|
|
29
|
+
"headers": {
|
|
30
|
+
"Content-Type": "application/json"
|
|
31
|
+
},
|
|
32
|
+
"times": 1
|
|
33
|
+
}
|
|
34
|
+
]
|
|
35
|
+
}
|
|
36
|
+
},
|
|
37
|
+
"llm": [
|
|
38
|
+
{
|
|
39
|
+
"toolCalls": [
|
|
40
|
+
{
|
|
41
|
+
"name": "bash",
|
|
42
|
+
"arguments": {
|
|
43
|
+
"command": "echo odd; exit 2"
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
]
|
|
47
|
+
},
|
|
48
|
+
{
|
|
49
|
+
"text": "hm"
|
|
50
|
+
}
|
|
51
|
+
],
|
|
52
|
+
"steps": [
|
|
53
|
+
{
|
|
54
|
+
"name": "prompt",
|
|
55
|
+
"rpc": {
|
|
56
|
+
"type": "prompt",
|
|
57
|
+
"message": "run it"
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
]
|
|
61
|
+
}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "shadow-flagged",
|
|
3
|
+
"description": "A destructive-looking bash call in shadow mode: one gate request, a warning and a status, the call runs, the clean output gets no notice.",
|
|
4
|
+
"agentFiles": {
|
|
5
|
+
"pi-jev.json": "{\"enabled\": true, \"acknowledged\": true, \"display\": \"plain\", \"backend\": \"typesafe\", \"endpoint\": \"{{server:jev}}/v1/systemone\", \"model\": \"jev-eq\", \"retries\": 0, \"timeoutMs\": 5000}"
|
|
6
|
+
},
|
|
7
|
+
"env": {
|
|
8
|
+
"TYPESAFE_API_KEY": "tsk-eq-key-0123456789"
|
|
9
|
+
},
|
|
10
|
+
"servers": {
|
|
11
|
+
"jev": {
|
|
12
|
+
"recordHeaders": [
|
|
13
|
+
"authorization"
|
|
14
|
+
],
|
|
15
|
+
"routes": [
|
|
16
|
+
{
|
|
17
|
+
"method": "POST",
|
|
18
|
+
"path": "/v1/systemone",
|
|
19
|
+
"body": "{\"model\": \"jev-eq\", \"answers\": {\"destructive\": {\"type\": \"noul\", \"noul\": 0.99}, \"exfiltration\": {\"type\": \"noul\", \"noul\": 0.79}, \"beyond_scope\": {\"type\": \"noul\", \"noul\": 0.98}, \"impact\": {\"type\": \"score\", \"score\": 3.0, \"confidence\": 0.91, \"legend\": {\"0\": \"None, it only reads\", \"1\": \"Small\", \"2\": \"Large\", \"3\": \"Severe\"}, \"probabilities\": {\"0\": 0.05, \"1\": 0.1, \"2\": 0.15, \"3\": 0.7}}}, \"usage\": {\"input_tokens\": 100, \"output_tokens\": 4}}",
|
|
20
|
+
"headers": {
|
|
21
|
+
"Content-Type": "application/json"
|
|
22
|
+
},
|
|
23
|
+
"times": 1
|
|
24
|
+
},
|
|
25
|
+
{
|
|
26
|
+
"method": "POST",
|
|
27
|
+
"path": "/v1/systemone",
|
|
28
|
+
"body": "{\"model\": \"jev-eq\", \"answers\": {\"leaks_secret\": {\"type\": \"noul\", \"noul\": 0.01}, \"failure_class\": {\"type\": \"choice\", \"choice\": \"no_failure\", \"confidence\": 0.99, \"probabilities\": {\"no_failure\": 0.99}}}}",
|
|
29
|
+
"headers": {
|
|
30
|
+
"Content-Type": "application/json"
|
|
31
|
+
},
|
|
32
|
+
"times": 1
|
|
33
|
+
}
|
|
34
|
+
]
|
|
35
|
+
}
|
|
36
|
+
},
|
|
37
|
+
"llm": [
|
|
38
|
+
{
|
|
39
|
+
"toolCalls": [
|
|
40
|
+
{
|
|
41
|
+
"name": "bash",
|
|
42
|
+
"arguments": {
|
|
43
|
+
"command": "echo hi"
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
]
|
|
47
|
+
},
|
|
48
|
+
{
|
|
49
|
+
"text": "done"
|
|
50
|
+
}
|
|
51
|
+
],
|
|
52
|
+
"steps": [
|
|
53
|
+
{
|
|
54
|
+
"name": "prompt",
|
|
55
|
+
"rpc": {
|
|
56
|
+
"type": "prompt",
|
|
57
|
+
"message": "please run echo hi"
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
]
|
|
61
|
+
}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "unjudged-tools",
|
|
3
|
+
"description": "read is not a judged tool: no request at all.",
|
|
4
|
+
"agentFiles": {
|
|
5
|
+
"pi-jev.json": "{\"enabled\": true, \"acknowledged\": true, \"display\": \"plain\", \"backend\": \"typesafe\", \"endpoint\": \"{{server:jev}}/v1/systemone\", \"model\": \"jev-eq\", \"retries\": 0, \"timeoutMs\": 5000}"
|
|
6
|
+
},
|
|
7
|
+
"env": {
|
|
8
|
+
"TYPESAFE_API_KEY": "tsk-eq-key-0123456789"
|
|
9
|
+
},
|
|
10
|
+
"servers": {
|
|
11
|
+
"jev": {
|
|
12
|
+
"recordHeaders": [
|
|
13
|
+
"authorization"
|
|
14
|
+
],
|
|
15
|
+
"routes": [
|
|
16
|
+
{
|
|
17
|
+
"method": "POST",
|
|
18
|
+
"path": "/v1/systemone",
|
|
19
|
+
"body": "{\"model\": \"jev-eq\", \"answers\": {\"destructive\": {\"type\": \"noul\", \"noul\": 0.03}, \"exfiltration\": {\"type\": \"noul\", \"noul\": 0.04}, \"beyond_scope\": {\"type\": \"noul\", \"noul\": 0.4}, \"impact\": {\"type\": \"score\", \"score\": 0.02, \"confidence\": 0.9, \"legend\": {\"0\": \"None, it only reads\", \"1\": \"Small\", \"2\": \"Large\", \"3\": \"Severe\"}, \"probabilities\": {\"0\": 0.05, \"1\": 0.1, \"2\": 0.15, \"3\": 0.7}}}, \"usage\": {\"input_tokens\": 100, \"output_tokens\": 4}}",
|
|
20
|
+
"headers": {
|
|
21
|
+
"Content-Type": "application/json"
|
|
22
|
+
},
|
|
23
|
+
"times": 0
|
|
24
|
+
}
|
|
25
|
+
]
|
|
26
|
+
}
|
|
27
|
+
},
|
|
28
|
+
"files": {
|
|
29
|
+
"a.txt": "hello\n"
|
|
30
|
+
},
|
|
31
|
+
"llm": [
|
|
32
|
+
{
|
|
33
|
+
"toolCalls": [
|
|
34
|
+
{
|
|
35
|
+
"name": "read",
|
|
36
|
+
"arguments": {
|
|
37
|
+
"path": "a.txt"
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
]
|
|
41
|
+
},
|
|
42
|
+
{
|
|
43
|
+
"text": "read"
|
|
44
|
+
}
|
|
45
|
+
],
|
|
46
|
+
"steps": [
|
|
47
|
+
{
|
|
48
|
+
"name": "prompt",
|
|
49
|
+
"rpc": {
|
|
50
|
+
"type": "prompt",
|
|
51
|
+
"message": "read a.txt"
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
]
|
|
55
|
+
}
|