@pi-in-go/pigpen-jev 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (128) hide show
  1. package/CREDITS.md +22 -0
  2. package/LICENSE +22 -0
  3. package/README.md +237 -0
  4. package/extensions/jev/ask.go +166 -0
  5. package/extensions/jev/ask_test.go +218 -0
  6. package/extensions/jev/backend.go +128 -0
  7. package/extensions/jev/bench_test.go +64 -0
  8. package/extensions/jev/boundaries_test.go +159 -0
  9. package/extensions/jev/command.go +224 -0
  10. package/extensions/jev/commands_test.go +214 -0
  11. package/extensions/jev/config.go +450 -0
  12. package/extensions/jev/errors_test.go +191 -0
  13. package/extensions/jev/extension.go +391 -0
  14. package/extensions/jev/fakehost_test.go +548 -0
  15. package/extensions/jev/gate.go +125 -0
  16. package/extensions/jev/gate_test.go +610 -0
  17. package/extensions/jev/gatekey_test.go +24 -0
  18. package/extensions/jev/go.mod +9 -0
  19. package/extensions/jev/go.sum +2 -0
  20. package/extensions/jev/go.work +10 -0
  21. package/extensions/jev/helpers_test.go +404 -0
  22. package/extensions/jev/memo.go +88 -0
  23. package/extensions/jev/output.go +89 -0
  24. package/extensions/jev/output_test.go +187 -0
  25. package/extensions/jev/ownmodel_test.go +118 -0
  26. package/extensions/jev/render.go +136 -0
  27. package/extensions/jev/review_test.go +310 -0
  28. package/extensions/jev/source_test.go +57 -0
  29. package/extensions/jev/text.go +174 -0
  30. package/extensions/jev/trust_test.go +335 -0
  31. package/extensions/jev/types.go +227 -0
  32. package/libs/typesafe/CONTRACT.md +125 -0
  33. package/libs/typesafe/CREDITS.md +37 -0
  34. package/libs/typesafe/LICENSE +23 -0
  35. package/libs/typesafe/README.md +19 -0
  36. package/libs/typesafe/go.mod +3 -0
  37. package/libs/typesafe/libraries/ownmodel/backend_test.go +496 -0
  38. package/libs/typesafe/libraries/ownmodel/canon.go +190 -0
  39. package/libs/typesafe/libraries/ownmodel/convert.go +199 -0
  40. package/libs/typesafe/libraries/ownmodel/doc.go +15 -0
  41. package/libs/typesafe/libraries/ownmodel/equivalence_test.go +199 -0
  42. package/libs/typesafe/libraries/ownmodel/helpers_test.go +155 -0
  43. package/libs/typesafe/libraries/ownmodel/mutation_test.go +31 -0
  44. package/libs/typesafe/libraries/ownmodel/ownmodel.go +225 -0
  45. package/libs/typesafe/libraries/ownmodel/plan.go +442 -0
  46. package/libs/typesafe/libraries/ownmodel/run.go +288 -0
  47. package/libs/typesafe/libraries/ownmodel/schema_test.go +254 -0
  48. package/libs/typesafe/libraries/ownmodel/twins_test.go +169 -0
  49. package/libs/typesafe/libraries/ownmodel/utils_test.go +125 -0
  50. package/libs/typesafe/libraries/pigmodel/pigmodel.go +264 -0
  51. package/libs/typesafe/libraries/pigmodel/pigmodel_test.go +410 -0
  52. package/libs/typesafe/libraries/typesafe/answers.go +268 -0
  53. package/libs/typesafe/libraries/typesafe/api_response_test.go +113 -0
  54. package/libs/typesafe/libraries/typesafe/batch.go +80 -0
  55. package/libs/typesafe/libraries/typesafe/batch_test.go +133 -0
  56. package/libs/typesafe/libraries/typesafe/bench_test.go +71 -0
  57. package/libs/typesafe/libraries/typesafe/client.go +561 -0
  58. package/libs/typesafe/libraries/typesafe/client_test.go +495 -0
  59. package/libs/typesafe/libraries/typesafe/crosscheck_test.go +464 -0
  60. package/libs/typesafe/libraries/typesafe/crosscheck_workflowevals_test.go +219 -0
  61. package/libs/typesafe/libraries/typesafe/doc.go +27 -0
  62. package/libs/typesafe/libraries/typesafe/entry.go +142 -0
  63. package/libs/typesafe/libraries/typesafe/env.go +11 -0
  64. package/libs/typesafe/libraries/typesafe/errors.go +310 -0
  65. package/libs/typesafe/libraries/typesafe/errors_test.go +175 -0
  66. package/libs/typesafe/libraries/typesafe/helpers_test.go +294 -0
  67. package/libs/typesafe/libraries/typesafe/live_test.go +96 -0
  68. package/libs/typesafe/libraries/typesafe/logging.go +160 -0
  69. package/libs/typesafe/libraries/typesafe/logging_test.go +259 -0
  70. package/libs/typesafe/libraries/typesafe/marshal_test.go +112 -0
  71. package/libs/typesafe/libraries/typesafe/mutation_test.go +39 -0
  72. package/libs/typesafe/libraries/typesafe/questions.go +490 -0
  73. package/libs/typesafe/libraries/typesafe/questions_test.go +166 -0
  74. package/libs/typesafe/libraries/typesafe/regressions_test.go +159 -0
  75. package/libs/typesafe/libraries/typesafe/reliability_test.go +649 -0
  76. package/libs/typesafe/libraries/typesafe/retry.go +350 -0
  77. package/libs/typesafe/libraries/typesafe/retry_test.go +297 -0
  78. package/libs/typesafe/libraries/typesafe/runtime_test.go +26 -0
  79. package/libs/typesafe/libraries/typesafe/transport_test.go +163 -0
  80. package/libs/typesafe/libraries/typesafe/twins_test.go +127 -0
  81. package/libs/typesafe/libraries/typesafe/types_test.go +165 -0
  82. package/libs/typesafe/libraries/typesafe/version.go +10 -0
  83. package/libs/typesafe/package.json +37 -0
  84. package/libs/typesafe/provenance.json +49 -0
  85. package/package.json +42 -0
  86. package/port/PORT.md +107 -0
  87. package/port/e2e/gate-and-output.py +35 -0
  88. package/port/e2e/jev-ask.py +36 -0
  89. package/port/e2e/model-switch.py +44 -0
  90. package/port/e2e/off-by-default.py +34 -0
  91. package/port/gen-scenarios.py +103 -0
  92. package/port/golden/cache-identical-calls.jsonl +30 -0
  93. package/port/golden/clear.jsonl +22 -0
  94. package/port/golden/commands.jsonl +43 -0
  95. package/port/golden/enforce-accept.jsonl +23 -0
  96. package/port/golden/enforce-decline.jsonl +22 -0
  97. package/port/golden/jev-ask.jsonl +20 -0
  98. package/port/golden/output-advice.jsonl +23 -0
  99. package/port/golden/output-leak.jsonl +24 -0
  100. package/port/golden/output-low-confidence.jsonl +22 -0
  101. package/port/golden/shadow-flagged.jsonl +23 -0
  102. package/port/golden/unjudged-tools.jsonl +19 -0
  103. package/port/golden/write-elision.jsonl +21 -0
  104. package/port/mutate-unit.py +63 -0
  105. package/port/mutations.json +578 -0
  106. package/port/oracle/LICENSE +21 -0
  107. package/port/oracle/README.md +181 -0
  108. package/port/oracle/SHA256SUMS +8 -0
  109. package/port/oracle/package.json +43 -0
  110. package/port/oracle/src/client.ts +409 -0
  111. package/port/oracle/src/config.ts +363 -0
  112. package/port/oracle/src/gate.ts +229 -0
  113. package/port/oracle/src/index.ts +649 -0
  114. package/port/oracle/src/output.ts +163 -0
  115. package/port/red-run.log +309 -0
  116. package/port/scenarios/cache-identical-calls.json +71 -0
  117. package/port/scenarios/clear.json +61 -0
  118. package/port/scenarios/commands.json +119 -0
  119. package/port/scenarios/enforce-accept.json +66 -0
  120. package/port/scenarios/enforce-decline.json +57 -0
  121. package/port/scenarios/jev-ask.json +83 -0
  122. package/port/scenarios/output-advice.json +61 -0
  123. package/port/scenarios/output-leak.json +61 -0
  124. package/port/scenarios/output-low-confidence.json +61 -0
  125. package/port/scenarios/shadow-flagged.json +61 -0
  126. package/port/scenarios/unjudged-tools.json +55 -0
  127. package/port/scenarios/write-elision.json +53 -0
  128. package/provenance.json +18 -0
@@ -0,0 +1,119 @@
1
+ {
2
+ "name": "commands",
3
+ "description": "/jev mode, last, output, check, off and on.",
4
+ "agentFiles": {
5
+ "pi-jev.json": "{\"enabled\": true, \"acknowledged\": true, \"display\": \"plain\", \"backend\": \"typesafe\", \"endpoint\": \"{{server:jev}}/v1/systemone\", \"model\": \"jev-eq\", \"retries\": 0, \"timeoutMs\": 5000}"
6
+ },
7
+ "env": {
8
+ "TYPESAFE_API_KEY": "tsk-eq-key-0123456789"
9
+ },
10
+ "servers": {
11
+ "jev": {
12
+ "recordHeaders": [
13
+ "authorization"
14
+ ],
15
+ "routes": [
16
+ {
17
+ "method": "POST",
18
+ "path": "/v1/systemone",
19
+ "body": "{\"model\": \"jev-eq\", \"answers\": {\"destructive\": {\"type\": \"noul\", \"noul\": 0.99}, \"exfiltration\": {\"type\": \"noul\", \"noul\": 0.79}, \"beyond_scope\": {\"type\": \"noul\", \"noul\": 0.98}, \"impact\": {\"type\": \"score\", \"score\": 3.0, \"confidence\": 0.91, \"legend\": {\"0\": \"None, it only reads\", \"1\": \"Small\", \"2\": \"Large\", \"3\": \"Severe\"}, \"probabilities\": {\"0\": 0.05, \"1\": 0.1, \"2\": 0.15, \"3\": 0.7}}}, \"usage\": {\"input_tokens\": 100, \"output_tokens\": 4}}",
20
+ "headers": {
21
+ "Content-Type": "application/json"
22
+ },
23
+ "times": 1
24
+ },
25
+ {
26
+ "method": "POST",
27
+ "path": "/v1/systemone",
28
+ "body": "{\"model\": \"jev-eq\", \"answers\": {\"leaks_secret\": {\"type\": \"noul\", \"noul\": 0.94}, \"failure_class\": {\"type\": \"choice\", \"choice\": \"transient\", \"confidence\": 0.8, \"probabilities\": {\"transient\": 0.8}}}}",
29
+ "headers": {
30
+ "Content-Type": "application/json"
31
+ },
32
+ "times": 1
33
+ },
34
+ {
35
+ "method": "POST",
36
+ "path": "/v1/systemone",
37
+ "body": "{\"model\": \"jev-eq\", \"answers\": {\"destructive\": {\"type\": \"noul\", \"noul\": 0.99}, \"exfiltration\": {\"type\": \"noul\", \"noul\": 0.1}, \"beyond_scope\": {\"type\": \"noul\", \"noul\": 0.1}, \"impact\": {\"type\": \"score\", \"score\": 1.0, \"confidence\": 0.9, \"legend\": {\"0\": \"None, it only reads\", \"1\": \"Small\", \"2\": \"Large\", \"3\": \"Severe\"}, \"probabilities\": {\"0\": 0.05, \"1\": 0.1, \"2\": 0.15, \"3\": 0.7}}}, \"usage\": {\"input_tokens\": 100, \"output_tokens\": 4}}",
38
+ "headers": {
39
+ "Content-Type": "application/json"
40
+ },
41
+ "times": 1
42
+ }
43
+ ]
44
+ }
45
+ },
46
+ "llm": [
47
+ {
48
+ "toolCalls": [
49
+ {
50
+ "name": "bash",
51
+ "arguments": {
52
+ "command": "echo hi"
53
+ }
54
+ }
55
+ ]
56
+ },
57
+ {
58
+ "text": "done"
59
+ }
60
+ ],
61
+ "steps": [
62
+ {
63
+ "name": "prompt",
64
+ "rpc": {
65
+ "type": "prompt",
66
+ "message": "please run echo hi"
67
+ }
68
+ },
69
+ {
70
+ "name": "last",
71
+ "rpc": {
72
+ "type": "prompt",
73
+ "message": "/jev last"
74
+ }
75
+ },
76
+ {
77
+ "name": "output",
78
+ "rpc": {
79
+ "type": "prompt",
80
+ "message": "/jev output"
81
+ }
82
+ },
83
+ {
84
+ "name": "mode-enforce",
85
+ "rpc": {
86
+ "type": "prompt",
87
+ "message": "/jev mode enforce"
88
+ }
89
+ },
90
+ {
91
+ "name": "mode-bogus",
92
+ "rpc": {
93
+ "type": "prompt",
94
+ "message": "/jev mode bogus"
95
+ }
96
+ },
97
+ {
98
+ "name": "check",
99
+ "rpc": {
100
+ "type": "prompt",
101
+ "message": "/jev check Rm -RF /tmp/Build"
102
+ }
103
+ },
104
+ {
105
+ "name": "off",
106
+ "rpc": {
107
+ "type": "prompt",
108
+ "message": "/jev off"
109
+ }
110
+ },
111
+ {
112
+ "name": "on",
113
+ "rpc": {
114
+ "type": "prompt",
115
+ "message": "/jev on"
116
+ }
117
+ }
118
+ ]
119
+ }
@@ -0,0 +1,66 @@
1
+ {
2
+ "name": "enforce-accept",
3
+ "description": "Enforce mode, the user allows the call: it runs, the output is judged.",
4
+ "agentFiles": {
5
+ "pi-jev.json": "{\"enabled\": true, \"acknowledged\": true, \"display\": \"plain\", \"backend\": \"typesafe\", \"endpoint\": \"{{server:jev}}/v1/systemone\", \"model\": \"jev-eq\", \"retries\": 0, \"timeoutMs\": 5000, \"gate\": {\"mode\": \"enforce\"}}"
6
+ },
7
+ "env": {
8
+ "TYPESAFE_API_KEY": "tsk-eq-key-0123456789"
9
+ },
10
+ "servers": {
11
+ "jev": {
12
+ "recordHeaders": [
13
+ "authorization"
14
+ ],
15
+ "routes": [
16
+ {
17
+ "method": "POST",
18
+ "path": "/v1/systemone",
19
+ "body": "{\"model\": \"jev-eq\", \"answers\": {\"destructive\": {\"type\": \"noul\", \"noul\": 0.99}, \"exfiltration\": {\"type\": \"noul\", \"noul\": 0.79}, \"beyond_scope\": {\"type\": \"noul\", \"noul\": 0.98}, \"impact\": {\"type\": \"score\", \"score\": 3.0, \"confidence\": 0.91, \"legend\": {\"0\": \"None, it only reads\", \"1\": \"Small\", \"2\": \"Large\", \"3\": \"Severe\"}, \"probabilities\": {\"0\": 0.05, \"1\": 0.1, \"2\": 0.15, \"3\": 0.7}}}, \"usage\": {\"input_tokens\": 100, \"output_tokens\": 4}}",
20
+ "headers": {
21
+ "Content-Type": "application/json"
22
+ },
23
+ "times": 1
24
+ },
25
+ {
26
+ "method": "POST",
27
+ "path": "/v1/systemone",
28
+ "body": "{\"model\": \"jev-eq\", \"answers\": {\"leaks_secret\": {\"type\": \"noul\", \"noul\": 0.01}, \"failure_class\": {\"type\": \"choice\", \"choice\": \"no_failure\", \"confidence\": 0.99, \"probabilities\": {\"no_failure\": 0.99}}}}",
29
+ "headers": {
30
+ "Content-Type": "application/json"
31
+ },
32
+ "times": 1
33
+ }
34
+ ]
35
+ }
36
+ },
37
+ "llm": [
38
+ {
39
+ "toolCalls": [
40
+ {
41
+ "name": "bash",
42
+ "arguments": {
43
+ "command": "echo hi"
44
+ }
45
+ }
46
+ ]
47
+ },
48
+ {
49
+ "text": "done"
50
+ }
51
+ ],
52
+ "steps": [
53
+ {
54
+ "name": "prompt",
55
+ "rpc": {
56
+ "type": "prompt",
57
+ "message": "please run echo hi"
58
+ },
59
+ "ui": [
60
+ {
61
+ "confirmed": true
62
+ }
63
+ ]
64
+ }
65
+ ]
66
+ }
@@ -0,0 +1,57 @@
1
+ {
2
+ "name": "enforce-decline",
3
+ "description": "Enforce mode, the user declines the confirmation: the call is blocked with the gate's reason.",
4
+ "agentFiles": {
5
+ "pi-jev.json": "{\"enabled\": true, \"acknowledged\": true, \"display\": \"plain\", \"backend\": \"typesafe\", \"endpoint\": \"{{server:jev}}/v1/systemone\", \"model\": \"jev-eq\", \"retries\": 0, \"timeoutMs\": 5000, \"gate\": {\"mode\": \"enforce\"}}"
6
+ },
7
+ "env": {
8
+ "TYPESAFE_API_KEY": "tsk-eq-key-0123456789"
9
+ },
10
+ "servers": {
11
+ "jev": {
12
+ "recordHeaders": [
13
+ "authorization"
14
+ ],
15
+ "routes": [
16
+ {
17
+ "method": "POST",
18
+ "path": "/v1/systemone",
19
+ "body": "{\"model\": \"jev-eq\", \"answers\": {\"destructive\": {\"type\": \"noul\", \"noul\": 0.99}, \"exfiltration\": {\"type\": \"noul\", \"noul\": 0.79}, \"beyond_scope\": {\"type\": \"noul\", \"noul\": 0.98}, \"impact\": {\"type\": \"score\", \"score\": 3.0, \"confidence\": 0.91, \"legend\": {\"0\": \"None, it only reads\", \"1\": \"Small\", \"2\": \"Large\", \"3\": \"Severe\"}, \"probabilities\": {\"0\": 0.05, \"1\": 0.1, \"2\": 0.15, \"3\": 0.7}}}, \"usage\": {\"input_tokens\": 100, \"output_tokens\": 4}}",
20
+ "headers": {
21
+ "Content-Type": "application/json"
22
+ },
23
+ "times": 1
24
+ }
25
+ ]
26
+ }
27
+ },
28
+ "llm": [
29
+ {
30
+ "toolCalls": [
31
+ {
32
+ "name": "bash",
33
+ "arguments": {
34
+ "command": "echo hi"
35
+ }
36
+ }
37
+ ]
38
+ },
39
+ {
40
+ "text": "blocked"
41
+ }
42
+ ],
43
+ "steps": [
44
+ {
45
+ "name": "prompt",
46
+ "rpc": {
47
+ "type": "prompt",
48
+ "message": "please run echo hi"
49
+ },
50
+ "ui": [
51
+ {
52
+ "confirmed": false
53
+ }
54
+ ]
55
+ }
56
+ ]
57
+ }
@@ -0,0 +1,83 @@
1
+ {
2
+ "name": "jev-ask",
3
+ "description": "The model calls jev_ask with a noul, a choice and a score question.",
4
+ "agentFiles": {
5
+ "pi-jev.json": "{\"enabled\": true, \"acknowledged\": true, \"display\": \"plain\", \"backend\": \"typesafe\", \"endpoint\": \"{{server:jev}}/v1/systemone\", \"model\": \"jev-eq\", \"retries\": 0, \"timeoutMs\": 5000}"
6
+ },
7
+ "env": {
8
+ "TYPESAFE_API_KEY": "tsk-eq-key-0123456789"
9
+ },
10
+ "servers": {
11
+ "jev": {
12
+ "recordHeaders": [
13
+ "authorization"
14
+ ],
15
+ "routes": [
16
+ {
17
+ "method": "POST",
18
+ "path": "/v1/systemone",
19
+ "body": "{\"model\": \"jev-eq\", \"usage\": {\"input_tokens\": 12, \"output_tokens\": 3}, \"answers\": {\"relevant\": {\"type\": \"noul\", \"noul\": 0.93}, \"label\": {\"type\": \"choice\", \"choice\": \"bug\", \"confidence\": 0.9, \"probabilities\": {\"feature\": 0.1, \"bug\": 0.9}}, \"quality\": {\"type\": \"score\", \"score\": 1.75, \"confidence\": 0.8, \"legend\": {\"0\": \"None, it only reads\", \"1\": \"Small\", \"2\": \"Large\", \"3\": \"Severe\"}, \"probabilities\": {\"0\": 0.05, \"1\": 0.1, \"2\": 0.15, \"3\": 0.7}}}}",
20
+ "headers": {
21
+ "Content-Type": "application/json"
22
+ },
23
+ "times": 1
24
+ }
25
+ ]
26
+ }
27
+ },
28
+ "llm": [
29
+ {
30
+ "toolCalls": [
31
+ {
32
+ "name": "jev_ask",
33
+ "arguments": {
34
+ "state": "the diff",
35
+ "questions": [
36
+ {
37
+ "id": "relevant",
38
+ "type": "noul",
39
+ "instructions": "Is this relevant?"
40
+ },
41
+ {
42
+ "id": "label",
43
+ "type": "choice",
44
+ "instructions": "Which bucket?",
45
+ "options": [
46
+ {
47
+ "name": "bug",
48
+ "description": "Defect"
49
+ },
50
+ {
51
+ "name": "feature"
52
+ }
53
+ ]
54
+ },
55
+ {
56
+ "id": "quality",
57
+ "type": "score",
58
+ "instructions": "How thorough?",
59
+ "levels": [
60
+ "Superficial",
61
+ "Adequate",
62
+ "Thorough"
63
+ ]
64
+ }
65
+ ]
66
+ }
67
+ }
68
+ ]
69
+ },
70
+ {
71
+ "text": "asked"
72
+ }
73
+ ],
74
+ "steps": [
75
+ {
76
+ "name": "prompt",
77
+ "rpc": {
78
+ "type": "prompt",
79
+ "message": "ask jev about the diff"
80
+ }
81
+ }
82
+ ]
83
+ }
@@ -0,0 +1,61 @@
1
+ {
2
+ "name": "output-advice",
3
+ "description": "The output judge classifies a failure: advice appended to the result, no notification.",
4
+ "agentFiles": {
5
+ "pi-jev.json": "{\"enabled\": true, \"acknowledged\": true, \"display\": \"plain\", \"backend\": \"typesafe\", \"endpoint\": \"{{server:jev}}/v1/systemone\", \"model\": \"jev-eq\", \"retries\": 0, \"timeoutMs\": 5000}"
6
+ },
7
+ "env": {
8
+ "TYPESAFE_API_KEY": "tsk-eq-key-0123456789"
9
+ },
10
+ "servers": {
11
+ "jev": {
12
+ "recordHeaders": [
13
+ "authorization"
14
+ ],
15
+ "routes": [
16
+ {
17
+ "method": "POST",
18
+ "path": "/v1/systemone",
19
+ "body": "{\"model\": \"jev-eq\", \"answers\": {\"destructive\": {\"type\": \"noul\", \"noul\": 0.03}, \"exfiltration\": {\"type\": \"noul\", \"noul\": 0.04}, \"beyond_scope\": {\"type\": \"noul\", \"noul\": 0.4}, \"impact\": {\"type\": \"score\", \"score\": 0.02, \"confidence\": 0.9, \"legend\": {\"0\": \"None, it only reads\", \"1\": \"Small\", \"2\": \"Large\", \"3\": \"Severe\"}, \"probabilities\": {\"0\": 0.05, \"1\": 0.1, \"2\": 0.15, \"3\": 0.7}}}, \"usage\": {\"input_tokens\": 100, \"output_tokens\": 4}}",
20
+ "headers": {
21
+ "Content-Type": "application/json"
22
+ },
23
+ "times": 1
24
+ },
25
+ {
26
+ "method": "POST",
27
+ "path": "/v1/systemone",
28
+ "body": "{\"model\": \"jev-eq\", \"answers\": {\"leaks_secret\": {\"type\": \"noul\", \"noul\": 0.01}, \"failure_class\": {\"type\": \"choice\", \"choice\": \"transient\", \"confidence\": 1.0, \"probabilities\": {\"transient\": 1.0}}}}",
29
+ "headers": {
30
+ "Content-Type": "application/json"
31
+ },
32
+ "times": 1
33
+ }
34
+ ]
35
+ }
36
+ },
37
+ "llm": [
38
+ {
39
+ "toolCalls": [
40
+ {
41
+ "name": "bash",
42
+ "arguments": {
43
+ "command": "echo timeout; exit 3"
44
+ }
45
+ }
46
+ ]
47
+ },
48
+ {
49
+ "text": "retry"
50
+ }
51
+ ],
52
+ "steps": [
53
+ {
54
+ "name": "prompt",
55
+ "rpc": {
56
+ "type": "prompt",
57
+ "message": "run the flaky command"
58
+ }
59
+ }
60
+ ]
61
+ }
@@ -0,0 +1,61 @@
1
+ {
2
+ "name": "output-leak",
3
+ "description": "The output judge flags a secret: a notification, a status, and a notice appended to the result the model reads.",
4
+ "agentFiles": {
5
+ "pi-jev.json": "{\"enabled\": true, \"acknowledged\": true, \"display\": \"plain\", \"backend\": \"typesafe\", \"endpoint\": \"{{server:jev}}/v1/systemone\", \"model\": \"jev-eq\", \"retries\": 0, \"timeoutMs\": 5000}"
6
+ },
7
+ "env": {
8
+ "TYPESAFE_API_KEY": "tsk-eq-key-0123456789"
9
+ },
10
+ "servers": {
11
+ "jev": {
12
+ "recordHeaders": [
13
+ "authorization"
14
+ ],
15
+ "routes": [
16
+ {
17
+ "method": "POST",
18
+ "path": "/v1/systemone",
19
+ "body": "{\"model\": \"jev-eq\", \"answers\": {\"destructive\": {\"type\": \"noul\", \"noul\": 0.03}, \"exfiltration\": {\"type\": \"noul\", \"noul\": 0.04}, \"beyond_scope\": {\"type\": \"noul\", \"noul\": 0.4}, \"impact\": {\"type\": \"score\", \"score\": 0.02, \"confidence\": 0.9, \"legend\": {\"0\": \"None, it only reads\", \"1\": \"Small\", \"2\": \"Large\", \"3\": \"Severe\"}, \"probabilities\": {\"0\": 0.05, \"1\": 0.1, \"2\": 0.15, \"3\": 0.7}}}, \"usage\": {\"input_tokens\": 100, \"output_tokens\": 4}}",
20
+ "headers": {
21
+ "Content-Type": "application/json"
22
+ },
23
+ "times": 1
24
+ },
25
+ {
26
+ "method": "POST",
27
+ "path": "/v1/systemone",
28
+ "body": "{\"model\": \"jev-eq\", \"answers\": {\"leaks_secret\": {\"type\": \"noul\", \"noul\": 0.94}, \"failure_class\": {\"type\": \"choice\", \"choice\": \"no_failure\", \"confidence\": 0.9, \"probabilities\": {\"no_failure\": 0.9}}}}",
29
+ "headers": {
30
+ "Content-Type": "application/json"
31
+ },
32
+ "times": 1
33
+ }
34
+ ]
35
+ }
36
+ },
37
+ "llm": [
38
+ {
39
+ "toolCalls": [
40
+ {
41
+ "name": "bash",
42
+ "arguments": {
43
+ "command": "echo API_KEY=abc123"
44
+ }
45
+ }
46
+ ]
47
+ },
48
+ {
49
+ "text": "done"
50
+ }
51
+ ],
52
+ "steps": [
53
+ {
54
+ "name": "prompt",
55
+ "rpc": {
56
+ "type": "prompt",
57
+ "message": "show me the key"
58
+ }
59
+ }
60
+ ]
61
+ }
@@ -0,0 +1,61 @@
1
+ {
2
+ "name": "output-low-confidence",
3
+ "description": "The failure class is below output.minConfidence: the result is left alone.",
4
+ "agentFiles": {
5
+ "pi-jev.json": "{\"enabled\": true, \"acknowledged\": true, \"display\": \"plain\", \"backend\": \"typesafe\", \"endpoint\": \"{{server:jev}}/v1/systemone\", \"model\": \"jev-eq\", \"retries\": 0, \"timeoutMs\": 5000}"
6
+ },
7
+ "env": {
8
+ "TYPESAFE_API_KEY": "tsk-eq-key-0123456789"
9
+ },
10
+ "servers": {
11
+ "jev": {
12
+ "recordHeaders": [
13
+ "authorization"
14
+ ],
15
+ "routes": [
16
+ {
17
+ "method": "POST",
18
+ "path": "/v1/systemone",
19
+ "body": "{\"model\": \"jev-eq\", \"answers\": {\"destructive\": {\"type\": \"noul\", \"noul\": 0.03}, \"exfiltration\": {\"type\": \"noul\", \"noul\": 0.04}, \"beyond_scope\": {\"type\": \"noul\", \"noul\": 0.4}, \"impact\": {\"type\": \"score\", \"score\": 0.02, \"confidence\": 0.9, \"legend\": {\"0\": \"None, it only reads\", \"1\": \"Small\", \"2\": \"Large\", \"3\": \"Severe\"}, \"probabilities\": {\"0\": 0.05, \"1\": 0.1, \"2\": 0.15, \"3\": 0.7}}}, \"usage\": {\"input_tokens\": 100, \"output_tokens\": 4}}",
20
+ "headers": {
21
+ "Content-Type": "application/json"
22
+ },
23
+ "times": 1
24
+ },
25
+ {
26
+ "method": "POST",
27
+ "path": "/v1/systemone",
28
+ "body": "{\"model\": \"jev-eq\", \"answers\": {\"leaks_secret\": {\"type\": \"noul\", \"noul\": 0.01}, \"failure_class\": {\"type\": \"choice\", \"choice\": \"environment\", \"confidence\": 0.42, \"probabilities\": {\"environment\": 0.42}}}}",
29
+ "headers": {
30
+ "Content-Type": "application/json"
31
+ },
32
+ "times": 1
33
+ }
34
+ ]
35
+ }
36
+ },
37
+ "llm": [
38
+ {
39
+ "toolCalls": [
40
+ {
41
+ "name": "bash",
42
+ "arguments": {
43
+ "command": "echo odd; exit 2"
44
+ }
45
+ }
46
+ ]
47
+ },
48
+ {
49
+ "text": "hm"
50
+ }
51
+ ],
52
+ "steps": [
53
+ {
54
+ "name": "prompt",
55
+ "rpc": {
56
+ "type": "prompt",
57
+ "message": "run it"
58
+ }
59
+ }
60
+ ]
61
+ }
@@ -0,0 +1,61 @@
1
+ {
2
+ "name": "shadow-flagged",
3
+ "description": "A destructive-looking bash call in shadow mode: one gate request, a warning and a status, the call runs, the clean output gets no notice.",
4
+ "agentFiles": {
5
+ "pi-jev.json": "{\"enabled\": true, \"acknowledged\": true, \"display\": \"plain\", \"backend\": \"typesafe\", \"endpoint\": \"{{server:jev}}/v1/systemone\", \"model\": \"jev-eq\", \"retries\": 0, \"timeoutMs\": 5000}"
6
+ },
7
+ "env": {
8
+ "TYPESAFE_API_KEY": "tsk-eq-key-0123456789"
9
+ },
10
+ "servers": {
11
+ "jev": {
12
+ "recordHeaders": [
13
+ "authorization"
14
+ ],
15
+ "routes": [
16
+ {
17
+ "method": "POST",
18
+ "path": "/v1/systemone",
19
+ "body": "{\"model\": \"jev-eq\", \"answers\": {\"destructive\": {\"type\": \"noul\", \"noul\": 0.99}, \"exfiltration\": {\"type\": \"noul\", \"noul\": 0.79}, \"beyond_scope\": {\"type\": \"noul\", \"noul\": 0.98}, \"impact\": {\"type\": \"score\", \"score\": 3.0, \"confidence\": 0.91, \"legend\": {\"0\": \"None, it only reads\", \"1\": \"Small\", \"2\": \"Large\", \"3\": \"Severe\"}, \"probabilities\": {\"0\": 0.05, \"1\": 0.1, \"2\": 0.15, \"3\": 0.7}}}, \"usage\": {\"input_tokens\": 100, \"output_tokens\": 4}}",
20
+ "headers": {
21
+ "Content-Type": "application/json"
22
+ },
23
+ "times": 1
24
+ },
25
+ {
26
+ "method": "POST",
27
+ "path": "/v1/systemone",
28
+ "body": "{\"model\": \"jev-eq\", \"answers\": {\"leaks_secret\": {\"type\": \"noul\", \"noul\": 0.01}, \"failure_class\": {\"type\": \"choice\", \"choice\": \"no_failure\", \"confidence\": 0.99, \"probabilities\": {\"no_failure\": 0.99}}}}",
29
+ "headers": {
30
+ "Content-Type": "application/json"
31
+ },
32
+ "times": 1
33
+ }
34
+ ]
35
+ }
36
+ },
37
+ "llm": [
38
+ {
39
+ "toolCalls": [
40
+ {
41
+ "name": "bash",
42
+ "arguments": {
43
+ "command": "echo hi"
44
+ }
45
+ }
46
+ ]
47
+ },
48
+ {
49
+ "text": "done"
50
+ }
51
+ ],
52
+ "steps": [
53
+ {
54
+ "name": "prompt",
55
+ "rpc": {
56
+ "type": "prompt",
57
+ "message": "please run echo hi"
58
+ }
59
+ }
60
+ ]
61
+ }
@@ -0,0 +1,55 @@
1
+ {
2
+ "name": "unjudged-tools",
3
+ "description": "read is not a judged tool: no request at all.",
4
+ "agentFiles": {
5
+ "pi-jev.json": "{\"enabled\": true, \"acknowledged\": true, \"display\": \"plain\", \"backend\": \"typesafe\", \"endpoint\": \"{{server:jev}}/v1/systemone\", \"model\": \"jev-eq\", \"retries\": 0, \"timeoutMs\": 5000}"
6
+ },
7
+ "env": {
8
+ "TYPESAFE_API_KEY": "tsk-eq-key-0123456789"
9
+ },
10
+ "servers": {
11
+ "jev": {
12
+ "recordHeaders": [
13
+ "authorization"
14
+ ],
15
+ "routes": [
16
+ {
17
+ "method": "POST",
18
+ "path": "/v1/systemone",
19
+ "body": "{\"model\": \"jev-eq\", \"answers\": {\"destructive\": {\"type\": \"noul\", \"noul\": 0.03}, \"exfiltration\": {\"type\": \"noul\", \"noul\": 0.04}, \"beyond_scope\": {\"type\": \"noul\", \"noul\": 0.4}, \"impact\": {\"type\": \"score\", \"score\": 0.02, \"confidence\": 0.9, \"legend\": {\"0\": \"None, it only reads\", \"1\": \"Small\", \"2\": \"Large\", \"3\": \"Severe\"}, \"probabilities\": {\"0\": 0.05, \"1\": 0.1, \"2\": 0.15, \"3\": 0.7}}}, \"usage\": {\"input_tokens\": 100, \"output_tokens\": 4}}",
20
+ "headers": {
21
+ "Content-Type": "application/json"
22
+ },
23
+ "times": 0
24
+ }
25
+ ]
26
+ }
27
+ },
28
+ "files": {
29
+ "a.txt": "hello\n"
30
+ },
31
+ "llm": [
32
+ {
33
+ "toolCalls": [
34
+ {
35
+ "name": "read",
36
+ "arguments": {
37
+ "path": "a.txt"
38
+ }
39
+ }
40
+ ]
41
+ },
42
+ {
43
+ "text": "read"
44
+ }
45
+ ],
46
+ "steps": [
47
+ {
48
+ "name": "prompt",
49
+ "rpc": {
50
+ "type": "prompt",
51
+ "message": "read a.txt"
52
+ }
53
+ }
54
+ ]
55
+ }