@agent-corpus/policy-manager 0.3.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +56 -5
- package/dist/adapters.d.ts +11 -1
- package/dist/adapters.js +69 -4
- package/dist/cli.js +19 -2
- package/dist/config.js +4 -2
- package/dist/flow.d.ts +9 -0
- package/dist/flow.js +698 -0
- package/dist/permissive.d.ts +15 -6
- package/dist/permissive.js +225 -79
- package/dist/policy.d.ts +2 -0
- package/dist/policy.js +60 -1
- package/dist/redact.d.ts +3 -0
- package/dist/redact.js +140 -0
- package/dist/replay.d.ts +2 -0
- package/dist/replay.js +9 -1
- package/dist/review.d.ts +55 -0
- package/dist/review.js +231 -0
- package/dist/script.d.ts +101 -0
- package/dist/script.js +912 -0
- package/dist/service.d.ts +1 -0
- package/dist/service.js +1 -1
- package/dist/session.d.ts +13 -1
- package/dist/session.js +114 -7
- package/dist/setup.js +3 -1
- package/dist/shell.d.ts +6 -3
- package/dist/shell.js +75 -353
- package/dist/types.d.ts +14 -2
- package/package.json +1 -1
package/dist/replay.js
CHANGED
|
@@ -27,7 +27,13 @@ function transcriptFiles(paths, limit) {
|
|
|
27
27
|
export function replayTranscripts(paths, profile, files = 50) {
|
|
28
28
|
const seen = new Set();
|
|
29
29
|
const reasons = new Map();
|
|
30
|
-
const report = {
|
|
30
|
+
const report = {
|
|
31
|
+
commands: 0,
|
|
32
|
+
approved: 0,
|
|
33
|
+
strictApproved: 0,
|
|
34
|
+
classified: 0,
|
|
35
|
+
reasons: [],
|
|
36
|
+
};
|
|
31
37
|
for (const file of transcriptFiles(paths, files))
|
|
32
38
|
for (const line of readFileSync(file, "utf8").split("\n")) {
|
|
33
39
|
if (!line.includes('"tool_use"'))
|
|
@@ -66,6 +72,8 @@ export function replayTranscripts(paths, profile, files = 50) {
|
|
|
66
72
|
report.approved++;
|
|
67
73
|
continue;
|
|
68
74
|
}
|
|
75
|
+
if (decision.tier === "ambiguous")
|
|
76
|
+
report.classified++;
|
|
69
77
|
const bucket = reasons.get(decision.reason) ?? { count: 0, examples: [] };
|
|
70
78
|
bucket.count++;
|
|
71
79
|
if (bucket.examples.length < 3)
|
package/dist/review.d.ts
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
/** One classification the AI gave for a request. A request can have several (a replay). */
|
|
2
|
+
export interface Attempt {
|
|
3
|
+
time: string;
|
|
4
|
+
category?: string;
|
|
5
|
+
confidence?: number;
|
|
6
|
+
recoverable?: boolean;
|
|
7
|
+
rationale?: string;
|
|
8
|
+
/** `invalid` when the answer was rejected before it could decide anything. */
|
|
9
|
+
outcome: "approved" | "denied" | "invalid";
|
|
10
|
+
}
|
|
11
|
+
/** One ambiguous request, followed through classification to its outcome. */
|
|
12
|
+
export interface ClassifiedRequest {
|
|
13
|
+
session: string;
|
|
14
|
+
requestHash: string;
|
|
15
|
+
time: string;
|
|
16
|
+
command?: string;
|
|
17
|
+
reason?: string;
|
|
18
|
+
/** Every classification, in order. Later ones never overwrite earlier ones. */
|
|
19
|
+
attempts: Attempt[];
|
|
20
|
+
/** What happened to the request: `approved`/`denied` by the AI, `human` after a fallback. */
|
|
21
|
+
outcome: "approved" | "denied" | "fallback" | "unresolved";
|
|
22
|
+
fallbackReason?: string;
|
|
23
|
+
execution?: {
|
|
24
|
+
status?: string;
|
|
25
|
+
exitCode?: number;
|
|
26
|
+
};
|
|
27
|
+
humanResponse?: string;
|
|
28
|
+
}
|
|
29
|
+
export interface ClassificationReview {
|
|
30
|
+
sessions: number;
|
|
31
|
+
requested: number;
|
|
32
|
+
approved: number;
|
|
33
|
+
denied: number;
|
|
34
|
+
fallback: number;
|
|
35
|
+
fallbackReasons: Record<string, number>;
|
|
36
|
+
unresolved: number;
|
|
37
|
+
/** Share of requested ambiguous requests that reached a person. */
|
|
38
|
+
fallbackRate: number;
|
|
39
|
+
histogram: Array<{
|
|
40
|
+
range: string;
|
|
41
|
+
count: number;
|
|
42
|
+
}>;
|
|
43
|
+
aiApproved: ClassifiedRequest[];
|
|
44
|
+
aiDenied: ClassifiedRequest[];
|
|
45
|
+
approvedButFailed: ClassifiedRequest[];
|
|
46
|
+
humanDenied: ClassifiedRequest[];
|
|
47
|
+
/** Requests classified more than once. A replay is the signal a risk review looks for first. */
|
|
48
|
+
reclassified: ClassifiedRequest[];
|
|
49
|
+
/** Requests the AI denied and later approved. */
|
|
50
|
+
deniedThenApproved: ClassifiedRequest[];
|
|
51
|
+
}
|
|
52
|
+
export declare function sessionDirectories(extra?: string[]): string[];
|
|
53
|
+
export declare function reviewClassifications(directories: string[], since?: string): Promise<ClassificationReview>;
|
|
54
|
+
export declare function formatReview(review: ClassificationReview): string;
|
|
55
|
+
//# sourceMappingURL=review.d.ts.map
|
package/dist/review.js
ADDED
|
@@ -0,0 +1,231 @@
|
|
|
1
|
+
import { existsSync, lstatSync, readdirSync } from "node:fs";
|
|
2
|
+
import { tmpdir } from "node:os";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
import { AuditLog } from "@agent-corpus/audit-log";
|
|
5
|
+
const BUCKETS = [
|
|
6
|
+
["0.00-0.50", 0, 0.5],
|
|
7
|
+
["0.50-0.80", 0.5, 0.8],
|
|
8
|
+
["0.80-0.90", 0.8, 0.9],
|
|
9
|
+
["0.90-0.95", 0.9, 0.95],
|
|
10
|
+
["0.95-0.99", 0.95, 0.99],
|
|
11
|
+
["0.99-1.00", 0.99, 1.0001],
|
|
12
|
+
];
|
|
13
|
+
export function sessionDirectories(extra = []) {
|
|
14
|
+
const root = tmpdir();
|
|
15
|
+
const found = existsSync(root)
|
|
16
|
+
? readdirSync(root)
|
|
17
|
+
.filter((name) => /^acp-[A-Za-z0-9]+-[a-f0-9]{24}$/.test(name))
|
|
18
|
+
.map((name) => join(root, name))
|
|
19
|
+
: [];
|
|
20
|
+
return [...found, ...extra].filter((path) => {
|
|
21
|
+
try {
|
|
22
|
+
return lstatSync(path).isDirectory();
|
|
23
|
+
}
|
|
24
|
+
catch {
|
|
25
|
+
return false;
|
|
26
|
+
}
|
|
27
|
+
});
|
|
28
|
+
}
|
|
29
|
+
const text = (value) => typeof value === "string" ? value : undefined;
|
|
30
|
+
const num = (value) => typeof value === "number" && Number.isFinite(value) ? value : undefined;
|
|
31
|
+
export async function reviewClassifications(directories, since) {
|
|
32
|
+
const requests = [];
|
|
33
|
+
let sessions = 0;
|
|
34
|
+
for (const directory of directories) {
|
|
35
|
+
const records = [];
|
|
36
|
+
try {
|
|
37
|
+
let after;
|
|
38
|
+
for (;;) {
|
|
39
|
+
const page = await new AuditLog(directory).query({
|
|
40
|
+
...(since ? { since } : {}),
|
|
41
|
+
...(after ? { after } : {}),
|
|
42
|
+
limit: 10000,
|
|
43
|
+
});
|
|
44
|
+
records.push(...page.records);
|
|
45
|
+
if (!page.nextCursor)
|
|
46
|
+
break;
|
|
47
|
+
after = page.nextCursor;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
catch {
|
|
51
|
+
continue; // an unreadable or foreign directory is skipped, never trusted
|
|
52
|
+
}
|
|
53
|
+
if (!records.length)
|
|
54
|
+
continue;
|
|
55
|
+
sessions++;
|
|
56
|
+
const byRequest = new Map();
|
|
57
|
+
const entry = (record) => {
|
|
58
|
+
if (!record.requestId)
|
|
59
|
+
return undefined;
|
|
60
|
+
let value = byRequest.get(record.requestId);
|
|
61
|
+
if (!value) {
|
|
62
|
+
value = {
|
|
63
|
+
session: record.sessionId,
|
|
64
|
+
requestHash: record.requestId,
|
|
65
|
+
time: record.timestamp,
|
|
66
|
+
attempts: [],
|
|
67
|
+
outcome: "unresolved",
|
|
68
|
+
};
|
|
69
|
+
byRequest.set(record.requestId, value);
|
|
70
|
+
}
|
|
71
|
+
return value;
|
|
72
|
+
};
|
|
73
|
+
for (const record of records) {
|
|
74
|
+
const a = record.attributes;
|
|
75
|
+
// Only requests that entered the classification flow are reviewed.
|
|
76
|
+
if (record.eventName !== "classification-requested" && !byRequest.has(record.requestId ?? ""))
|
|
77
|
+
continue;
|
|
78
|
+
if (record.eventName === "classification-requested") {
|
|
79
|
+
const value = entry(record);
|
|
80
|
+
if (!value)
|
|
81
|
+
continue;
|
|
82
|
+
value.time = record.timestamp;
|
|
83
|
+
const command = text(a.redactedCommand);
|
|
84
|
+
if (command)
|
|
85
|
+
value.command = command;
|
|
86
|
+
const reason = text(a.reason);
|
|
87
|
+
if (reason)
|
|
88
|
+
value.reason = reason;
|
|
89
|
+
if (!requests.includes(value))
|
|
90
|
+
requests.push(value);
|
|
91
|
+
}
|
|
92
|
+
else if (record.eventName === "classification") {
|
|
93
|
+
const value = entry(record);
|
|
94
|
+
if (!value)
|
|
95
|
+
continue;
|
|
96
|
+
const attempt = { time: record.timestamp, outcome: "invalid" };
|
|
97
|
+
const category = text(a.category);
|
|
98
|
+
if (category)
|
|
99
|
+
attempt.category = category;
|
|
100
|
+
const confidence = num(a.confidence);
|
|
101
|
+
if (confidence !== undefined)
|
|
102
|
+
attempt.confidence = confidence;
|
|
103
|
+
if (typeof a.recoverable === "boolean")
|
|
104
|
+
attempt.recoverable = a.recoverable;
|
|
105
|
+
const rationale = text(a.rationale);
|
|
106
|
+
if (rationale)
|
|
107
|
+
attempt.rationale = rationale;
|
|
108
|
+
value.attempts.push(attempt);
|
|
109
|
+
}
|
|
110
|
+
else if (record.eventName === "decision") {
|
|
111
|
+
const value = entry(record);
|
|
112
|
+
if (!value)
|
|
113
|
+
continue;
|
|
114
|
+
const latest = value.attempts.at(-1);
|
|
115
|
+
if (a.classification === "approved" || a.classification === "denied") {
|
|
116
|
+
// The decision that follows a classification settles that attempt; a retry's decision
|
|
117
|
+
// repeats the same outcome and changes nothing.
|
|
118
|
+
if (latest?.outcome === "invalid")
|
|
119
|
+
latest.outcome = a.classification;
|
|
120
|
+
value.outcome = a.classification;
|
|
121
|
+
}
|
|
122
|
+
else if (a.classification === "fallback")
|
|
123
|
+
value.outcome = "fallback";
|
|
124
|
+
}
|
|
125
|
+
else if (record.eventName === "classification-fallback") {
|
|
126
|
+
const value = entry(record);
|
|
127
|
+
if (!value)
|
|
128
|
+
continue;
|
|
129
|
+
value.outcome = "fallback";
|
|
130
|
+
const reason = text(a.fallbackReason);
|
|
131
|
+
if (reason)
|
|
132
|
+
value.fallbackReason = reason;
|
|
133
|
+
}
|
|
134
|
+
else if (record.eventName === "execution-result") {
|
|
135
|
+
const value = entry(record);
|
|
136
|
+
if (!value)
|
|
137
|
+
continue;
|
|
138
|
+
value.execution = {
|
|
139
|
+
...(text(a.status) ? { status: text(a.status) } : {}),
|
|
140
|
+
...(num(a.exitCode) !== undefined ? { exitCode: num(a.exitCode) } : {}),
|
|
141
|
+
};
|
|
142
|
+
}
|
|
143
|
+
else if (record.eventName === "human-response") {
|
|
144
|
+
const value = entry(record);
|
|
145
|
+
if (value)
|
|
146
|
+
value.humanResponse = text(a.response) ?? "";
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
const approved = requests.filter((r) => r.outcome === "approved");
|
|
151
|
+
const denied = requests.filter((r) => r.outcome === "denied");
|
|
152
|
+
const fallback = requests.filter((r) => r.outcome === "fallback");
|
|
153
|
+
const fallbackReasons = {};
|
|
154
|
+
for (const r of fallback) {
|
|
155
|
+
const key = r.fallbackReason ?? "unavailable";
|
|
156
|
+
fallbackReasons[key] = (fallbackReasons[key] ?? 0) + 1;
|
|
157
|
+
}
|
|
158
|
+
return {
|
|
159
|
+
sessions,
|
|
160
|
+
requested: requests.length,
|
|
161
|
+
approved: approved.length,
|
|
162
|
+
denied: denied.length,
|
|
163
|
+
fallback: fallback.length,
|
|
164
|
+
fallbackReasons,
|
|
165
|
+
unresolved: requests.length - approved.length - denied.length - fallback.length,
|
|
166
|
+
fallbackRate: requests.length ? fallback.length / requests.length : 0,
|
|
167
|
+
histogram: BUCKETS.map(([range, low, high]) => ({
|
|
168
|
+
range,
|
|
169
|
+
count: requests
|
|
170
|
+
.flatMap((r) => r.attempts)
|
|
171
|
+
.filter((t) => t.confidence !== undefined && t.confidence >= low && t.confidence < high)
|
|
172
|
+
.length,
|
|
173
|
+
})),
|
|
174
|
+
aiApproved: approved,
|
|
175
|
+
aiDenied: denied,
|
|
176
|
+
approvedButFailed: approved.filter((r) => r.execution &&
|
|
177
|
+
(r.execution.status === "failed" ||
|
|
178
|
+
(r.execution.exitCode !== undefined && r.execution.exitCode !== 0))),
|
|
179
|
+
humanDenied: requests.filter((r) => r.humanResponse === "denied"),
|
|
180
|
+
reclassified: requests.filter((r) => r.attempts.length > 1),
|
|
181
|
+
deniedThenApproved: requests.filter((r) => {
|
|
182
|
+
const first = r.attempts.findIndex((t) => t.outcome === "denied");
|
|
183
|
+
return first >= 0 && r.attempts.slice(first + 1).some((t) => t.outcome === "approved");
|
|
184
|
+
}),
|
|
185
|
+
};
|
|
186
|
+
}
|
|
187
|
+
function row(r) {
|
|
188
|
+
const outcome = r.execution
|
|
189
|
+
? ` -> ${r.execution.status ?? "ran"}${r.execution.exitCode !== undefined ? ` (exit ${r.execution.exitCode})` : ""}`
|
|
190
|
+
: "";
|
|
191
|
+
const attempts = r.attempts
|
|
192
|
+
.map((t, i) => ` ${r.attempts.length > 1 ? `attempt ${i + 1}: ` : ""}${t.outcome} ${t.category ?? "?"} ${t.confidence ?? "?"} recoverable=${t.recoverable ?? "?"}${t.rationale ? ` why: ${t.rationale}` : ""}`)
|
|
193
|
+
.join("\n");
|
|
194
|
+
return ` ${r.time} ${r.command ?? r.requestHash}${outcome}${attempts ? `\n${attempts}` : ""}`;
|
|
195
|
+
}
|
|
196
|
+
export function formatReview(review) {
|
|
197
|
+
const pct = (n) => `${((100 * n) / Math.max(review.requested, 1)).toFixed(1)}%`;
|
|
198
|
+
const lines = [
|
|
199
|
+
`Sessions scanned: ${review.sessions}`,
|
|
200
|
+
`Ambiguous requests sent to classification: ${review.requested}`,
|
|
201
|
+
` AI approved: ${review.approved} (${pct(review.approved)})`,
|
|
202
|
+
` AI denied: ${review.denied} (${pct(review.denied)})`,
|
|
203
|
+
` Fell back to human: ${review.fallback} (${pct(review.fallback)})${Object.entries(review.fallbackReasons)
|
|
204
|
+
.map(([k, v]) => ` ${k}=${v}`)
|
|
205
|
+
.join("")}`,
|
|
206
|
+
` Never resolved: ${review.unresolved} (${pct(review.unresolved)}; the model abandoned the command)`,
|
|
207
|
+
"",
|
|
208
|
+
"Confidence histogram (classifier self-report):",
|
|
209
|
+
...review.histogram.map((b) => ` ${b.range} ${b.count}`),
|
|
210
|
+
"",
|
|
211
|
+
`Requests classified more than once (${review.reclassified.length}):`,
|
|
212
|
+
...review.reclassified.map(row),
|
|
213
|
+
"",
|
|
214
|
+
`AI-denied, then approved on a later classification (${review.deniedThenApproved.length}):`,
|
|
215
|
+
...review.deniedThenApproved.map(row),
|
|
216
|
+
"",
|
|
217
|
+
`AI-approved commands that failed (${review.approvedButFailed.length}):`,
|
|
218
|
+
...review.approvedButFailed.map(row),
|
|
219
|
+
"",
|
|
220
|
+
`Commands a person later denied (${review.humanDenied.length}):`,
|
|
221
|
+
...review.humanDenied.map(row),
|
|
222
|
+
"",
|
|
223
|
+
`All AI-approved commands (${review.aiApproved.length}):`,
|
|
224
|
+
...review.aiApproved.map(row),
|
|
225
|
+
"",
|
|
226
|
+
`AI-denied commands (${review.aiDenied.length}):`,
|
|
227
|
+
...review.aiDenied.map(row),
|
|
228
|
+
];
|
|
229
|
+
return lines.join("\n");
|
|
230
|
+
}
|
|
231
|
+
//# sourceMappingURL=review.js.map
|
package/dist/script.d.ts
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
/** Stands for the unknown output of an expansion inside a word. */
|
|
2
|
+
export declare const SUBSTITUTION = "\uE000";
|
|
3
|
+
export interface VarRef {
|
|
4
|
+
name: string;
|
|
5
|
+
quoted: boolean;
|
|
6
|
+
}
|
|
7
|
+
export interface UnknownPart {
|
|
8
|
+
unknown: true;
|
|
9
|
+
quoted: boolean;
|
|
10
|
+
/** Body of the command substitution it came from, when it did. */
|
|
11
|
+
inner?: string;
|
|
12
|
+
}
|
|
13
|
+
export type Part = string | VarRef | UnknownPart;
|
|
14
|
+
export interface Word {
|
|
15
|
+
parts: Part[];
|
|
16
|
+
/** Command substitution bodies (and process substitutions) inside this word, in order. */
|
|
17
|
+
subs: string[];
|
|
18
|
+
/** True when any part of the word was quoted or escaped. */
|
|
19
|
+
quoted: boolean;
|
|
20
|
+
/** Set when the word is `NAME=value`: the name, with `parts` holding only the value. */
|
|
21
|
+
assign?: string;
|
|
22
|
+
/** True when the word has an unquoted `*`, `?` or `[`: the shell may replace it by file names. */
|
|
23
|
+
glob?: boolean;
|
|
24
|
+
}
|
|
25
|
+
export interface Redir {
|
|
26
|
+
fd: string;
|
|
27
|
+
op: ">" | ">>" | "<" | ">&" | "<<" | "<<<";
|
|
28
|
+
target?: Word;
|
|
29
|
+
/** Literal body of a heredoc. */
|
|
30
|
+
body?: string;
|
|
31
|
+
}
|
|
32
|
+
export interface Simple {
|
|
33
|
+
kind: "simple";
|
|
34
|
+
assigns: Array<{
|
|
35
|
+
name: string;
|
|
36
|
+
value: Word;
|
|
37
|
+
}>;
|
|
38
|
+
argv: Word[];
|
|
39
|
+
redirects: Redir[];
|
|
40
|
+
}
|
|
41
|
+
export interface Subshell {
|
|
42
|
+
kind: "subshell";
|
|
43
|
+
body: ListNode;
|
|
44
|
+
redirects: Redir[];
|
|
45
|
+
}
|
|
46
|
+
export interface Group {
|
|
47
|
+
kind: "group";
|
|
48
|
+
body: ListNode;
|
|
49
|
+
redirects: Redir[];
|
|
50
|
+
}
|
|
51
|
+
export interface ForNode {
|
|
52
|
+
kind: "for";
|
|
53
|
+
name: string;
|
|
54
|
+
words: Word[];
|
|
55
|
+
body: ListNode;
|
|
56
|
+
redirects: Redir[];
|
|
57
|
+
}
|
|
58
|
+
export interface WhileNode {
|
|
59
|
+
kind: "while";
|
|
60
|
+
cond: ListNode;
|
|
61
|
+
body: ListNode;
|
|
62
|
+
redirects: Redir[];
|
|
63
|
+
}
|
|
64
|
+
export interface IfNode {
|
|
65
|
+
kind: "if";
|
|
66
|
+
clauses: Array<{
|
|
67
|
+
cond: ListNode;
|
|
68
|
+
body: ListNode;
|
|
69
|
+
}>;
|
|
70
|
+
otherwise?: ListNode;
|
|
71
|
+
redirects: Redir[];
|
|
72
|
+
}
|
|
73
|
+
export interface CaseNode {
|
|
74
|
+
kind: "case";
|
|
75
|
+
word: Word;
|
|
76
|
+
branches: Array<{
|
|
77
|
+
patterns: Word[];
|
|
78
|
+
body: ListNode;
|
|
79
|
+
}>;
|
|
80
|
+
redirects: Redir[];
|
|
81
|
+
}
|
|
82
|
+
export type Command = Simple | Subshell | Group | ForNode | WhileNode | IfNode | CaseNode;
|
|
83
|
+
export interface PipelineNode {
|
|
84
|
+
stages: Command[];
|
|
85
|
+
negated: boolean;
|
|
86
|
+
}
|
|
87
|
+
export interface AndOr {
|
|
88
|
+
first: PipelineNode;
|
|
89
|
+
rest: Array<{
|
|
90
|
+
op: "&&" | "||";
|
|
91
|
+
node: PipelineNode;
|
|
92
|
+
}>;
|
|
93
|
+
}
|
|
94
|
+
export interface ListNode {
|
|
95
|
+
items: Array<{
|
|
96
|
+
node: AndOr;
|
|
97
|
+
background: boolean;
|
|
98
|
+
}>;
|
|
99
|
+
}
|
|
100
|
+
export declare function parseScript(command: string): ListNode | undefined;
|
|
101
|
+
//# sourceMappingURL=script.d.ts.map
|