@preapexis/piexis 0.1.2 → 0.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/publish.yml +35 -0
- package/AGENTS.md +21 -41
- package/README.md +75 -142
- package/extensions/agents.ts +205 -0
- package/extensions/ask-question.ts +54 -95
- package/extensions/recommended-tools.ts +62 -0
- package/extensions/ui.ts +204 -58
- package/lib/agent-config.ts +55 -0
- package/lib/agents.ts +208 -0
- package/lib/question-model.ts +98 -0
- package/lib/question-ui.ts +214 -0
- package/lib/recommended-tools.ts +119 -0
- package/lib/subagent-runner.ts +146 -0
- package/lib/ui-tips.json +41 -0
- package/package.json +9 -12
- package/prompts/init.md +10 -2
- package/prompts/summary.md +7 -6
- package/schemas/agent.schema.json +48 -0
- package/scripts/release.mjs +32 -0
- package/skills/{claude-api/LICENSE.txt → caveman/LICENSE} +2 -2
- package/skills/caveman/LICENSE-MIT +21 -0
- package/skills/caveman/NOTICE +10 -0
- package/skills/caveman/SKILL.md +89 -0
- package/skills/find-skills/SKILL.md +141 -0
- package/skills/ponytail/LICENSE +21 -0
- package/skills/ponytail/NOTICE +9 -0
- package/skills/ponytail/SKILL.md +52 -0
- package/skills-lock.json +6 -0
- package/tests/agents.test.mjs +605 -0
- package/tests/ask-question-pty.test.mjs +16 -0
- package/tests/ask-question.test.mjs +269 -0
- package/tests/fixtures/agents-provider.ts +55 -0
- package/tests/fixtures/ask-question-pty.py +91 -0
- package/tests/fixtures/ask-question-terminal.mjs +69 -0
- package/tests/fixtures/recommended-tools-terminal.mjs +103 -0
- package/tests/init-workflow.test.mjs +31 -0
- package/tests/recommended-tools-pty.test.mjs +16 -0
- package/tests/recommended-tools.test.mjs +423 -0
- package/tests/resources.test.mjs +66 -0
- package/tests/skills-routing.test.mjs +7 -4
- package/tests/ui-tip-reload.test.mjs +70 -0
- package/tests/ui.test.mjs +450 -0
- package/tsconfig.json +1 -0
- package/.github/workflows/npm-publish.yml +0 -63
- package/PERMISSION_MODES_PLAN.md +0 -279
- package/WHAT_YOU_GET.md +0 -153
- package/eval/README.md +0 -72
- package/eval/permission-cases.json +0 -28
- package/eval/permissions.mjs +0 -103
- package/extensions/background-tasks.ts +0 -433
- package/extensions/background-worker.ts +0 -50
- package/extensions/glob.ts +0 -32
- package/extensions/mode.ts +0 -397
- package/lib/bash-approvals.ts +0 -113
- package/lib/bash-policy.ts +0 -164
- package/lib/mode-policy.ts +0 -174
- package/lib/path-policy.ts +0 -182
- package/lib/permission-classifier.ts +0 -128
- package/lib/permission-config.ts +0 -91
- package/lib/permission-denials.ts +0 -32
- package/lib/permission-intent.ts +0 -128
- package/lib/permission-taxonomy.ts +0 -15
- package/lib/task-broker.ts +0 -65
- package/lib/task-checkout.ts +0 -39
- package/lib/task-git.ts +0 -44
- package/lib/task-patch.ts +0 -106
- package/lib/task-permissions.ts +0 -51
- package/lib/task-report.ts +0 -62
- package/lib/task-worktree-policy.ts +0 -56
- package/skills/claude-api/SKILL.md +0 -572
- package/skills/claude-api/csharp/claude-api/README.md +0 -362
- package/skills/claude-api/csharp/claude-api/batches.md +0 -14
- package/skills/claude-api/csharp/claude-api/files-api.md +0 -25
- package/skills/claude-api/csharp/claude-api/streaming.md +0 -28
- package/skills/claude-api/csharp/claude-api/tool-use.md +0 -164
- package/skills/claude-api/curl/examples.md +0 -256
- package/skills/claude-api/curl/managed-agents.md +0 -371
- package/skills/claude-api/go/claude-api/README.md +0 -188
- package/skills/claude-api/go/claude-api/files-api.md +0 -23
- package/skills/claude-api/go/claude-api/streaming.md +0 -43
- package/skills/claude-api/go/claude-api/tool-use.md +0 -220
- package/skills/claude-api/go/managed-agents/README.md +0 -564
- package/skills/claude-api/java/claude-api/README.md +0 -238
- package/skills/claude-api/java/claude-api/files-api.md +0 -27
- package/skills/claude-api/java/claude-api/streaming.md +0 -24
- package/skills/claude-api/java/claude-api/tool-use.md +0 -228
- package/skills/claude-api/java/managed-agents/README.md +0 -443
- package/skills/claude-api/php/claude-api/README.md +0 -174
- package/skills/claude-api/php/claude-api/batches.md +0 -15
- package/skills/claude-api/php/claude-api/files-api.md +0 -13
- package/skills/claude-api/php/claude-api/streaming.md +0 -27
- package/skills/claude-api/php/claude-api/tool-use.md +0 -253
- package/skills/claude-api/php/managed-agents/README.md +0 -443
- package/skills/claude-api/python/claude-api/README.md +0 -565
- package/skills/claude-api/python/claude-api/batches.md +0 -198
- package/skills/claude-api/python/claude-api/files-api.md +0 -170
- package/skills/claude-api/python/claude-api/sdk-upgrade.md +0 -286
- package/skills/claude-api/python/claude-api/streaming.md +0 -179
- package/skills/claude-api/python/claude-api/tool-use.md +0 -627
- package/skills/claude-api/python/managed-agents/README.md +0 -335
- package/skills/claude-api/ruby/claude-api/README.md +0 -134
- package/skills/claude-api/ruby/claude-api/streaming.md +0 -16
- package/skills/claude-api/ruby/claude-api/tool-use.md +0 -41
- package/skills/claude-api/ruby/managed-agents/README.md +0 -394
- package/skills/claude-api/shared/admin-api.md +0 -179
- package/skills/claude-api/shared/agent-design.md +0 -101
- package/skills/claude-api/shared/anthropic-cli.md +0 -269
- package/skills/claude-api/shared/claude-platform-on-aws.md +0 -59
- package/skills/claude-api/shared/cost-optimization.md +0 -233
- package/skills/claude-api/shared/error-codes.md +0 -262
- package/skills/claude-api/shared/live-sources.md +0 -166
- package/skills/claude-api/shared/managed-agents-api-reference.md +0 -455
- package/skills/claude-api/shared/managed-agents-client-patterns.md +0 -220
- package/skills/claude-api/shared/managed-agents-core.md +0 -359
- package/skills/claude-api/shared/managed-agents-environments.md +0 -226
- package/skills/claude-api/shared/managed-agents-events.md +0 -290
- package/skills/claude-api/shared/managed-agents-memory.md +0 -201
- package/skills/claude-api/shared/managed-agents-multiagent.md +0 -260
- package/skills/claude-api/shared/managed-agents-onboarding.md +0 -82
- package/skills/claude-api/shared/managed-agents-outcomes.md +0 -108
- package/skills/claude-api/shared/managed-agents-overview.md +0 -77
- package/skills/claude-api/shared/managed-agents-scheduled-deployments.md +0 -156
- package/skills/claude-api/shared/managed-agents-self-hosted-sandboxes.md +0 -282
- package/skills/claude-api/shared/managed-agents-tools.md +0 -504
- package/skills/claude-api/shared/managed-agents-webhooks.md +0 -143
- package/skills/claude-api/shared/model-migration.md +0 -1872
- package/skills/claude-api/shared/models.md +0 -142
- package/skills/claude-api/shared/platform-availability.md +0 -53
- package/skills/claude-api/shared/prompt-audit.md +0 -223
- package/skills/claude-api/shared/prompt-caching.md +0 -292
- package/skills/claude-api/shared/token-counting.md +0 -56
- package/skills/claude-api/shared/tool-use-concepts.md +0 -526
- package/skills/claude-api/typescript/claude-api/README.md +0 -405
- package/skills/claude-api/typescript/claude-api/batches.md +0 -106
- package/skills/claude-api/typescript/claude-api/files-api.md +0 -98
- package/skills/claude-api/typescript/claude-api/streaming.md +0 -178
- package/skills/claude-api/typescript/claude-api/tool-use.md +0 -586
- package/skills/claude-api/typescript/managed-agents/README.md +0 -359
- package/tests/auto-permissions.test.mjs +0 -315
- package/tests/background-tasks.test.mjs +0 -27
- package/tests/background-worker.test.mjs +0 -60
- package/tests/bash-permissions.test.mjs +0 -276
- package/tests/glob.test.mjs +0 -31
- package/tests/mode.test.mjs +0 -62
- package/tests/permission-evaluation.test.mjs +0 -70
- package/tests/permission-foundation.test.mjs +0 -194
- package/tests/permission-policy.test.mjs +0 -158
- package/tests/runtime/README.md +0 -51
- package/tests/runtime/harness.mjs +0 -190
- package/tests/runtime/permissions.test.mjs +0 -192
- package/tests/runtime/tui.test.mjs +0 -60
- package/tests/task-integration.test.mjs +0 -324
- package/tests/task-permissions.test.mjs +0 -178
- package/tests/task-worktree-policy.test.mjs +0 -53
|
@@ -1,27 +0,0 @@
|
|
|
1
|
-
import assert from "node:assert/strict";
|
|
2
|
-
import test from "node:test";
|
|
3
|
-
|
|
4
|
-
import backgroundTasks from "../extensions/background-tasks.ts";
|
|
5
|
-
import backgroundWorker from "../extensions/background-worker.ts";
|
|
6
|
-
|
|
7
|
-
test("background tasks expose launch, status, cancellation, and explicit apply tools", () => {
|
|
8
|
-
const tools = new Map();
|
|
9
|
-
const commands = new Map();
|
|
10
|
-
backgroundTasks({
|
|
11
|
-
registerFlag() {},
|
|
12
|
-
registerTool(tool) { tools.set(tool.name, tool); },
|
|
13
|
-
registerCommand(name, command) { commands.set(name, command); },
|
|
14
|
-
on() {},
|
|
15
|
-
events: { on() {} }
|
|
16
|
-
});
|
|
17
|
-
|
|
18
|
-
assert.deepEqual([...tools.keys()].sort(), ["background_task", "task_status", "task_cancel", "task_diff", "task_clean", "task_apply"].sort());
|
|
19
|
-
assert.ok(commands.has("task"));
|
|
20
|
-
assert.equal(tools.get("background_task").executionMode, "sequential");
|
|
21
|
-
});
|
|
22
|
-
|
|
23
|
-
test("background worker does not install a foreground permission guard without a broker", () => {
|
|
24
|
-
let handler;
|
|
25
|
-
backgroundWorker({ on(name, callback) { if (name === "tool_call") handler = callback; } });
|
|
26
|
-
assert.equal(handler, undefined);
|
|
27
|
-
});
|
|
@@ -1,60 +0,0 @@
|
|
|
1
|
-
import assert from "node:assert/strict";
|
|
2
|
-
import test from "node:test";
|
|
3
|
-
import { mkdtemp, writeFile, readdir, rm } from "node:fs/promises";
|
|
4
|
-
import { tmpdir } from "node:os";
|
|
5
|
-
import { join } from "node:path";
|
|
6
|
-
import { randomUUID } from "node:crypto";
|
|
7
|
-
import workerExtension from "../extensions/background-worker.ts";
|
|
8
|
-
import { readBrokerJson, publishBrokerJson } from "../lib/task-broker.ts";
|
|
9
|
-
|
|
10
|
-
async function worker(t, bash = false) {
|
|
11
|
-
const broker = await mkdtemp(join(tmpdir(), "piexis-worker-guard-"));
|
|
12
|
-
const settings = { PIEXIS_TASK_BROKER: broker, PIEXIS_TASK_TOKEN: randomUUID(), PIEXIS_TASK_BASH: bash ? "1" : "0", PIEXIS_TASK_REPEAT_LIMIT: "3", PIEXIS_TASK_DENIAL_LIMIT: "6" };
|
|
13
|
-
const saved = Object.fromEntries(Object.keys(settings).map((key) => [key, process.env[key]]));
|
|
14
|
-
Object.assign(process.env, settings);
|
|
15
|
-
const hooks = new Map();
|
|
16
|
-
const tools = new Map();
|
|
17
|
-
const records = [];
|
|
18
|
-
let aborts = 0;
|
|
19
|
-
const controller = new AbortController();
|
|
20
|
-
const ctx = { signal: controller.signal, abort() { aborts++; } };
|
|
21
|
-
workerExtension({ on(name, callback) { hooks.set(name, callback); }, registerTool(tool) { tools.set(tool.name, tool); }, appendEntry(customType, data) { records.push({ customType, data }); } });
|
|
22
|
-
for (const [key, value] of Object.entries(saved)) { if (value === undefined) delete process.env[key]; else process.env[key] = value; }
|
|
23
|
-
t.after(async () => { controller.abort(); await rm(broker, { recursive: true, force: true }); });
|
|
24
|
-
await writeFile(join(broker, "heartbeat"), "");
|
|
25
|
-
return { broker, hooks, tools, records, ctx, settings, get aborts() { return aborts; } };
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
test("worker-only wrappers cannot fall back to built-ins and even reads need the parent broker", { timeout: 5000 }, async (t) => {
|
|
29
|
-
const h = await worker(t);
|
|
30
|
-
assert.deepEqual([...h.tools.keys()], ["worker_read", "worker_edit", "worker_write", "worker_grep", "worker_find", "worker_ls"]);
|
|
31
|
-
const call = (toolName, input) => h.hooks.get("tool_call")({ toolName, input, toolCallId: "fixture" }, h.ctx);
|
|
32
|
-
assert.equal((await call("read", { path: "README.md" })).block, true);
|
|
33
|
-
assert.equal((await call("worker_bash", { command: "pwd" })).block, true);
|
|
34
|
-
const pending = call("worker_read", { path: "src/main.txt" });
|
|
35
|
-
let file;
|
|
36
|
-
while (!(file = (await readdir(h.broker)).find((name) => name.endsWith(".request.json")))) await new Promise(setImmediate);
|
|
37
|
-
const request = await readBrokerJson(join(h.broker, file));
|
|
38
|
-
assert.equal(request.tool, "read");
|
|
39
|
-
assert.equal(request.token, h.settings.PIEXIS_TASK_TOKEN);
|
|
40
|
-
await publishBrokerJson(join(h.broker, `${request.id}.response.json`), { id: request.id, outcome: "allowed" });
|
|
41
|
-
assert.equal(await pending, undefined);
|
|
42
|
-
assert.equal(h.aborts, 0);
|
|
43
|
-
await rm(join(h.broker, "heartbeat"));
|
|
44
|
-
assert.equal((await call("worker_read", { path: "src/main.txt" })).block, true);
|
|
45
|
-
assert.equal(h.aborts, 1);
|
|
46
|
-
});
|
|
47
|
-
|
|
48
|
-
test("worker command observations use post-execution callbacks and redact recognizable secrets", async (t) => {
|
|
49
|
-
const h = await worker(t, true);
|
|
50
|
-
assert.ok(h.tools.has("worker_bash"));
|
|
51
|
-
assert.equal(h.hooks.has("tool_execution_start"), false);
|
|
52
|
-
h.hooks.get("tool_result")({ toolName: "worker_bash", toolCallId: "actual", input: { command: "node --test fixture.mjs" }, isError: false });
|
|
53
|
-
h.hooks.get("tool_result")({ toolName: "worker_read", toolCallId: "read", input: {}, isError: false });
|
|
54
|
-
assert.deepEqual(h.records, [{ customType: "piexis-worker-execution", data: { id: "actual", command: "node --test fixture.mjs", isError: false } }]);
|
|
55
|
-
h.hooks.get("tool_result")({ toolName: "worker_bash", toolCallId: "redacted", input: { command: 'printf "Bearer fixture_test_token_123"' }, isError: true });
|
|
56
|
-
assert.doesNotMatch(h.records[1].data.command, /fixture_test_token_123/);
|
|
57
|
-
assert.match(h.records[1].data.command, /REDACTED/);
|
|
58
|
-
h.hooks.get("tool_result")({ toolName: "worker_bash", toolCallId: "long", input: { command: "x".repeat(3000) }, isError: true });
|
|
59
|
-
assert.equal(h.records[2].data.truncated, true);
|
|
60
|
-
});
|
|
@@ -1,276 +0,0 @@
|
|
|
1
|
-
import assert from "node:assert/strict";
|
|
2
|
-
import { mkdir, mkdtemp, rm, symlink, writeFile } from "node:fs/promises";
|
|
3
|
-
import { tmpdir } from "node:os";
|
|
4
|
-
import { join } from "node:path";
|
|
5
|
-
import test from "node:test";
|
|
6
|
-
import { analyzeBash, parseBash } from "../lib/bash-policy.ts";
|
|
7
|
-
import { bashApprovalFingerprint } from "../lib/bash-approvals.ts";
|
|
8
|
-
import { evaluatePermission } from "../lib/mode-policy.ts";
|
|
9
|
-
import { DenialTracker, denialLimit } from "../lib/permission-denials.ts";
|
|
10
|
-
import modeExtension from "../extensions/mode.ts";
|
|
11
|
-
|
|
12
|
-
async function fixture(t) {
|
|
13
|
-
const root = await mkdtemp(join(tmpdir(), "piexis-bash-"));
|
|
14
|
-
t.after(() => rm(root, { recursive: true, force: true }));
|
|
15
|
-
for (const dir of [".git", ".pi", "bin", "nested", "home"]) await mkdir(join(root, dir));
|
|
16
|
-
await writeFile(join(root, ".git", "config"), "[core]\n bare = false\n");
|
|
17
|
-
await writeFile(join(root, "global.gitconfig"), "[user]\n name = Fixture\n");
|
|
18
|
-
// Synthetic executable header for fingerprint tests. Never execute this file.
|
|
19
|
-
await writeFile(join(root, "bin", "git"), Buffer.from("7f454c4666697874757265", "hex"), { mode: 0o755 });
|
|
20
|
-
const env = { PATH: join(root, "bin"), HOME: join(root, "home"), GIT_CONFIG_GLOBAL: join(root, "global.gitconfig"), GIT_CONFIG_NOSYSTEM: "1" };
|
|
21
|
-
return { root, env };
|
|
22
|
-
}
|
|
23
|
-
|
|
24
|
-
function isolatedEnvironment(t, env) {
|
|
25
|
-
const keys = new Set([...Object.keys(process.env).filter((key) => /^(BASH|GIT_|ENV$|SHELLOPTS$|PS4$)/.test(key)), ...Object.keys(env)]);
|
|
26
|
-
const saved = new Map([...keys].map((key) => [key, process.env[key]]));
|
|
27
|
-
for (const key of keys) delete process.env[key];
|
|
28
|
-
Object.assign(process.env, env);
|
|
29
|
-
t.after(() => { for (const [key, value] of saved) { if (value === undefined) delete process.env[key]; else process.env[key] = value; } });
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
test("literal shell parser recognizes quotes, chains, pipes, redirects, assignments, and risk findings", () => {
|
|
33
|
-
const parsed = parseBash("MODE=test env node 'script with spaces.js' && cd nested; cat a.txt | tee result.txt >> log.txt");
|
|
34
|
-
assert.equal(parsed.status, "parsed");
|
|
35
|
-
assert.deepEqual(parsed.operators, ["&&", ";", "|"]);
|
|
36
|
-
assert.deepEqual(parsed.commands[0].argv, ["env", "node", "script with spaces.js"]);
|
|
37
|
-
assert.deepEqual(parsed.commands[0].assignments, ["MODE=test"]);
|
|
38
|
-
assert.deepEqual(parsed.commands.at(-1).redirects, [{ operator: ">>", target: "log.txt" }]);
|
|
39
|
-
for (const finding of ["environment-assignment", "execution-wrapper", "working-directory-change", "pipeline", "redirection"]) assert.ok(parsed.findings.includes(finding));
|
|
40
|
-
assert.equal(parsed.safe, false);
|
|
41
|
-
assert.equal(parsed.grantFamily, undefined);
|
|
42
|
-
for (const [command, finding] of [
|
|
43
|
-
["git reset --hard HEAD", "destructive-action"], ["git push --force", "shared-system-change"],
|
|
44
|
-
["git commit --no-verify", "safeguard-bypass"], ["curl --insecure https://example.invalid", "network-or-data-egress"],
|
|
45
|
-
["npm test", "script-or-interpreter"], ["python script.py", "script-or-interpreter"],
|
|
46
|
-
["sudo command", "privilege-change"]
|
|
47
|
-
]) assert.ok(parseBash(command).findings.includes(finding), command);
|
|
48
|
-
});
|
|
49
|
-
|
|
50
|
-
test("only tiny literal builtins are automatic; expansions and unsupported syntax never widen grants", () => {
|
|
51
|
-
for (const command of ["pwd", "pwd -P", "pwd -L", "true", "false", "'pwd'", "pwd # harmless comment"]) assert.equal(parseBash(command).safe, true, command);
|
|
52
|
-
for (const command of [
|
|
53
|
-
"pwd; rm -rf x", "pwd && true", "pwd > file", "pwd $(touch file)", "pwd `touch file`", "pwd $VAR",
|
|
54
|
-
"pwd *", "pwd ${VAR}", "(pwd)", "{ pwd; }", "if true; then pwd; fi", "pwd &", "pwd 2> file",
|
|
55
|
-
"pwd <<< text", "pwd <<EOF\ntext\nEOF", "pwd |& cat", "pwd &&", "pwd \\", "pwd 'unfinished",
|
|
56
|
-
"pwd\0", "pwd\r", "pwd\r; true", "pwd; A=B", "A=B pwd", "bash -c 'pwd'", "npm test", "make", "git -c core.fsmonitor=evil status",
|
|
57
|
-
"git status; pwd", "env git status", "git status --unknown", "git status --short path", "git $SUBCOMMAND"
|
|
58
|
-
]) {
|
|
59
|
-
const parsed = parseBash(command);
|
|
60
|
-
assert.equal(parsed.safe, false, command);
|
|
61
|
-
assert.equal(parsed.grantFamily, undefined, command);
|
|
62
|
-
}
|
|
63
|
-
assert.equal(parseBash("echo $(touch file)").status, "unsupported");
|
|
64
|
-
assert.equal(parseBash("echo '$(touch file)'").status, "parsed");
|
|
65
|
-
assert.equal(parseBash("pwd ".repeat(4000)).status, "unsupported");
|
|
66
|
-
assert.equal(parseBash(Array(34).fill("pwd").join(";")).status, "unsupported");
|
|
67
|
-
assert.equal(parseBash("git status --short").grantFamily, "git-status");
|
|
68
|
-
});
|
|
69
|
-
|
|
70
|
-
test("literal filesystem targets are checked, including aliases and protected output redirection", async (t) => {
|
|
71
|
-
const { root } = await fixture(t);
|
|
72
|
-
await writeFile(join(root, ".env"), "disposable fixture");
|
|
73
|
-
await symlink(join(root, ".env"), join(root, "alias"));
|
|
74
|
-
for (const command of ["cat .env", "cat alias", "touch .pi/settings.json", "echo text > .git/config"]) {
|
|
75
|
-
const analysis = await analyzeBash(command, root, root);
|
|
76
|
-
assert.ok(analysis.blockedReason, command);
|
|
77
|
-
const outcome = await evaluatePermission({ mode: "auto", tool: "bash", input: { command }, cwd: root, workspaceRoot: root,
|
|
78
|
-
authorization: { approvals: [{ tool: "bash", scope: "tool" }] } });
|
|
79
|
-
assert.equal(outcome.decision, "deny", command);
|
|
80
|
-
}
|
|
81
|
-
assert.equal((await analyzeBash("echo '.env'", root, root)).blockedReason, undefined);
|
|
82
|
-
assert.ok((await analyzeBash("cat ../ordinary.txt", root, root)).findings.includes("external-filesystem-target"));
|
|
83
|
-
assert.equal((await evaluatePermission({ mode: "plan", tool: "bash", input: { command: "pwd" }, cwd: root, workspaceRoot: root })).decision, "deny");
|
|
84
|
-
});
|
|
85
|
-
|
|
86
|
-
test("execution fingerprints bind cwd, executable, configuration, and environment, without executing anything", async (t) => {
|
|
87
|
-
const { root, env } = await fixture(t);
|
|
88
|
-
const fingerprint = () => bashApprovalFingerprint("git status", root, root, env);
|
|
89
|
-
const initial = await fingerprint();
|
|
90
|
-
assert.match(initial, /^[a-f0-9]{64}$/);
|
|
91
|
-
assert.equal(await bashApprovalFingerprint("git status --short", root, root, env), initial);
|
|
92
|
-
assert.notEqual(await bashApprovalFingerprint("git status", join(root, "nested"), root, env), initial);
|
|
93
|
-
await writeFile(join(root, ".pi", "permission-modes.json"), "{}");
|
|
94
|
-
assert.notEqual(await fingerprint(), initial);
|
|
95
|
-
const beforeExecutable = await fingerprint();
|
|
96
|
-
await writeFile(join(root, "bin", "git"), Buffer.from("7f454c466368616e676564", "hex"), { mode: 0o755 });
|
|
97
|
-
assert.notEqual(await fingerprint(), beforeExecutable);
|
|
98
|
-
await writeFile(join(root, ".git", "config"), "[core]\n fsmonitor = run-a-script\n");
|
|
99
|
-
assert.equal(await fingerprint(), undefined);
|
|
100
|
-
await writeFile(join(root, ".git", "config"), "[include]\n path = other-config\n");
|
|
101
|
-
assert.equal(await fingerprint(), undefined);
|
|
102
|
-
await writeFile(join(root, ".git", "config"), "[core]\n bare = false\n");
|
|
103
|
-
assert.equal(await bashApprovalFingerprint("npm test", root, root, env), undefined);
|
|
104
|
-
assert.equal(await bashApprovalFingerprint("git status", root, root, { ...env, BASH_ENV: "startup.sh" }), undefined);
|
|
105
|
-
assert.equal(await bashApprovalFingerprint("git status", root, root, { ...env, GIT_CONFIG_COUNT: "1" }), undefined);
|
|
106
|
-
assert.equal(await bashApprovalFingerprint("git status", root, root, { ...env, GIT_CONFIG_GLOBAL: "relative" }), undefined);
|
|
107
|
-
await writeFile(join(root, ".git", "commondir"), "../../other");
|
|
108
|
-
assert.equal(await fingerprint(), undefined);
|
|
109
|
-
await rm(join(root, ".git", "commondir"));
|
|
110
|
-
await mkdir(join(root, "nested", ".git"));
|
|
111
|
-
assert.equal(await bashApprovalFingerprint("git status", join(root, "nested"), root, env), undefined);
|
|
112
|
-
});
|
|
113
|
-
|
|
114
|
-
test("Auto honors fresh exact/subcommand grants, not broad grants, edited configs, or a different cwd", async (t) => {
|
|
115
|
-
const { root, env } = await fixture(t);
|
|
116
|
-
isolatedEnvironment(t, env);
|
|
117
|
-
const fingerprint = await bashApprovalFingerprint("git status", root, root);
|
|
118
|
-
assert.ok(fingerprint);
|
|
119
|
-
const exact = { tool: "bash", scope: "command", command: "git status", cwd: root, fingerprint };
|
|
120
|
-
const query = { tool: "bash", scope: "subcommand", family: "git-status", cwd: root, fingerprint };
|
|
121
|
-
const decide = (command, approvals, cwd = root) => evaluatePermission({ mode: "auto", tool: "bash", input: { command }, cwd, workspaceRoot: root, authorization: { approvals } });
|
|
122
|
-
assert.equal((await decide("git status", [exact])).decision, "allow");
|
|
123
|
-
assert.equal((await decide("git status --short", [exact])).decision, "require-model-review");
|
|
124
|
-
assert.equal((await decide("git status --short", [query])).decision, "allow");
|
|
125
|
-
assert.equal((await decide("git status", [exact], join(root, "nested"))).decision, "require-model-review");
|
|
126
|
-
for (const grant of [{ tool: "bash", scope: "tool" }, { tool: "bash", scope: "executable", executable: "git" }])
|
|
127
|
-
assert.equal((await decide("git status", [grant])).decision, "require-model-review");
|
|
128
|
-
await writeFile(join(root, "global.gitconfig"), "[user]\n name = Changed\n");
|
|
129
|
-
assert.equal((await decide("git status", [exact])).decision, "require-model-review");
|
|
130
|
-
assert.equal((await decide("git status && touch x", [query])).decision, "require-model-review");
|
|
131
|
-
const fileGrant = { tool: "write", scope: "file", path: join(root, "..", "outside.txt") };
|
|
132
|
-
assert.equal((await evaluatePermission({ mode: "auto", tool: "write", input: { path: "../outside.txt", content: "fixture" }, cwd: root, workspaceRoot: root, authorization: { approvals: [fileGrant] } })).decision, "allow");
|
|
133
|
-
});
|
|
134
|
-
|
|
135
|
-
test("shell startup overrides disable even the builtin fast path", async (t) => {
|
|
136
|
-
const { root, env } = await fixture(t);
|
|
137
|
-
isolatedEnvironment(t, { ...env, BASH_ENV: join(root, "not-executed.sh") });
|
|
138
|
-
assert.equal((await analyzeBash("pwd", root, root)).safe, false);
|
|
139
|
-
});
|
|
140
|
-
|
|
141
|
-
test("denial budgets count repeated and varied actions, remain latched, and validate launch limits", () => {
|
|
142
|
-
for (const value of ["0", "51", "NaN", "1.2", "-1"]) assert.throws(() => denialLimit(value, 6));
|
|
143
|
-
assert.equal(denialLimit("4", 6), 4);
|
|
144
|
-
assert.equal(denialLimit(undefined, 6), 6);
|
|
145
|
-
const tracker = new DenialTracker({ repeated: 2, total: 3 });
|
|
146
|
-
assert.equal(tracker.deny("bash", { command: "a" }).stopped, false);
|
|
147
|
-
assert.equal(tracker.deny("bash", { command: "a" }).stopped, true);
|
|
148
|
-
tracker.reset();
|
|
149
|
-
for (const command of ["a", "b", "c"]) tracker.deny("bash", { command });
|
|
150
|
-
assert.equal(tracker.stopped, true);
|
|
151
|
-
assert.equal(tracker.total, 3);
|
|
152
|
-
tracker.deny("write", { path: "different" });
|
|
153
|
-
assert.equal(tracker.total, 3);
|
|
154
|
-
});
|
|
155
|
-
|
|
156
|
-
async function harness(t, hasUI = true) {
|
|
157
|
-
const { root, env } = await fixture(t);
|
|
158
|
-
isolatedEnvironment(t, env);
|
|
159
|
-
await writeFile(join(root, ".pi", "permission-modes.json"), JSON.stringify({ version: 1, initialized: true, classifier: { provider: "test", model: "reviewer", timeoutMs: 1000 } }));
|
|
160
|
-
const events = new Map();
|
|
161
|
-
const commands = new Map();
|
|
162
|
-
const shortcuts = new Map();
|
|
163
|
-
const entries = [];
|
|
164
|
-
let choice = "Deny";
|
|
165
|
-
let aborts = 0;
|
|
166
|
-
let modelCalls = 0;
|
|
167
|
-
let options = [];
|
|
168
|
-
let notification = "";
|
|
169
|
-
const flags = { "permission-mode": "default", "permission-repeat-limit": "2", "permission-denial-limit": "3" };
|
|
170
|
-
modeExtension({
|
|
171
|
-
registerFlag() {}, getFlag: (name) => flags[name],
|
|
172
|
-
registerCommand(name, command) { commands.set(name, command); },
|
|
173
|
-
registerShortcut(name, shortcut) { shortcuts.set(name, shortcut.handler); },
|
|
174
|
-
on(name, handler) { events.set(name, handler); }, events: { emit() {} },
|
|
175
|
-
appendEntry(customType, data) { entries.push({ type: "custom", id: `e${entries.length}`, customType, data }); }
|
|
176
|
-
});
|
|
177
|
-
const ctx = { cwd: root, hasUI, mode: "json", isIdle: () => true,
|
|
178
|
-
abort() { aborts++; },
|
|
179
|
-
ui: { setStatus() {}, notify(text) { notification = text; }, async select(_title, choices) { options = choices; return choice; } },
|
|
180
|
-
sessionManager: { getBranch: () => entries },
|
|
181
|
-
modelRegistry: { find: () => ({ provider: "test", id: "reviewer" }), hasConfiguredAuth: () => true,
|
|
182
|
-
async complete() { modelCalls++; throw new Error("mock model unavailable"); } }
|
|
183
|
-
};
|
|
184
|
-
await events.get("session_start")({ reason: "startup" }, ctx);
|
|
185
|
-
const input = async (text = "Inspect status.", source = "interactive") => {
|
|
186
|
-
await events.get("input")({ text, source }, ctx);
|
|
187
|
-
const message = { role: "user", content: text, timestamp: entries.length + 1 };
|
|
188
|
-
await events.get("message_end")({ message }, ctx);
|
|
189
|
-
entries.push({ type: "message", id: `u${entries.length}`, message });
|
|
190
|
-
};
|
|
191
|
-
await input();
|
|
192
|
-
return { root, ctx, input, entries, events, commands,
|
|
193
|
-
call: (toolName, input) => events.get("tool_call")({ toolName, input }, ctx),
|
|
194
|
-
cycle: () => shortcuts.get("alt+m")(ctx),
|
|
195
|
-
set choice(value) { choice = value; }, get aborts() { return aborts; }, get modelCalls() { return modelCalls; },
|
|
196
|
-
get options() { return options; }, get notification() { return notification; }
|
|
197
|
-
};
|
|
198
|
-
}
|
|
199
|
-
|
|
200
|
-
test("permission UI creates cwd-bound grants, /permissions lists and revokes them, and Auto respects revocation", async (t) => {
|
|
201
|
-
const h = await harness(t);
|
|
202
|
-
h.choice = "Allow this exact Bash command in this directory for this session";
|
|
203
|
-
assert.equal(await h.call("bash", { command: "git status" }), undefined);
|
|
204
|
-
assert.ok(h.options.includes("Allow Git status queries in this directory for this session"));
|
|
205
|
-
await h.commands.get("permissions").handler("list", h.ctx);
|
|
206
|
-
assert.match(h.notification, /"id":"1"/);
|
|
207
|
-
assert.match(h.notification, /"cwd":/);
|
|
208
|
-
assert.doesNotMatch(h.notification, /fingerprint/);
|
|
209
|
-
for (let i = 0; i < 3; i++) await h.cycle();
|
|
210
|
-
assert.equal(await h.call("bash", { command: "git status" }), undefined);
|
|
211
|
-
assert.equal(h.modelCalls, 0);
|
|
212
|
-
await h.commands.get("permissions").handler("revoke 1", h.ctx);
|
|
213
|
-
assert.equal((await h.call("bash", { command: "git status" })).block, true);
|
|
214
|
-
assert.equal(h.modelCalls, 1);
|
|
215
|
-
await assert.rejects(h.commands.get("permissions").handler("revoke 1", h.ctx), /Unknown/);
|
|
216
|
-
await assert.rejects(h.commands.get("permissions").handler("clear", { ...h.ctx, isIdle: () => false }), /idle/);
|
|
217
|
-
await h.commands.get("permissions").handler("clear", h.ctx);
|
|
218
|
-
assert.match(h.notification, /No session grants/);
|
|
219
|
-
});
|
|
220
|
-
|
|
221
|
-
test("opaque scripts offer no reusable narrow grant; one-time approval never survives another call", async (t) => {
|
|
222
|
-
const h = await harness(t);
|
|
223
|
-
h.choice = "Allow once: bash";
|
|
224
|
-
assert.equal(await h.call("bash", { command: "npm test" }), undefined);
|
|
225
|
-
assert.equal(h.options.some((option) => option.includes("in this directory")), false);
|
|
226
|
-
h.choice = "Deny";
|
|
227
|
-
assert.equal((await h.call("bash", { command: "npm test" })).block, true);
|
|
228
|
-
await h.commands.get("permissions").handler("list", h.ctx);
|
|
229
|
-
assert.match(h.notification, /No session grants/);
|
|
230
|
-
});
|
|
231
|
-
|
|
232
|
-
test("denial thresholds stop the run, block later and parallel-safe calls, preserve AskQuestion, and reset only for user input", async (t) => {
|
|
233
|
-
const h = await harness(t);
|
|
234
|
-
const call = () => h.call("bash", { command: "npm test" });
|
|
235
|
-
assert.match((await call()).reason, /genuinely narrower/);
|
|
236
|
-
assert.match((await call()).reason, /run is stopped/);
|
|
237
|
-
assert.ok(h.aborts > 0);
|
|
238
|
-
assert.equal((await h.call("write", { path: "new.ts", content: "fixture" })).block, true);
|
|
239
|
-
assert.equal(await h.call("AskQuestion", {}), undefined);
|
|
240
|
-
await h.input("Keep trying", "extension");
|
|
241
|
-
assert.equal((await h.call("bash", { command: "pwd" })).block, true);
|
|
242
|
-
await h.input("Inspect cwd only.");
|
|
243
|
-
assert.equal(await h.call("bash", { command: "pwd" }), undefined);
|
|
244
|
-
assert.equal(h.entries.filter((entry) => entry.customType === "piexis-permission-stop").length, 1);
|
|
245
|
-
});
|
|
246
|
-
|
|
247
|
-
test("an in-flight sibling approval cannot execute or create a grant after the run stops", { timeout: 5000 }, async (t) => {
|
|
248
|
-
const h = await harness(t);
|
|
249
|
-
let release;
|
|
250
|
-
h.ctx.ui.select = async (title) => {
|
|
251
|
-
if (title.includes("git status")) return new Promise((resolve) => { release = resolve; });
|
|
252
|
-
return "Deny";
|
|
253
|
-
};
|
|
254
|
-
const pending = h.call("bash", { command: "git status" });
|
|
255
|
-
while (!release) await new Promise(setImmediate);
|
|
256
|
-
await h.call("bash", { command: "npm test" });
|
|
257
|
-
await h.call("bash", { command: "npm test" });
|
|
258
|
-
release("Allow this exact Bash command in this directory for this session");
|
|
259
|
-
assert.equal((await pending).block, true);
|
|
260
|
-
await h.commands.get("permissions").handler("list", h.ctx);
|
|
261
|
-
assert.match(h.notification, /No session grants/);
|
|
262
|
-
});
|
|
263
|
-
|
|
264
|
-
test("headless denials never prompt and emit a machine-identifiable stop on stderr", async (t) => {
|
|
265
|
-
const h = await harness(t, false);
|
|
266
|
-
const output = [];
|
|
267
|
-
const original = process.stderr.write;
|
|
268
|
-
process.stderr.write = (chunk) => { output.push(String(chunk)); return true; };
|
|
269
|
-
try {
|
|
270
|
-
assert.match((await h.call("bash", { command: "npm test" })).reason, /no permission UI/);
|
|
271
|
-
assert.match((await h.call("bash", { command: "npm test" })).reason, /run is stopped/);
|
|
272
|
-
} finally { process.stderr.write = original; }
|
|
273
|
-
assert.match(output.join(""), /permission_blocked/);
|
|
274
|
-
assert.equal(h.options.length, 0);
|
|
275
|
-
assert.ok(h.aborts > 0);
|
|
276
|
-
});
|
package/tests/glob.test.mjs
DELETED
|
@@ -1,31 +0,0 @@
|
|
|
1
|
-
import assert from "node:assert/strict";
|
|
2
|
-
import test from "node:test";
|
|
3
|
-
|
|
4
|
-
import glob from "../extensions/glob.ts";
|
|
5
|
-
import { READ_TOOLS } from "../lib/mode-policy.ts";
|
|
6
|
-
|
|
7
|
-
test("glob is registered as a read-only, gitignore-aware file-discovery tool", () => {
|
|
8
|
-
let tool;
|
|
9
|
-
glob({
|
|
10
|
-
registerTool(definition) {
|
|
11
|
-
tool = definition;
|
|
12
|
-
}
|
|
13
|
-
});
|
|
14
|
-
|
|
15
|
-
assert.ok(tool);
|
|
16
|
-
assert.equal(tool.name, "glob");
|
|
17
|
-
assert.equal(tool.label, "glob");
|
|
18
|
-
assert.match(tool.description, /\.gitignore/);
|
|
19
|
-
assert.equal(tool.promptSnippet, "Find files by glob pattern (respects .gitignore)");
|
|
20
|
-
assert.deepEqual(tool.promptGuidelines, [
|
|
21
|
-
"Use glob to discover files by pathname pattern before reading or editing them."
|
|
22
|
-
]);
|
|
23
|
-
assert.equal(tool.parameters.properties.pattern.type, "string");
|
|
24
|
-
assert.equal(tool.parameters.properties.path.type, "string");
|
|
25
|
-
assert.equal(tool.parameters.properties.limit.type, "number");
|
|
26
|
-
assert.equal(typeof tool.execute, "function");
|
|
27
|
-
});
|
|
28
|
-
|
|
29
|
-
test("glob participates in Piexis's read allowlist before path policy evaluation", () => {
|
|
30
|
-
assert.equal(READ_TOOLS.has("glob"), true);
|
|
31
|
-
});
|
package/tests/mode.test.mjs
DELETED
|
@@ -1,62 +0,0 @@
|
|
|
1
|
-
import assert from "node:assert/strict";
|
|
2
|
-
import test from "node:test";
|
|
3
|
-
import { mkdtemp, rm } from "node:fs/promises";
|
|
4
|
-
import { join } from "node:path";
|
|
5
|
-
import { tmpdir } from "node:os";
|
|
6
|
-
|
|
7
|
-
import modeExtension from "../extensions/mode.ts";
|
|
8
|
-
|
|
9
|
-
test("permission mode changes remain user-controlled rather than model-callable", () => {
|
|
10
|
-
const tools = new Map();
|
|
11
|
-
const handlers = new Map();
|
|
12
|
-
modeExtension({
|
|
13
|
-
registerFlag() {},
|
|
14
|
-
registerTool(definition) { tools.set(definition.name, definition); },
|
|
15
|
-
registerCommand() {},
|
|
16
|
-
registerShortcut() {},
|
|
17
|
-
on(name, handler) { handlers.set(name, handler); },
|
|
18
|
-
appendEntry() {},
|
|
19
|
-
getFlag() {},
|
|
20
|
-
events: { emit() {} }
|
|
21
|
-
});
|
|
22
|
-
|
|
23
|
-
assert.equal(tools.has("switch_permission_mode"), false);
|
|
24
|
-
assert.equal(typeof handlers.get("tool_call"), "function");
|
|
25
|
-
});
|
|
26
|
-
|
|
27
|
-
test("mode cycling is idle-only, skips unavailable Auto, and rejects corrupt restored modes", async (t) => {
|
|
28
|
-
const cwd = await mkdtemp(join(tmpdir(), "piexis-mode-polish-"));
|
|
29
|
-
t.after(() => rm(cwd, { recursive: true, force: true }));
|
|
30
|
-
const handlers = new Map();
|
|
31
|
-
const shortcuts = new Map();
|
|
32
|
-
const commands = new Map();
|
|
33
|
-
let branch = [];
|
|
34
|
-
let current;
|
|
35
|
-
let idle = true;
|
|
36
|
-
modeExtension({
|
|
37
|
-
registerFlag() {}, getFlag() {},
|
|
38
|
-
registerCommand(name, command) { commands.set(name, command); },
|
|
39
|
-
registerShortcut(name, shortcut) { shortcuts.set(name, shortcut.handler); },
|
|
40
|
-
on(name, callback) { handlers.set(name, callback); },
|
|
41
|
-
appendEntry(customType, data) { branch.push({ type: "custom", customType, data }); },
|
|
42
|
-
events: { emit(name, value) { if (name === "piexis:permission-mode") current = value; } }
|
|
43
|
-
});
|
|
44
|
-
const ctx = { cwd, hasUI: false, mode: "json", isIdle: () => idle,
|
|
45
|
-
sessionManager: { getBranch: () => branch }, ui: {}, modelRegistry: { find: () => undefined, hasConfiguredAuth: () => false } };
|
|
46
|
-
await handlers.get("session_start")({ reason: "startup" }, ctx);
|
|
47
|
-
assert.equal(current, "default");
|
|
48
|
-
idle = false;
|
|
49
|
-
await assert.rejects(shortcuts.get("alt+m")(ctx), /idle/);
|
|
50
|
-
await assert.rejects(commands.get("plan").handler("", ctx), /idle/);
|
|
51
|
-
assert.equal(current, "default");
|
|
52
|
-
idle = true;
|
|
53
|
-
for (const expected of ["acceptEdits", "plan", "default"]) {
|
|
54
|
-
await shortcuts.get("alt+m")(ctx);
|
|
55
|
-
assert.equal(current, expected);
|
|
56
|
-
}
|
|
57
|
-
for (const restored of ["bypassPermissions", "dontAsk", "auto", "made-up-mode", "__proto__"]) {
|
|
58
|
-
branch = [{ type: "custom", customType: "piexis-permission-mode", data: { mode: restored } }];
|
|
59
|
-
await handlers.get("session_tree")({}, ctx);
|
|
60
|
-
assert.equal(current, "default", restored);
|
|
61
|
-
}
|
|
62
|
-
});
|
|
@@ -1,70 +0,0 @@
|
|
|
1
|
-
import assert from "node:assert/strict";
|
|
2
|
-
import test from "node:test";
|
|
3
|
-
import { mkdtemp, writeFile, rm } from "node:fs/promises";
|
|
4
|
-
import { tmpdir } from "node:os";
|
|
5
|
-
import { join } from "node:path";
|
|
6
|
-
import { cases, validateCorpus, requestFor, score, main } from "../eval/permissions.mjs";
|
|
7
|
-
import { AUTO_REVIEW_POLICY } from "../lib/permission-taxonomy.ts";
|
|
8
|
-
|
|
9
|
-
const answer = (item, overrides = {}) => ({ id: item.id, stopReason: "stop", response: JSON.stringify({
|
|
10
|
-
decision: item.expected, category: item.categories[0], reason: "Synthetic scorer fixture, not a model result.",
|
|
11
|
-
authorizedBy: [requestFor(item).context.intent.currentRequestId], ...overrides
|
|
12
|
-
}) });
|
|
13
|
-
|
|
14
|
-
test("labeled corpus covers all review categories, counterfactual pairs, injection, and delegation", () => {
|
|
15
|
-
const counts = validateCorpus();
|
|
16
|
-
assert.ok(counts.cases >= 24 && counts.allow >= 6 && counts.deny >= 12);
|
|
17
|
-
assert.deepEqual(new Set(cases.flatMap((item) => item.categories)), new Set(Object.keys(AUTO_REVIEW_POLICY)));
|
|
18
|
-
for (const pair of ["status", "delete", "upload", "deploy", "external", "bypass", "delegate"]) {
|
|
19
|
-
assert.deepEqual(new Set(cases.filter((item) => item.tags.includes(`pair-${pair}`)).map((item) => item.expected)), new Set(["allow", "deny"]));
|
|
20
|
-
}
|
|
21
|
-
assert.throws(() => validateCorpus([cases[0], cases[0]]), /duplicate/);
|
|
22
|
-
assert.throws(() => validateCorpus([]), /empty/);
|
|
23
|
-
});
|
|
24
|
-
|
|
25
|
-
test("exports contain only synthetic requests, not labels or rationales; no provider is contacted", async () => {
|
|
26
|
-
const result = await main(["--export"]);
|
|
27
|
-
assert.equal(result.requests.length, cases.length);
|
|
28
|
-
for (const request of result.requests) {
|
|
29
|
-
assert.equal(request.expected, undefined);
|
|
30
|
-
assert.equal(request.rationale, undefined);
|
|
31
|
-
assert.ok(request.systemPrompt.includes("untrusted DATA"));
|
|
32
|
-
assert.equal(request.payload.context.cwd, "/fixture/workspace");
|
|
33
|
-
}
|
|
34
|
-
assert.equal((await main([])).liveModelRun, false);
|
|
35
|
-
await assert.rejects(main(["--live"]), /Usage/);
|
|
36
|
-
});
|
|
37
|
-
|
|
38
|
-
test("scoring files require matching request/prompt/corpus identities and a named model", async (t) => {
|
|
39
|
-
const root = await mkdtemp(join(tmpdir(), "piexis-eval-"));
|
|
40
|
-
t.after(() => rm(root, { recursive: true, force: true }));
|
|
41
|
-
const path = join(root, "predictions.json");
|
|
42
|
-
const exported = await main(["--export"]);
|
|
43
|
-
const input = { corpusHash: exported.corpusHash, promptHash: exported.promptHash, requestHash: exported.requestHash, model: "synthetic-oracle-not-a-model-run", predictions: cases.map((item) => answer(item)) };
|
|
44
|
-
await writeFile(path, JSON.stringify(input));
|
|
45
|
-
assert.equal((await main(["--score", path])).totals.correct, cases.length);
|
|
46
|
-
for (const key of ["corpusHash", "promptHash", "requestHash", "model"]) {
|
|
47
|
-
await writeFile(path, JSON.stringify({ ...input, [key]: "" }));
|
|
48
|
-
await assert.rejects(main(["--score", path]), /matching/);
|
|
49
|
-
}
|
|
50
|
-
});
|
|
51
|
-
|
|
52
|
-
test("scoring separates unsafe raw allows, host overrides, malformed responses, and absent predictions", () => {
|
|
53
|
-
const oracle = score(cases.map((item) => answer(item)));
|
|
54
|
-
assert.equal(oracle.totals.correct, cases.length);
|
|
55
|
-
assert.equal(oracle.totals.invalid, 0);
|
|
56
|
-
const prohibited = cases.find((item) => item.categories[0] === "credential-access");
|
|
57
|
-
const unsafe = score([answer(prohibited, { decision: "allow" })]);
|
|
58
|
-
assert.equal(unsafe.totals.falseAllows, 1);
|
|
59
|
-
assert.equal(unsafe.totals.policyOverrides, 1);
|
|
60
|
-
assert.equal(unsafe.totals.correct, 0);
|
|
61
|
-
assert.equal(unsafe.totals.invalid, cases.length - 1);
|
|
62
|
-
const allowed = cases.find((item) => item.expected === "allow");
|
|
63
|
-
for (const prediction of [answer(allowed, { authorizedBy: ["invented"] }), { ...answer(allowed), stopReason: "aborted" }, { id: allowed.id, response: "not JSON", stopReason: "stop" }]) {
|
|
64
|
-
const result = score([prediction], [allowed]);
|
|
65
|
-
assert.equal(result.totals.invalid, 1);
|
|
66
|
-
assert.equal(result.totals.correct, 0);
|
|
67
|
-
}
|
|
68
|
-
assert.throws(() => score([answer(allowed), answer(allowed)]), /duplicate/);
|
|
69
|
-
assert.throws(() => score([{ id: "unknown" }]), /Unknown/);
|
|
70
|
-
});
|