efaimo 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +47 -0
- package/LICENSE +202 -0
- package/NOTICE +6 -0
- package/README.md +281 -0
- package/SKILL.md +57 -0
- package/dist/check/check.d.ts +31 -0
- package/dist/check/check.js +77 -0
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +372 -0
- package/dist/clients/introspect.d.ts +8 -0
- package/dist/clients/introspect.js +198 -0
- package/dist/clients/rawprobe.d.ts +69 -0
- package/dist/clients/rawprobe.js +597 -0
- package/dist/clients/stateless.d.ts +7 -0
- package/dist/clients/stateless.js +121 -0
- package/dist/core/engine.d.ts +6 -0
- package/dist/core/engine.js +17 -0
- package/dist/core/grade.d.ts +8 -0
- package/dist/core/grade.js +19 -0
- package/dist/core/types.d.ts +243 -0
- package/dist/core/types.js +1 -0
- package/dist/index.d.ts +16 -0
- package/dist/index.js +26 -0
- package/dist/mcp/server.d.ts +3 -0
- package/dist/mcp/server.js +88 -0
- package/dist/reporters/badge.d.ts +12 -0
- package/dist/reporters/badge.js +37 -0
- package/dist/reporters/json.d.ts +1 -0
- package/dist/reporters/json.js +10 -0
- package/dist/reporters/markdown.d.ts +7 -0
- package/dist/reporters/markdown.js +121 -0
- package/dist/reporters/pretty.d.ts +15 -0
- package/dist/reporters/pretty.js +221 -0
- package/dist/rules/injection.d.ts +20 -0
- package/dist/rules/injection.js +106 -0
- package/dist/rules/mcp/index.d.ts +4 -0
- package/dist/rules/mcp/index.js +643 -0
- package/dist/rules/skill/index.d.ts +2 -0
- package/dist/rules/skill/index.js +279 -0
- package/dist/skills/parse.d.ts +4 -0
- package/dist/skills/parse.js +113 -0
- package/dist/targets/clientConfigs.d.ts +17 -0
- package/dist/targets/clientConfigs.js +110 -0
- package/dist/targets/repoScan.d.ts +2 -0
- package/dist/targets/repoScan.js +0 -0
- package/dist/targets/resolve.d.ts +28 -0
- package/dist/targets/resolve.js +48 -0
- package/dist/testing/anthropicRunner.d.ts +6 -0
- package/dist/testing/anthropicRunner.js +33 -0
- package/dist/testing/harness.d.ts +45 -0
- package/dist/testing/harness.js +96 -0
- package/dist/testing/openaiRunner.d.ts +12 -0
- package/dist/testing/openaiRunner.js +40 -0
- package/dist/util/childEnv.d.ts +1 -0
- package/dist/util/childEnv.js +43 -0
- package/dist/util/dotenv.d.ts +12 -0
- package/dist/util/dotenv.js +53 -0
- package/dist/util/misc.d.ts +15 -0
- package/dist/util/misc.js +121 -0
- package/dist/version.d.ts +2 -0
- package/dist/version.js +2 -0
- package/dist/weigh/anthropicExact.d.ts +11 -0
- package/dist/weigh/anthropicExact.js +46 -0
- package/dist/weigh/diff.d.ts +14 -0
- package/dist/weigh/diff.js +19 -0
- package/dist/weigh/serializers.d.ts +21 -0
- package/dist/weigh/serializers.js +44 -0
- package/dist/weigh/tokens.d.ts +12 -0
- package/dist/weigh/tokens.js +23 -0
- package/dist/weigh/weigh.d.ts +6 -0
- package/dist/weigh/weigh.js +104 -0
- package/docs/INTEGRATIONS.md +119 -0
- package/docs/METHODOLOGY.md +83 -0
- package/docs/RULES.md +76 -0
- package/package.json +77 -0
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* efaimo test: does a skill actually improve task completion? A scenario runs a
|
|
3
|
+
* task with and without the skill loaded, N trials each, and an LLM judge scores
|
|
4
|
+
* each attempt. The runner is injected so the A/B logic is testable without any
|
|
5
|
+
* API calls; the live runner (anthropicRunner) is used only with `--live`.
|
|
6
|
+
*/
|
|
7
|
+
export interface Scenario {
|
|
8
|
+
name: string;
|
|
9
|
+
skillPath: string;
|
|
10
|
+
skillName: string;
|
|
11
|
+
skillBody: string;
|
|
12
|
+
model: string;
|
|
13
|
+
trials: number;
|
|
14
|
+
task: string;
|
|
15
|
+
judge: string;
|
|
16
|
+
}
|
|
17
|
+
export interface RunnerRequest {
|
|
18
|
+
system?: string;
|
|
19
|
+
user: string;
|
|
20
|
+
model: string;
|
|
21
|
+
}
|
|
22
|
+
export type Runner = (req: RunnerRequest) => Promise<string>;
|
|
23
|
+
export interface ArmResult {
|
|
24
|
+
trials: number;
|
|
25
|
+
passes: number;
|
|
26
|
+
passRate: number;
|
|
27
|
+
}
|
|
28
|
+
export interface TestReport {
|
|
29
|
+
scenario: string;
|
|
30
|
+
skill: string;
|
|
31
|
+
model: string;
|
|
32
|
+
withSkill: ArmResult;
|
|
33
|
+
withoutSkill: ArmResult;
|
|
34
|
+
/** withSkill.passRate - withoutSkill.passRate, in percentage points. */
|
|
35
|
+
deltaPoints: number;
|
|
36
|
+
verdict: "helps" | "hurts" | "no measurable effect" | "inconclusive";
|
|
37
|
+
notes: string[];
|
|
38
|
+
}
|
|
39
|
+
export declare function parseScenario(file: string): Scenario;
|
|
40
|
+
/** The two system prompts an A/B trial compares: base, and base + the skill. */
|
|
41
|
+
export declare function armSystems(scenario: Scenario): {
|
|
42
|
+
withoutSkill: string | undefined;
|
|
43
|
+
withSkill: string;
|
|
44
|
+
};
|
|
45
|
+
export declare function runScenario(scenario: Scenario, runner: Runner): Promise<TestReport>;
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import YAML from "yaml";
|
|
4
|
+
import { findSkills } from "../skills/parse.js";
|
|
5
|
+
const JUDGE_SYSTEM = "You are a strict grader. Read the task, the rubric, and the assistant's answer. " +
|
|
6
|
+
"Reply with exactly one word: PASS or FAIL. No explanation.";
|
|
7
|
+
export function parseScenario(file) {
|
|
8
|
+
const raw = fs.readFileSync(file, "utf8");
|
|
9
|
+
const doc = YAML.parse(raw);
|
|
10
|
+
if (!doc || typeof doc !== "object")
|
|
11
|
+
throw new Error(`${file}: not a YAML mapping`);
|
|
12
|
+
const name = str(doc.name) ?? path.basename(file);
|
|
13
|
+
const skillRel = str(doc.skill);
|
|
14
|
+
if (!skillRel)
|
|
15
|
+
throw new Error(`${file}: 'skill' (path to a skill dir or SKILL.md) is required`);
|
|
16
|
+
const task = str(doc.task);
|
|
17
|
+
if (!task)
|
|
18
|
+
throw new Error(`${file}: 'task' (the prompt to run) is required`);
|
|
19
|
+
const judge = str(doc.judge);
|
|
20
|
+
if (!judge)
|
|
21
|
+
throw new Error(`${file}: 'judge' (a PASS/FAIL rubric) is required`);
|
|
22
|
+
const skillPath = path.resolve(path.dirname(file), skillRel);
|
|
23
|
+
const set = findSkills(skillPath);
|
|
24
|
+
const skill = set.skills[0];
|
|
25
|
+
if (!skill)
|
|
26
|
+
throw new Error(`${file}: no SKILL.md found at '${skillRel}'`);
|
|
27
|
+
const trials = Math.min(50, Math.max(1, Math.round(num(doc.trials) ?? 5)));
|
|
28
|
+
return {
|
|
29
|
+
name,
|
|
30
|
+
skillPath,
|
|
31
|
+
skillName: skill.name ?? path.basename(skill.dir),
|
|
32
|
+
skillBody: skill.body.trim(),
|
|
33
|
+
model: str(doc.model) ?? "claude-sonnet-5",
|
|
34
|
+
trials,
|
|
35
|
+
task,
|
|
36
|
+
judge,
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
function str(v) {
|
|
40
|
+
return typeof v === "string" && v.trim() ? v.trim() : undefined;
|
|
41
|
+
}
|
|
42
|
+
function num(v) {
|
|
43
|
+
return typeof v === "number" && Number.isFinite(v) ? v : undefined;
|
|
44
|
+
}
|
|
45
|
+
/** The two system prompts an A/B trial compares: base, and base + the skill. */
|
|
46
|
+
export function armSystems(scenario) {
|
|
47
|
+
const withSkill = `You have access to the following skill. Use it if relevant.\n\n<skill name="${scenario.skillName}">\n${scenario.skillBody}\n</skill>`;
|
|
48
|
+
return { withoutSkill: undefined, withSkill };
|
|
49
|
+
}
|
|
50
|
+
async function judgeOne(runner, scenario, answer) {
|
|
51
|
+
const out = await runner({
|
|
52
|
+
system: JUDGE_SYSTEM,
|
|
53
|
+
user: `TASK:\n${scenario.task}\n\nRUBRIC:\n${scenario.judge}\n\nASSISTANT ANSWER:\n${answer}\n\nVerdict (PASS or FAIL):`,
|
|
54
|
+
model: scenario.model,
|
|
55
|
+
});
|
|
56
|
+
return /\bpass\b/i.test(out) && !/\bfail\b/i.test(out);
|
|
57
|
+
}
|
|
58
|
+
async function runArm(runner, scenario, system) {
|
|
59
|
+
let passes = 0;
|
|
60
|
+
for (let i = 0; i < scenario.trials; i++) {
|
|
61
|
+
const answer = await runner({ system, user: scenario.task, model: scenario.model });
|
|
62
|
+
if (await judgeOne(runner, scenario, answer))
|
|
63
|
+
passes++;
|
|
64
|
+
}
|
|
65
|
+
return { trials: scenario.trials, passes, passRate: (passes / scenario.trials) * 100 };
|
|
66
|
+
}
|
|
67
|
+
export async function runScenario(scenario, runner) {
|
|
68
|
+
const systems = armSystems(scenario);
|
|
69
|
+
const withoutSkill = await runArm(runner, scenario, systems.withoutSkill);
|
|
70
|
+
const withSkill = await runArm(runner, scenario, systems.withSkill);
|
|
71
|
+
const deltaPoints = Math.round((withSkill.passRate - withoutSkill.passRate) * 10) / 10;
|
|
72
|
+
const notes = [
|
|
73
|
+
`${scenario.trials} trials per arm; probabilistic. Treat small deltas as noise and raise trials for confidence.`,
|
|
74
|
+
];
|
|
75
|
+
let verdict;
|
|
76
|
+
if (scenario.trials < 5) {
|
|
77
|
+
verdict = "inconclusive";
|
|
78
|
+
notes.push("fewer than 5 trials: not enough signal to conclude.");
|
|
79
|
+
}
|
|
80
|
+
else if (deltaPoints >= 15)
|
|
81
|
+
verdict = "helps";
|
|
82
|
+
else if (deltaPoints <= -15)
|
|
83
|
+
verdict = "hurts";
|
|
84
|
+
else
|
|
85
|
+
verdict = "no measurable effect";
|
|
86
|
+
return {
|
|
87
|
+
scenario: scenario.name,
|
|
88
|
+
skill: scenario.skillName,
|
|
89
|
+
model: scenario.model,
|
|
90
|
+
withSkill,
|
|
91
|
+
withoutSkill,
|
|
92
|
+
deltaPoints,
|
|
93
|
+
verdict,
|
|
94
|
+
notes,
|
|
95
|
+
};
|
|
96
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { Runner } from "./harness.js";
|
|
2
|
+
/**
|
|
3
|
+
* The OpenAI live runner (Chat Completions). Used by `efaimo test --live` when the
|
|
4
|
+
* scenario model is an OpenAI model. Spends tokens on the caller's OPENAI_API_KEY.
|
|
5
|
+
*/
|
|
6
|
+
export declare function openaiRunner(apiKey: string): Runner;
|
|
7
|
+
/**
|
|
8
|
+
* Pick a provider from the model name. Returns "unknown" for a model that is
|
|
9
|
+
* neither a recognized OpenAI nor Claude name, so the caller can fail clearly
|
|
10
|
+
* instead of silently routing, say, a Gemini model to the Anthropic API.
|
|
11
|
+
*/
|
|
12
|
+
export declare function providerForModel(model: string): "openai" | "anthropic" | "unknown";
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The OpenAI live runner (Chat Completions). Used by `efaimo test --live` when the
|
|
3
|
+
* scenario model is an OpenAI model. Spends tokens on the caller's OPENAI_API_KEY.
|
|
4
|
+
*/
|
|
5
|
+
export function openaiRunner(apiKey) {
|
|
6
|
+
return async (req) => {
|
|
7
|
+
const messages = [];
|
|
8
|
+
if (req.system)
|
|
9
|
+
messages.push({ role: "system", content: req.system });
|
|
10
|
+
messages.push({ role: "user", content: req.user });
|
|
11
|
+
const res = await fetch("https://api.openai.com/v1/chat/completions", {
|
|
12
|
+
method: "POST",
|
|
13
|
+
headers: {
|
|
14
|
+
"content-type": "application/json",
|
|
15
|
+
authorization: `Bearer ${apiKey}`,
|
|
16
|
+
},
|
|
17
|
+
body: JSON.stringify({ model: req.model, messages, max_completion_tokens: 1024 }),
|
|
18
|
+
redirect: "error",
|
|
19
|
+
signal: AbortSignal.timeout(60000),
|
|
20
|
+
});
|
|
21
|
+
if (!res.ok) {
|
|
22
|
+
const body = await res.text().catch(() => "");
|
|
23
|
+
throw new Error(`openai API ${res.status}: ${body.slice(0, 160)}`);
|
|
24
|
+
}
|
|
25
|
+
const json = (await res.json());
|
|
26
|
+
return json.choices?.[0]?.message?.content ?? "";
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Pick a provider from the model name. Returns "unknown" for a model that is
|
|
31
|
+
* neither a recognized OpenAI nor Claude name, so the caller can fail clearly
|
|
32
|
+
* instead of silently routing, say, a Gemini model to the Anthropic API.
|
|
33
|
+
*/
|
|
34
|
+
export function providerForModel(model) {
|
|
35
|
+
if (/^(gpt|o[0-9]|chatgpt|text-|davinci)/i.test(model))
|
|
36
|
+
return "openai";
|
|
37
|
+
if (/^claude/i.test(model))
|
|
38
|
+
return "anthropic";
|
|
39
|
+
return "unknown";
|
|
40
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function minimalChildEnv(): Record<string, string>;
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Minimal environment for spawned MCP servers: enough to run node/npx/python
|
|
3
|
+
* on each OS without leaking the caller's full environment (mirrors the
|
|
4
|
+
* conservative default the official SDK uses).
|
|
5
|
+
*/
|
|
6
|
+
const KEEP = [
|
|
7
|
+
"PATH",
|
|
8
|
+
"PATHEXT",
|
|
9
|
+
"SYSTEMROOT",
|
|
10
|
+
"SYSTEMDRIVE",
|
|
11
|
+
"COMSPEC",
|
|
12
|
+
"WINDIR",
|
|
13
|
+
"HOME",
|
|
14
|
+
"USERPROFILE",
|
|
15
|
+
"USERNAME",
|
|
16
|
+
"USER",
|
|
17
|
+
"LOGNAME",
|
|
18
|
+
"SHELL",
|
|
19
|
+
"TEMP",
|
|
20
|
+
"TMP",
|
|
21
|
+
"TMPDIR",
|
|
22
|
+
"APPDATA",
|
|
23
|
+
"LOCALAPPDATA",
|
|
24
|
+
"PROGRAMFILES",
|
|
25
|
+
"PROGRAMFILES(X86)",
|
|
26
|
+
"PROGRAMDATA",
|
|
27
|
+
"LANG",
|
|
28
|
+
"LC_ALL",
|
|
29
|
+
"TERM",
|
|
30
|
+
"NODE",
|
|
31
|
+
"NVM_BIN",
|
|
32
|
+
];
|
|
33
|
+
export function minimalChildEnv() {
|
|
34
|
+
const out = {};
|
|
35
|
+
for (const key of Object.keys(process.env)) {
|
|
36
|
+
if (KEEP.includes(key.toUpperCase())) {
|
|
37
|
+
const v = process.env[key];
|
|
38
|
+
if (v !== undefined)
|
|
39
|
+
out[key] = v;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
return out;
|
|
43
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Minimal, dependency-free .env loader for the two commands that need an API key
|
|
3
|
+
* (`test --live`, `weigh --anthropic`). It reads KEY=VALUE lines from a .env file
|
|
4
|
+
* and fills process.env only for keys that are not already set, so a real shell
|
|
5
|
+
* environment always wins. A missing file is a silent no-op.
|
|
6
|
+
*
|
|
7
|
+
* Values are never logged; the return value is the list of key NAMES loaded, so a
|
|
8
|
+
* caller can confirm "loaded X from .env" without ever printing a secret. Loaded
|
|
9
|
+
* secrets are not forwarded to spawned MCP servers: minimalChildEnv() passes an
|
|
10
|
+
* allowlist that excludes API keys.
|
|
11
|
+
*/
|
|
12
|
+
export declare function loadDotEnv(dir?: string): string[];
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { readFileSync } from "node:fs";
|
|
2
|
+
import { resolve } from "node:path";
|
|
3
|
+
/**
|
|
4
|
+
* Minimal, dependency-free .env loader for the two commands that need an API key
|
|
5
|
+
* (`test --live`, `weigh --anthropic`). It reads KEY=VALUE lines from a .env file
|
|
6
|
+
* and fills process.env only for keys that are not already set, so a real shell
|
|
7
|
+
* environment always wins. A missing file is a silent no-op.
|
|
8
|
+
*
|
|
9
|
+
* Values are never logged; the return value is the list of key NAMES loaded, so a
|
|
10
|
+
* caller can confirm "loaded X from .env" without ever printing a secret. Loaded
|
|
11
|
+
* secrets are not forwarded to spawned MCP servers: minimalChildEnv() passes an
|
|
12
|
+
* allowlist that excludes API keys.
|
|
13
|
+
*/
|
|
14
|
+
export function loadDotEnv(dir = process.cwd()) {
|
|
15
|
+
let text;
|
|
16
|
+
try {
|
|
17
|
+
text = readFileSync(resolve(dir, ".env"), "utf8");
|
|
18
|
+
}
|
|
19
|
+
catch {
|
|
20
|
+
return [];
|
|
21
|
+
}
|
|
22
|
+
const loaded = [];
|
|
23
|
+
for (const raw of text.split(/\r?\n/)) {
|
|
24
|
+
let line = raw.trim();
|
|
25
|
+
if (line === "" || line.startsWith("#"))
|
|
26
|
+
continue;
|
|
27
|
+
if (line.startsWith("export "))
|
|
28
|
+
line = line.slice(7).trim();
|
|
29
|
+
const eq = line.indexOf("=");
|
|
30
|
+
if (eq <= 0)
|
|
31
|
+
continue;
|
|
32
|
+
const key = line.slice(0, eq).trim();
|
|
33
|
+
if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(key))
|
|
34
|
+
continue;
|
|
35
|
+
if (process.env[key] !== undefined)
|
|
36
|
+
continue;
|
|
37
|
+
let val = line.slice(eq + 1).trim();
|
|
38
|
+
const quote = val[0];
|
|
39
|
+
if (val.length >= 2 && (quote === '"' || quote === "'") && val[val.length - 1] === quote) {
|
|
40
|
+
val = val.slice(1, -1);
|
|
41
|
+
}
|
|
42
|
+
else {
|
|
43
|
+
// Unquoted: a "#" preceded by whitespace starts an inline comment. Keys never
|
|
44
|
+
// contain a space, so this cannot truncate a real value.
|
|
45
|
+
const hash = val.search(/\s#/);
|
|
46
|
+
if (hash !== -1)
|
|
47
|
+
val = val.slice(0, hash).trimEnd();
|
|
48
|
+
}
|
|
49
|
+
process.env[key] = val;
|
|
50
|
+
loaded.push(key);
|
|
51
|
+
}
|
|
52
|
+
return loaded;
|
|
53
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/** Split a command string into command + args, honoring single/double quotes. */
|
|
2
|
+
export declare function parseCommandString(input: string): {
|
|
3
|
+
command: string;
|
|
4
|
+
args: string[];
|
|
5
|
+
};
|
|
6
|
+
export declare function walkFiles(root: string, opts?: {
|
|
7
|
+
maxDepth?: number;
|
|
8
|
+
skipDirs?: Set<string>;
|
|
9
|
+
}): Generator<string>;
|
|
10
|
+
export declare function readTextSafe(file: string, maxBytes?: number): string | undefined;
|
|
11
|
+
export declare function isBinaryPath(file: string): boolean;
|
|
12
|
+
export declare function extractVersion(s: string): number[] | undefined;
|
|
13
|
+
export declare function truncate(s: string, n: number): string;
|
|
14
|
+
export declare function formatTokens(n: number): string;
|
|
15
|
+
export declare function withTimeout<T>(p: Promise<T>, ms: number, what: string): Promise<T>;
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
/** Split a command string into command + args, honoring single/double quotes. */
|
|
4
|
+
export function parseCommandString(input) {
|
|
5
|
+
const tokens = [];
|
|
6
|
+
let current = "";
|
|
7
|
+
let quote = null;
|
|
8
|
+
for (const ch of input.trim()) {
|
|
9
|
+
if (quote) {
|
|
10
|
+
if (ch === quote)
|
|
11
|
+
quote = null;
|
|
12
|
+
else
|
|
13
|
+
current += ch;
|
|
14
|
+
}
|
|
15
|
+
else if (ch === '"' || ch === "'") {
|
|
16
|
+
quote = ch;
|
|
17
|
+
}
|
|
18
|
+
else if (/\s/.test(ch)) {
|
|
19
|
+
if (current) {
|
|
20
|
+
tokens.push(current);
|
|
21
|
+
current = "";
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
else {
|
|
25
|
+
current += ch;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
if (current)
|
|
29
|
+
tokens.push(current);
|
|
30
|
+
const [command, ...args] = tokens;
|
|
31
|
+
return { command: command ?? "", args };
|
|
32
|
+
}
|
|
33
|
+
const DEFAULT_SKIP_DIRS = new Set([
|
|
34
|
+
"node_modules",
|
|
35
|
+
".git",
|
|
36
|
+
"dist",
|
|
37
|
+
"build",
|
|
38
|
+
"out",
|
|
39
|
+
".next",
|
|
40
|
+
".venv",
|
|
41
|
+
"venv",
|
|
42
|
+
"__pycache__",
|
|
43
|
+
"coverage",
|
|
44
|
+
".pnpm",
|
|
45
|
+
]);
|
|
46
|
+
export function* walkFiles(root, opts = {}) {
|
|
47
|
+
const maxDepth = opts.maxDepth ?? 8;
|
|
48
|
+
const skip = opts.skipDirs ?? DEFAULT_SKIP_DIRS;
|
|
49
|
+
const stack = [{ dir: root, depth: 0 }];
|
|
50
|
+
while (stack.length) {
|
|
51
|
+
const { dir, depth } = stack.pop();
|
|
52
|
+
let entries;
|
|
53
|
+
try {
|
|
54
|
+
entries = fs.readdirSync(dir, { withFileTypes: true });
|
|
55
|
+
}
|
|
56
|
+
catch {
|
|
57
|
+
continue;
|
|
58
|
+
}
|
|
59
|
+
for (const e of entries) {
|
|
60
|
+
const full = path.join(dir, e.name);
|
|
61
|
+
if (e.isDirectory()) {
|
|
62
|
+
if (depth < maxDepth && !skip.has(e.name) && !e.name.startsWith(".")) {
|
|
63
|
+
stack.push({ dir: full, depth: depth + 1 });
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
else if (e.isFile()) {
|
|
67
|
+
yield full;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
export function readTextSafe(file, maxBytes = 512 * 1024) {
|
|
73
|
+
try {
|
|
74
|
+
const stat = fs.statSync(file);
|
|
75
|
+
if (stat.size > maxBytes) {
|
|
76
|
+
const fd = fs.openSync(file, "r");
|
|
77
|
+
const buf = Buffer.alloc(maxBytes);
|
|
78
|
+
fs.readSync(fd, buf, 0, maxBytes, 0);
|
|
79
|
+
fs.closeSync(fd);
|
|
80
|
+
return buf.toString("utf8");
|
|
81
|
+
}
|
|
82
|
+
return fs.readFileSync(file, "utf8");
|
|
83
|
+
}
|
|
84
|
+
catch {
|
|
85
|
+
return undefined;
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
const BINARY_EXTS = new Set([
|
|
89
|
+
".png", ".jpg", ".jpeg", ".gif", ".webp", ".ico", ".pdf", ".zip", ".gz",
|
|
90
|
+
".tar", ".woff", ".woff2", ".ttf", ".otf", ".eot", ".mp4", ".mp3", ".wav",
|
|
91
|
+
".exe", ".dll", ".so", ".dylib", ".wasm", ".pyc", ".jar", ".class",
|
|
92
|
+
]);
|
|
93
|
+
export function isBinaryPath(file) {
|
|
94
|
+
return BINARY_EXTS.has(path.extname(file).toLowerCase());
|
|
95
|
+
}
|
|
96
|
+
export function extractVersion(s) {
|
|
97
|
+
const m = s.match(/(\d+)\.(\d+)(?:\.(\d+))?/);
|
|
98
|
+
if (!m)
|
|
99
|
+
return undefined;
|
|
100
|
+
return [Number(m[1]), Number(m[2]), Number(m[3] ?? 0)];
|
|
101
|
+
}
|
|
102
|
+
export function truncate(s, n) {
|
|
103
|
+
return s.length <= n ? s : s.slice(0, Math.max(0, n - 3)) + "...";
|
|
104
|
+
}
|
|
105
|
+
export function formatTokens(n) {
|
|
106
|
+
if (n >= 10000)
|
|
107
|
+
return `${(n / 1000).toFixed(1)}k`;
|
|
108
|
+
return String(n);
|
|
109
|
+
}
|
|
110
|
+
export async function withTimeout(p, ms, what) {
|
|
111
|
+
let timer;
|
|
112
|
+
const timeout = new Promise((_, reject) => {
|
|
113
|
+
timer = setTimeout(() => reject(new Error(`timed out after ${ms}ms: ${what}`)), ms);
|
|
114
|
+
});
|
|
115
|
+
try {
|
|
116
|
+
return await Promise.race([p, timeout]);
|
|
117
|
+
}
|
|
118
|
+
finally {
|
|
119
|
+
clearTimeout(timer);
|
|
120
|
+
}
|
|
121
|
+
}
|
package/dist/version.js
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { ToolDef } from "../core/types.js";
|
|
2
|
+
/**
|
|
3
|
+
* Exact tool-definition cost for Claude models, measured as the delta of
|
|
4
|
+
* /v1/messages/count_tokens with and without the tools array (the endpoint
|
|
5
|
+
* accepts `tools`; verified against the official API reference 2026-07-16).
|
|
6
|
+
* Returns undefined on any failure; callers treat this as optional precision.
|
|
7
|
+
*/
|
|
8
|
+
export declare function countClaudeToolTokens(tools: ToolDef[], opts: {
|
|
9
|
+
apiKey: string;
|
|
10
|
+
model?: string;
|
|
11
|
+
}): Promise<number | undefined>;
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
const COUNT_TOKENS_URL = "https://api.anthropic.com/v1/messages/count_tokens";
|
|
2
|
+
const DEFAULT_MODEL = "claude-sonnet-5";
|
|
3
|
+
/**
|
|
4
|
+
* Exact tool-definition cost for Claude models, measured as the delta of
|
|
5
|
+
* /v1/messages/count_tokens with and without the tools array (the endpoint
|
|
6
|
+
* accepts `tools`; verified against the official API reference 2026-07-16).
|
|
7
|
+
* Returns undefined on any failure; callers treat this as optional precision.
|
|
8
|
+
*/
|
|
9
|
+
export async function countClaudeToolTokens(tools, opts) {
|
|
10
|
+
const model = opts.model ?? DEFAULT_MODEL;
|
|
11
|
+
const base = { model, messages: [{ role: "user", content: "hi" }] };
|
|
12
|
+
const apiTools = tools.map((t) => ({
|
|
13
|
+
name: t.name,
|
|
14
|
+
description: t.description ?? "",
|
|
15
|
+
input_schema: t.inputSchema ?? { type: "object", properties: {} },
|
|
16
|
+
}));
|
|
17
|
+
try {
|
|
18
|
+
const [withTools, withoutTools] = await Promise.all([
|
|
19
|
+
postCount({ ...base, tools: apiTools }, opts.apiKey),
|
|
20
|
+
postCount(base, opts.apiKey),
|
|
21
|
+
]);
|
|
22
|
+
if (withTools === undefined || withoutTools === undefined)
|
|
23
|
+
return undefined;
|
|
24
|
+
const delta = withTools - withoutTools;
|
|
25
|
+
return delta >= 0 ? delta : undefined;
|
|
26
|
+
}
|
|
27
|
+
catch {
|
|
28
|
+
return undefined;
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
async function postCount(body, apiKey) {
|
|
32
|
+
const res = await fetch(COUNT_TOKENS_URL, {
|
|
33
|
+
method: "POST",
|
|
34
|
+
headers: {
|
|
35
|
+
"content-type": "application/json",
|
|
36
|
+
"x-api-key": apiKey,
|
|
37
|
+
"anthropic-version": "2023-06-01",
|
|
38
|
+
},
|
|
39
|
+
body: JSON.stringify(body),
|
|
40
|
+
signal: AbortSignal.timeout(15000),
|
|
41
|
+
});
|
|
42
|
+
if (!res.ok)
|
|
43
|
+
return undefined;
|
|
44
|
+
const json = (await res.json());
|
|
45
|
+
return typeof json.input_tokens === "number" ? json.input_tokens : undefined;
|
|
46
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { ServerWeighResult } from "../core/types.js";
|
|
2
|
+
export interface WeighDiff {
|
|
3
|
+
before: number;
|
|
4
|
+
after: number;
|
|
5
|
+
delta: number;
|
|
6
|
+
pct: number;
|
|
7
|
+
toolChanges: {
|
|
8
|
+
name: string;
|
|
9
|
+
before?: number;
|
|
10
|
+
after?: number;
|
|
11
|
+
}[];
|
|
12
|
+
}
|
|
13
|
+
/** Primary budget metric is the claudeStyle total (a realistic host template). */
|
|
14
|
+
export declare function diffServerWeigh(baseline: ServerWeighResult, current: ServerWeighResult): WeighDiff;
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/** Primary budget metric is the claudeStyle total (a realistic host template). */
|
|
2
|
+
export function diffServerWeigh(baseline, current) {
|
|
3
|
+
const before = baseline.totals.claudeStyle;
|
|
4
|
+
const after = current.totals.claudeStyle;
|
|
5
|
+
const delta = after - before;
|
|
6
|
+
const pct = before > 0 ? (delta / before) * 100 : after > 0 ? 100 : 0;
|
|
7
|
+
const beforeMap = new Map(baseline.perTool.map((t) => [t.name, t.tokens.claudeStyle]));
|
|
8
|
+
const afterMap = new Map(current.perTool.map((t) => [t.name, t.tokens.claudeStyle]));
|
|
9
|
+
const names = new Set([...beforeMap.keys(), ...afterMap.keys()]);
|
|
10
|
+
const toolChanges = [];
|
|
11
|
+
for (const name of names) {
|
|
12
|
+
const b = beforeMap.get(name);
|
|
13
|
+
const a = afterMap.get(name);
|
|
14
|
+
if (b !== a)
|
|
15
|
+
toolChanges.push({ name, before: b, after: a });
|
|
16
|
+
}
|
|
17
|
+
toolChanges.sort((x, y) => Math.abs((y.after ?? 0) - (y.before ?? 0)) - Math.abs((x.after ?? 0) - (x.before ?? 0)));
|
|
18
|
+
return { before, after, delta, pct, toolChanges };
|
|
19
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import type { ToolDef } from "../core/types.js";
|
|
2
|
+
/**
|
|
3
|
+
* Three documented serializations of tool definitions (docs/METHODOLOGY.md):
|
|
4
|
+
* - rawJson: the tools/list payload, minified. A neutral lower-bound proxy.
|
|
5
|
+
* - claudeStyle: one JSON object per tool with description/name/parameters,
|
|
6
|
+
* wrapped in a <functions> block, mirroring how Claude-family harnesses
|
|
7
|
+
* present tools in the system prompt.
|
|
8
|
+
* - openaiTools: the Chat Completions `tools` array shape.
|
|
9
|
+
* Hosts add their own fixed framing text around these; that overhead is
|
|
10
|
+
* per-host constant and excluded on purpose.
|
|
11
|
+
*/
|
|
12
|
+
export declare function toolToClaudeLine(tool: ToolDef): string;
|
|
13
|
+
export declare function serializeClaudeStyle(tools: ToolDef[]): string;
|
|
14
|
+
export declare function toolToOpenAIEntry(tool: ToolDef): object;
|
|
15
|
+
export declare function serializeOpenAITools(tools: ToolDef[]): string;
|
|
16
|
+
export declare function serializeRawJson(tools: ToolDef[]): string;
|
|
17
|
+
export declare function serializeSingle(tool: ToolDef): {
|
|
18
|
+
rawJson: string;
|
|
19
|
+
claudeStyle: string;
|
|
20
|
+
openaiTools: string;
|
|
21
|
+
};
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Three documented serializations of tool definitions (docs/METHODOLOGY.md):
|
|
3
|
+
* - rawJson: the tools/list payload, minified. A neutral lower-bound proxy.
|
|
4
|
+
* - claudeStyle: one JSON object per tool with description/name/parameters,
|
|
5
|
+
* wrapped in a <functions> block, mirroring how Claude-family harnesses
|
|
6
|
+
* present tools in the system prompt.
|
|
7
|
+
* - openaiTools: the Chat Completions `tools` array shape.
|
|
8
|
+
* Hosts add their own fixed framing text around these; that overhead is
|
|
9
|
+
* per-host constant and excluded on purpose.
|
|
10
|
+
*/
|
|
11
|
+
export function toolToClaudeLine(tool) {
|
|
12
|
+
return JSON.stringify({
|
|
13
|
+
description: tool.description ?? "",
|
|
14
|
+
name: tool.name,
|
|
15
|
+
parameters: tool.inputSchema ?? { type: "object", properties: {} },
|
|
16
|
+
});
|
|
17
|
+
}
|
|
18
|
+
export function serializeClaudeStyle(tools) {
|
|
19
|
+
const lines = tools.map((t) => `<function>${toolToClaudeLine(t)}</function>`);
|
|
20
|
+
return `<functions>\n${lines.join("\n")}\n</functions>`;
|
|
21
|
+
}
|
|
22
|
+
export function toolToOpenAIEntry(tool) {
|
|
23
|
+
return {
|
|
24
|
+
type: "function",
|
|
25
|
+
function: {
|
|
26
|
+
name: tool.name,
|
|
27
|
+
description: tool.description ?? "",
|
|
28
|
+
parameters: tool.inputSchema ?? { type: "object", properties: {} },
|
|
29
|
+
},
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
export function serializeOpenAITools(tools) {
|
|
33
|
+
return JSON.stringify(tools.map(toolToOpenAIEntry));
|
|
34
|
+
}
|
|
35
|
+
export function serializeRawJson(tools) {
|
|
36
|
+
return JSON.stringify(tools);
|
|
37
|
+
}
|
|
38
|
+
export function serializeSingle(tool) {
|
|
39
|
+
return {
|
|
40
|
+
rawJson: JSON.stringify(tool),
|
|
41
|
+
claudeStyle: toolToClaudeLine(tool),
|
|
42
|
+
openaiTools: JSON.stringify(toolToOpenAIEntry(tool)),
|
|
43
|
+
};
|
|
44
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Token counts use OpenAI's o200k_base as the canonical tokenizer: it is
|
|
3
|
+
* publicly reproducible, and vendor tokenizers track each other closely on
|
|
4
|
+
* English/JSON text. Anthropic-exact counts are available via the
|
|
5
|
+
* count_tokens API when a key is provided. See docs/METHODOLOGY.md.
|
|
6
|
+
*
|
|
7
|
+
* Load the tokenizer once, then count synchronously. Weighing tokenizes many
|
|
8
|
+
* strings in a loop; awaiting each one adds a microtask hop per string for no
|
|
9
|
+
* benefit, since the underlying encode is synchronous.
|
|
10
|
+
*/
|
|
11
|
+
export declare function loadTokenizer(): Promise<(text: string) => number>;
|
|
12
|
+
export declare function countTokens(text: string): Promise<number>;
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
let encodeFn = null;
|
|
2
|
+
/**
|
|
3
|
+
* Token counts use OpenAI's o200k_base as the canonical tokenizer: it is
|
|
4
|
+
* publicly reproducible, and vendor tokenizers track each other closely on
|
|
5
|
+
* English/JSON text. Anthropic-exact counts are available via the
|
|
6
|
+
* count_tokens API when a key is provided. See docs/METHODOLOGY.md.
|
|
7
|
+
*
|
|
8
|
+
* Load the tokenizer once, then count synchronously. Weighing tokenizes many
|
|
9
|
+
* strings in a loop; awaiting each one adds a microtask hop per string for no
|
|
10
|
+
* benefit, since the underlying encode is synchronous.
|
|
11
|
+
*/
|
|
12
|
+
export async function loadTokenizer() {
|
|
13
|
+
if (!encodeFn) {
|
|
14
|
+
const mod = await import("gpt-tokenizer/encoding/o200k_base");
|
|
15
|
+
encodeFn = mod.encode;
|
|
16
|
+
}
|
|
17
|
+
const enc = encodeFn;
|
|
18
|
+
return (text) => enc(text).length;
|
|
19
|
+
}
|
|
20
|
+
export async function countTokens(text) {
|
|
21
|
+
const count = await loadTokenizer();
|
|
22
|
+
return count(text);
|
|
23
|
+
}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import type { ServerIntrospection, ServerWeighResult, SkillSet, SkillSetWeighResult } from "../core/types.js";
|
|
2
|
+
export declare function weighServer(intro: ServerIntrospection, opts?: {
|
|
3
|
+
anthropicApiKey?: string;
|
|
4
|
+
anthropicModel?: string;
|
|
5
|
+
}): Promise<ServerWeighResult>;
|
|
6
|
+
export declare function weighSkills(set: SkillSet): Promise<SkillSetWeighResult>;
|