@polygraphso/litmus 0.8.1 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +54 -0
- package/dist/{chunk-ZR6XRGMQ.js → chunk-44R4ZYOE.js} +67 -0
- package/dist/chunk-AVF3GYCS.js +692 -0
- package/dist/{chunk-35UOPCBW.js → chunk-DN2OX4RT.js} +456 -2
- package/dist/{chunk-VOPISHBU.js → chunk-M5HXKZVN.js} +2 -2
- package/dist/cli-skill.d.ts +1 -0
- package/dist/cli-skill.js +98 -0
- package/dist/cli.js +2 -2
- package/dist/index.d.ts +437 -2
- package/dist/index.js +86 -8
- package/dist/mcp.js +130 -122
- package/dist/src-TG44QXFV.js +67 -0
- package/package.json +5 -4
- package/dist/chunk-BPS4YCDL.js +0 -250
- package/dist/src-RSTPCEYU.js +0 -31
|
@@ -3,7 +3,7 @@ import {
|
|
|
3
3
|
METHODOLOGY_VERSION,
|
|
4
4
|
parseServerRef,
|
|
5
5
|
serverKey
|
|
6
|
-
} from "./chunk-
|
|
6
|
+
} from "./chunk-44R4ZYOE.js";
|
|
7
7
|
|
|
8
8
|
// ../probes/src/harness.ts
|
|
9
9
|
import { execFile as execFile3 } from "child_process";
|
|
@@ -2157,6 +2157,442 @@ function checkDocker() {
|
|
|
2157
2157
|
});
|
|
2158
2158
|
}
|
|
2159
2159
|
|
|
2160
|
+
// ../probes/src/skills/load-skill.ts
|
|
2161
|
+
import { readFileSync, readdirSync, statSync } from "fs";
|
|
2162
|
+
import { join as join3, relative, sep } from "path";
|
|
2163
|
+
import { createHash as createHash2 } from "crypto";
|
|
2164
|
+
var SkillLoadError = class extends Error {
|
|
2165
|
+
};
|
|
2166
|
+
var MAX_FILES = 4096;
|
|
2167
|
+
var EXEC_EXT = /\.(?:sh|bash|zsh|py|js|mjs|cjs|ts|rb|pl|php)$/i;
|
|
2168
|
+
function sha256hex(buf) {
|
|
2169
|
+
return createHash2("sha256").update(buf).digest("hex");
|
|
2170
|
+
}
|
|
2171
|
+
function looksExecutable(relPath, bytes) {
|
|
2172
|
+
if (EXEC_EXT.test(relPath)) return true;
|
|
2173
|
+
return bytes.subarray(0, 2).toString("latin1") === "#!";
|
|
2174
|
+
}
|
|
2175
|
+
function enumerateFiles(dir) {
|
|
2176
|
+
const out = [];
|
|
2177
|
+
const walk = (d) => {
|
|
2178
|
+
let entries;
|
|
2179
|
+
try {
|
|
2180
|
+
entries = readdirSync(d);
|
|
2181
|
+
} catch {
|
|
2182
|
+
return;
|
|
2183
|
+
}
|
|
2184
|
+
for (const name of entries) {
|
|
2185
|
+
if (name === "node_modules" || name === ".git") continue;
|
|
2186
|
+
const p = join3(d, name);
|
|
2187
|
+
let st;
|
|
2188
|
+
try {
|
|
2189
|
+
st = statSync(p);
|
|
2190
|
+
} catch {
|
|
2191
|
+
continue;
|
|
2192
|
+
}
|
|
2193
|
+
if (st.isDirectory()) walk(p);
|
|
2194
|
+
else if (st.isFile()) out.push(relative(dir, p).split(sep).join("/").normalize("NFC"));
|
|
2195
|
+
}
|
|
2196
|
+
};
|
|
2197
|
+
walk(dir);
|
|
2198
|
+
return out.sort();
|
|
2199
|
+
}
|
|
2200
|
+
function splitFrontmatter(src) {
|
|
2201
|
+
if (!src.startsWith("---")) return { frontmatter: "", body: src };
|
|
2202
|
+
const firstNL = src.indexOf("\n");
|
|
2203
|
+
if (firstNL < 0) return { frontmatter: "", body: src };
|
|
2204
|
+
const end = src.indexOf("\n---", firstNL);
|
|
2205
|
+
if (end < 0) return { frontmatter: "", body: src };
|
|
2206
|
+
const close = src.indexOf("\n", end + 1);
|
|
2207
|
+
return {
|
|
2208
|
+
frontmatter: src.slice(firstNL + 1, end),
|
|
2209
|
+
body: close < 0 ? "" : src.slice(close + 1)
|
|
2210
|
+
};
|
|
2211
|
+
}
|
|
2212
|
+
function extractDescription(frontmatter) {
|
|
2213
|
+
const lines = frontmatter.split("\n");
|
|
2214
|
+
for (let i = 0; i < lines.length; i++) {
|
|
2215
|
+
const m = /^description\s*:\s*(.*)$/i.exec(lines[i]);
|
|
2216
|
+
if (!m) continue;
|
|
2217
|
+
const v = m[1].trim();
|
|
2218
|
+
if (/^[>|][+-]?$/.test(v)) {
|
|
2219
|
+
const collected = [];
|
|
2220
|
+
for (let j = i + 1; j < lines.length; j++) {
|
|
2221
|
+
const line = lines[j];
|
|
2222
|
+
if (line.trim() === "") continue;
|
|
2223
|
+
if (/^\s/.test(line)) collected.push(line.trim());
|
|
2224
|
+
else break;
|
|
2225
|
+
}
|
|
2226
|
+
return collected.join(" ");
|
|
2227
|
+
}
|
|
2228
|
+
return v.replace(/^['"]|['"]$/g, "");
|
|
2229
|
+
}
|
|
2230
|
+
return "";
|
|
2231
|
+
}
|
|
2232
|
+
function loadSkill(dir) {
|
|
2233
|
+
const relPaths = enumerateFiles(dir);
|
|
2234
|
+
if (relPaths.length > MAX_FILES) relPaths.length = MAX_FILES;
|
|
2235
|
+
const skillMdRel = relPaths.find((p) => p.toLowerCase() === "skill.md");
|
|
2236
|
+
if (!skillMdRel) throw new SkillLoadError(`no SKILL.md in ${dir}`);
|
|
2237
|
+
const files = [];
|
|
2238
|
+
for (const relPath of relPaths) {
|
|
2239
|
+
let bytes;
|
|
2240
|
+
try {
|
|
2241
|
+
bytes = readFileSync(join3(dir, relPath));
|
|
2242
|
+
} catch {
|
|
2243
|
+
continue;
|
|
2244
|
+
}
|
|
2245
|
+
files.push({ relPath, bytes, isExecutable: looksExecutable(relPath, bytes) });
|
|
2246
|
+
}
|
|
2247
|
+
const manifest = files.map((f) => `${f.relPath}\0${sha256hex(f.bytes)}`).join("\n");
|
|
2248
|
+
const contentHash = "0x" + sha256hex(manifest);
|
|
2249
|
+
const src = files.find((f) => f.relPath === skillMdRel).bytes.toString("utf8");
|
|
2250
|
+
const { frontmatter, body } = splitFrontmatter(src);
|
|
2251
|
+
return {
|
|
2252
|
+
dir,
|
|
2253
|
+
frontmatter,
|
|
2254
|
+
description: extractDescription(frontmatter),
|
|
2255
|
+
body,
|
|
2256
|
+
files,
|
|
2257
|
+
contentHash
|
|
2258
|
+
};
|
|
2259
|
+
}
|
|
2260
|
+
|
|
2261
|
+
// ../probes/src/skills/scanners-skill.ts
|
|
2262
|
+
function stripExamples(md) {
|
|
2263
|
+
return md.replace(/```[\s\S]*?```/g, " ").replace(/~~~[\s\S]*?~~~/g, " ").replace(/`[^`\n]*`/g, " ").split("\n").filter((line) => !/^\s*>/.test(line)).join("\n");
|
|
2264
|
+
}
|
|
2265
|
+
function isBareSystemColon(f) {
|
|
2266
|
+
return f.kind === "instruction-mimicry" && /(?:^|[\s>])system\s*:/i.test(f.match) && !f.match.includes("<");
|
|
2267
|
+
}
|
|
2268
|
+
function skillInjection(body) {
|
|
2269
|
+
const text = stripExamples(body);
|
|
2270
|
+
return [
|
|
2271
|
+
...invisibleUnicode(text),
|
|
2272
|
+
...instructionMimicry(text).filter((f) => !isBareSystemColon(f)),
|
|
2273
|
+
...markdownTricks(text)
|
|
2274
|
+
];
|
|
2275
|
+
}
|
|
2276
|
+
function skillInjectionFails(findings) {
|
|
2277
|
+
return hasHighSeverity(findings);
|
|
2278
|
+
}
|
|
2279
|
+
var TRANSMIT_VERB = /\b(?:send|post|put|upload|exfiltrate|transmit|e-?mail|forward|leak|publish|curl|wget|fetch)\b/i;
|
|
2280
|
+
var SECRET_NOUN = /\b(?:credentials?|secrets?|tokens?|api[\s_-]?keys?|access[\s_-]?keys?|passwords?|passwd|private[\s_-]?keys?|ssh[\s_-]?keys?|\.env\b|env(?:ironment)?\s+variables?|aws[\s_-]?(?:secret|access)|bearer\s+token)\b/i;
|
|
2281
|
+
var SINK = /(?:https?:\/\/\S+|\bto\s+(?:a\s+|an\s+|the\s+|your\s+|our\s+)?(?:remote|external|attacker|third[\s-]?party)?\s*(?:server|endpoint|host|url|webhook|domain))/i;
|
|
2282
|
+
function exfilInstruction(text) {
|
|
2283
|
+
const findings = [];
|
|
2284
|
+
const stripped = stripExamples(text);
|
|
2285
|
+
for (const raw of stripped.split(/(?<=[.!?\n])/)) {
|
|
2286
|
+
const sentence = raw.trim();
|
|
2287
|
+
if (!sentence) continue;
|
|
2288
|
+
if (TRANSMIT_VERB.test(sentence) && SECRET_NOUN.test(sentence) && SINK.test(sentence)) {
|
|
2289
|
+
findings.push({ kind: "exfil-instruction", severity: "high", match: sentence.slice(0, 160) });
|
|
2290
|
+
}
|
|
2291
|
+
}
|
|
2292
|
+
return findings;
|
|
2293
|
+
}
|
|
2294
|
+
var DANGEROUS = [
|
|
2295
|
+
// pipe a network fetch straight into a shell — the classic remote-exec.
|
|
2296
|
+
{ re: /\b(?:curl|wget|fetch)\b[^\n|]*\|\s*(?:sudo\s+)?(?:ba)?sh\b/i, severity: "high" },
|
|
2297
|
+
// base64/hex decode piped into a shell or eval'd.
|
|
2298
|
+
{ re: /\bbase64\s+(?:--decode|-d|-D)\b[^\n|]*\|\s*(?:ba)?sh\b/i, severity: "high" },
|
|
2299
|
+
// reverse shells.
|
|
2300
|
+
{ re: /\b(?:bash|sh)\s+-i\b[^\n]*(?:>&|\d>&)/i, severity: "high" },
|
|
2301
|
+
{ re: /\/dev\/tcp\/[^\s/]+\/\d+/i, severity: "high" },
|
|
2302
|
+
{ re: /\bn(?:et)?cat?\b[^\n]*\s-e\b/i, severity: "high" },
|
|
2303
|
+
// lower-confidence: dynamic exec of strings / blanket destructive fs — MEDIUM,
|
|
2304
|
+
// recorded but does not floor the letter on its own.
|
|
2305
|
+
{ re: /\beval\s*\(/i, severity: "medium" },
|
|
2306
|
+
{ re: /\bsubprocess\.[A-Za-z]+\([^)]*shell\s*=\s*True/i, severity: "medium" },
|
|
2307
|
+
{ re: /\bos\.system\s*\(/i, severity: "medium" },
|
|
2308
|
+
{ re: /\brm\s+-rf\s+(?:\/|~|\$)/i, severity: "medium" }
|
|
2309
|
+
];
|
|
2310
|
+
function dangerousCommand(text, file) {
|
|
2311
|
+
const findings = [];
|
|
2312
|
+
const scan = (s, label) => {
|
|
2313
|
+
for (const { re, severity } of DANGEROUS) {
|
|
2314
|
+
const m = re.exec(s);
|
|
2315
|
+
if (m) {
|
|
2316
|
+
findings.push({
|
|
2317
|
+
kind: "dangerous-command",
|
|
2318
|
+
severity,
|
|
2319
|
+
match: (label ? `${label}: ` : "") + m[0].slice(0, 120),
|
|
2320
|
+
offset: m.index,
|
|
2321
|
+
...file ? { file } : {}
|
|
2322
|
+
});
|
|
2323
|
+
}
|
|
2324
|
+
}
|
|
2325
|
+
};
|
|
2326
|
+
scan(text);
|
|
2327
|
+
for (const m of text.matchAll(/[A-Za-z0-9+/]{16,}={0,2}/g)) {
|
|
2328
|
+
const d = decode(m[0], "base64");
|
|
2329
|
+
if (d && /\|\s*(?:ba)?sh\b|\/dev\/tcp\//i.test(d)) scan(d, "base64-decoded");
|
|
2330
|
+
}
|
|
2331
|
+
return findings;
|
|
2332
|
+
}
|
|
2333
|
+
function decode(s, enc) {
|
|
2334
|
+
try {
|
|
2335
|
+
const d = Buffer.from(s, enc).toString("utf8");
|
|
2336
|
+
return /[\x20-\x7e]/.test(d) ? d : null;
|
|
2337
|
+
} catch {
|
|
2338
|
+
return null;
|
|
2339
|
+
}
|
|
2340
|
+
}
|
|
2341
|
+
var OVER_BROAD = /\b(?:always|every\s+(?:file|request|time|message|prompt)|all\s+(?:requests|files|prompts|messages)|regardless\s+of|no\s+matter\s+what)\b/i;
|
|
2342
|
+
function overBroadTrigger(description) {
|
|
2343
|
+
const m = OVER_BROAD.exec(description);
|
|
2344
|
+
return m ? [{ kind: "over-broad-trigger", severity: "low", match: m[0], offset: m.index }] : [];
|
|
2345
|
+
}
|
|
2346
|
+
|
|
2347
|
+
// ../probes/src/skills/grade-skill.ts
|
|
2348
|
+
var DISQUALIFYING = /* @__PURE__ */ new Set(["S-01", "S-03"]);
|
|
2349
|
+
var CAPPING = /* @__PURE__ */ new Set(["S-04", "S-05"]);
|
|
2350
|
+
function gradeSkillCategories(categories) {
|
|
2351
|
+
const byCode = (code) => categories.find((c) => c.code === code);
|
|
2352
|
+
const s01 = byCode("S-01");
|
|
2353
|
+
const failed = categories.filter((c) => c.status === "fail").map((c) => c.code);
|
|
2354
|
+
const skipped = categories.filter((c) => c.status === "skipped").map((c) => c.code);
|
|
2355
|
+
if (failed.some((c) => DISQUALIFYING.has(c))) {
|
|
2356
|
+
const which = failed.filter((c) => DISQUALIFYING.has(c)).join(", ");
|
|
2357
|
+
return {
|
|
2358
|
+
grade: "F",
|
|
2359
|
+
rationale: `Disqualifying failure in ${which} \u2014 the skill instructs prompt injection or data exfiltration into an agent that loads it.`
|
|
2360
|
+
};
|
|
2361
|
+
}
|
|
2362
|
+
if (failed.some((c) => CAPPING.has(c))) {
|
|
2363
|
+
const which = failed.filter((c) => CAPPING.has(c)).join(", ");
|
|
2364
|
+
return {
|
|
2365
|
+
grade: "D",
|
|
2366
|
+
rationale: `Overreach in ${which} \u2014 a dangerous bundled command or an undeclared capability. No injection or exfil instruction, so the grade caps at D.`
|
|
2367
|
+
};
|
|
2368
|
+
}
|
|
2369
|
+
if (s01?.status !== "pass") {
|
|
2370
|
+
return {
|
|
2371
|
+
grade: "F",
|
|
2372
|
+
rationale: "S-01 did not complete \u2014 the SKILL.md could not be parsed/scanned, so the skill is treated as ungraded/unsafe."
|
|
2373
|
+
};
|
|
2374
|
+
}
|
|
2375
|
+
const allPass = categories.every((c) => c.status === "pass");
|
|
2376
|
+
if (allPass) {
|
|
2377
|
+
return {
|
|
2378
|
+
grade: "A",
|
|
2379
|
+
rationale: "All skill categories passed: no injection or exfil instruction in the body, and no dangerous bundled command or undeclared capability. A reflects static scanning, not behavioral proof."
|
|
2380
|
+
};
|
|
2381
|
+
}
|
|
2382
|
+
const note = skipped.length ? ` Not verified: ${skipped.join(", ")} (${skipped.map((c) => byCode(c)?.reason).filter(Boolean).join("; ")}).` : "";
|
|
2383
|
+
return {
|
|
2384
|
+
grade: "B",
|
|
2385
|
+
rationale: `Injection and exfil checks passed; some categories not verified.${note}`
|
|
2386
|
+
};
|
|
2387
|
+
}
|
|
2388
|
+
|
|
2389
|
+
// ../probes/src/skills/skill-harness.ts
|
|
2390
|
+
var SKILL_METHODOLOGY_VERSION = "litmus-skill-v1";
|
|
2391
|
+
var SKILL_BUNDLE_SCHEMA_VERSION = "0.1.0";
|
|
2392
|
+
var DISCLAIMER2 = "litmus-skill-v1 is a deterministic STATIC scan of the skill's text and bundled files. It is not behavioral proof: a skill's instructions are interpreted by an agent at runtime, bundled scripts are not executed in this version, and a command constructed or fetched at runtime is not detectable by static scanning. An A means the static checks found no injection, exfil instruction, or dangerous bundled command \u2014 not that the skill is safe to run unsupervised.";
|
|
2393
|
+
function cat(code, status, findings, reason) {
|
|
2394
|
+
return { code, status, findings, ...reason ? { reason } : {} };
|
|
2395
|
+
}
|
|
2396
|
+
function runSkillLitmus(dir, opts = {}) {
|
|
2397
|
+
const ranAt = opts.ranAt ?? (/* @__PURE__ */ new Date()).toISOString();
|
|
2398
|
+
const harness = { package: "@polygraph/probes", version: opts.harnessVersion ?? SKILL_METHODOLOGY_VERSION, node: process.version };
|
|
2399
|
+
const base = { schemaVersion: SKILL_BUNDLE_SCHEMA_VERSION, methodologyVersion: SKILL_METHODOLOGY_VERSION, ranAt, harness, disclaimer: DISCLAIMER2 };
|
|
2400
|
+
let loaded;
|
|
2401
|
+
try {
|
|
2402
|
+
loaded = loadSkill(dir);
|
|
2403
|
+
} catch (e) {
|
|
2404
|
+
const reason = e instanceof SkillLoadError ? e.message : "failed to load skill";
|
|
2405
|
+
const categories2 = [cat("S-01", "skipped", [], reason)];
|
|
2406
|
+
const { grade: grade2, rationale: rationale2 } = gradeSkillCategories(categories2);
|
|
2407
|
+
return { ...base, skillRef: opts.skillRef ?? dir, contentHash: "0x", categories: categories2, advisories: [], grade: grade2, gradeRationale: rationale2 };
|
|
2408
|
+
}
|
|
2409
|
+
const injFindings = [...skillInjection(loaded.body), ...skillInjection(loaded.frontmatter)];
|
|
2410
|
+
const s01 = cat("S-01", skillInjectionFails(injFindings) ? "fail" : "pass", injFindings);
|
|
2411
|
+
const exfil = exfilInstruction(loaded.body);
|
|
2412
|
+
const s03 = cat("S-03", exfil.some((f) => f.severity === "high") ? "fail" : "pass", exfil);
|
|
2413
|
+
const execFiles = loaded.files.filter((f) => f.isExecutable);
|
|
2414
|
+
const dangFindings = [];
|
|
2415
|
+
for (const f of execFiles) dangFindings.push(...dangerousCommand(f.bytes.toString("utf8"), f.relPath));
|
|
2416
|
+
const dangHigh = dangFindings.filter((f) => f.severity === "high");
|
|
2417
|
+
const s04 = cat(
|
|
2418
|
+
"S-04",
|
|
2419
|
+
dangHigh.length > 0 ? "fail" : "pass",
|
|
2420
|
+
dangHigh,
|
|
2421
|
+
execFiles.length === 0 ? "no bundled executable scripts" : void 0
|
|
2422
|
+
);
|
|
2423
|
+
const categories = [s01, s03, s04];
|
|
2424
|
+
const { grade, rationale } = gradeSkillCategories(categories);
|
|
2425
|
+
const advisories = [
|
|
2426
|
+
...overBroadTrigger(loaded.description),
|
|
2427
|
+
...dangFindings.filter((f) => f.severity !== "high")
|
|
2428
|
+
];
|
|
2429
|
+
return {
|
|
2430
|
+
...base,
|
|
2431
|
+
skillRef: opts.skillRef ?? dir,
|
|
2432
|
+
contentHash: loaded.contentHash,
|
|
2433
|
+
categories,
|
|
2434
|
+
advisories,
|
|
2435
|
+
grade,
|
|
2436
|
+
gradeRationale: rationale
|
|
2437
|
+
};
|
|
2438
|
+
}
|
|
2439
|
+
|
|
2440
|
+
// ../probes/src/skills/quality-judge.ts
|
|
2441
|
+
function openAICompatJudge(cfg) {
|
|
2442
|
+
const url = `${cfg.baseUrl.replace(/\/+$/, "")}/chat/completions`;
|
|
2443
|
+
return {
|
|
2444
|
+
id: `openai-compat:${cfg.model}`,
|
|
2445
|
+
async complete(system, user) {
|
|
2446
|
+
const res = await fetch(url, {
|
|
2447
|
+
method: "POST",
|
|
2448
|
+
headers: { "content-type": "application/json", authorization: `Bearer ${cfg.apiKey}` },
|
|
2449
|
+
body: JSON.stringify({
|
|
2450
|
+
model: cfg.model,
|
|
2451
|
+
max_tokens: 1024,
|
|
2452
|
+
messages: [
|
|
2453
|
+
{ role: "system", content: system },
|
|
2454
|
+
{ role: "user", content: user }
|
|
2455
|
+
]
|
|
2456
|
+
})
|
|
2457
|
+
});
|
|
2458
|
+
if (!res.ok) throw new Error(`judge request failed: HTTP ${res.status}`);
|
|
2459
|
+
const data = await res.json();
|
|
2460
|
+
return data?.choices?.[0]?.message?.content ?? "";
|
|
2461
|
+
}
|
|
2462
|
+
};
|
|
2463
|
+
}
|
|
2464
|
+
function judgeFromEnv(env = process.env) {
|
|
2465
|
+
const apiKey = env.LITMUS_LLM_API_KEY;
|
|
2466
|
+
const model = env.LITMUS_LLM_MODEL;
|
|
2467
|
+
if (!apiKey || !model) return null;
|
|
2468
|
+
return openAICompatJudge({ apiKey, model, baseUrl: env.LITMUS_LLM_BASE_URL ?? "https://api.openai.com/v1" });
|
|
2469
|
+
}
|
|
2470
|
+
var AXES = ["honesty", "coherence"];
|
|
2471
|
+
var RATINGS = ["good", "concern", "bad"];
|
|
2472
|
+
var SYSTEM = 'You are a strict, terse reviewer of Claude Code "skills" (instruction files an AI agent loads). Judge ONLY two axes. honesty: does the body do anything material its frontmatter `description` does not disclose, or claim a far broader/narrower scope than it delivers? coherence: are the instructions clear, internally consistent, and actually followable? Rate each good|concern|bad. Reply with ONLY a JSON object: {"honesty":{"rating":"...","why":"<=20 words"},"coherence":{"rating":"...","why":"<=20 words"}}. No prose.';
|
|
2473
|
+
function buildUserPrompt(loaded) {
|
|
2474
|
+
const body = loaded.body.length > 12e3 ? loaded.body.slice(0, 12e3) + "\n\u2026[truncated]" : loaded.body;
|
|
2475
|
+
return `description: ${loaded.description || "(none)"}
|
|
2476
|
+
|
|
2477
|
+
--- SKILL BODY ---
|
|
2478
|
+
${body}`;
|
|
2479
|
+
}
|
|
2480
|
+
function parseVerdict(text) {
|
|
2481
|
+
const start = text.indexOf("{");
|
|
2482
|
+
const end = text.lastIndexOf("}");
|
|
2483
|
+
if (start < 0 || end <= start) return null;
|
|
2484
|
+
let obj;
|
|
2485
|
+
try {
|
|
2486
|
+
obj = JSON.parse(text.slice(start, end + 1));
|
|
2487
|
+
} catch {
|
|
2488
|
+
return null;
|
|
2489
|
+
}
|
|
2490
|
+
const out = {};
|
|
2491
|
+
for (const axis of AXES) {
|
|
2492
|
+
const r = obj?.[axis]?.rating;
|
|
2493
|
+
if (typeof r !== "string" || !RATINGS.includes(r)) return null;
|
|
2494
|
+
out[axis] = r;
|
|
2495
|
+
}
|
|
2496
|
+
return out;
|
|
2497
|
+
}
|
|
2498
|
+
function majority(ratings) {
|
|
2499
|
+
const tally = /* @__PURE__ */ new Map();
|
|
2500
|
+
for (const r of ratings) tally.set(r, (tally.get(r) ?? 0) + 1);
|
|
2501
|
+
let best = "good";
|
|
2502
|
+
let bestN = -1;
|
|
2503
|
+
for (const r of RATINGS) {
|
|
2504
|
+
const n = tally.get(r) ?? 0;
|
|
2505
|
+
if (n > bestN || n === bestN && RATINGS.indexOf(r) > RATINGS.indexOf(best)) {
|
|
2506
|
+
best = r;
|
|
2507
|
+
bestN = n;
|
|
2508
|
+
}
|
|
2509
|
+
}
|
|
2510
|
+
return { rating: best, count: bestN };
|
|
2511
|
+
}
|
|
2512
|
+
async function judgeSkillQuality(loaded, judge, opts = {}) {
|
|
2513
|
+
const samples = Math.max(1, Math.min(opts.samples ?? 1, 5));
|
|
2514
|
+
const user = buildUserPrompt(loaded);
|
|
2515
|
+
const verdicts = [];
|
|
2516
|
+
for (let i = 0; i < samples; i++) {
|
|
2517
|
+
const v = parseVerdict(await judge.complete(SYSTEM, user));
|
|
2518
|
+
if (v) verdicts.push(v);
|
|
2519
|
+
}
|
|
2520
|
+
if (verdicts.length === 0) throw new Error("judge returned no parseable verdict");
|
|
2521
|
+
let minAgreement = 1;
|
|
2522
|
+
const axes = AXES.map((axis) => {
|
|
2523
|
+
const m = majority(verdicts.map((v) => v[axis]));
|
|
2524
|
+
minAgreement = Math.min(minAgreement, m.count / verdicts.length);
|
|
2525
|
+
return { axis, rating: m.rating, rationale: `majority of ${verdicts.length} sample(s)` };
|
|
2526
|
+
});
|
|
2527
|
+
return {
|
|
2528
|
+
judge: judge.id,
|
|
2529
|
+
samples: verdicts.length,
|
|
2530
|
+
agreement: Number(minAgreement.toFixed(2)),
|
|
2531
|
+
axes,
|
|
2532
|
+
note: "Advisory, non-deterministic: produced by an LLM judge, not the reproducible static scan. Repeatability is majority-over-k, not bit-identical. Never affects the safety letter and is never minted."
|
|
2533
|
+
};
|
|
2534
|
+
}
|
|
2535
|
+
|
|
2536
|
+
// ../probes/src/skills/quality.ts
|
|
2537
|
+
var SKILL_QUALITY_VERSION = "skill-quality-v1";
|
|
2538
|
+
var QUALITY_DISCLAIMER = "skill-quality-v1 is an ADVISORY signal, separate from the safety grade. It is never an A\u2013F letter and is never minted on-chain. This version runs only the deterministic well-formedness checks; the non-deterministic, LLM-judged axes (outcome fidelity, trigger calibration) are not included, so it does not assert that the skill actually works.";
|
|
2539
|
+
function brokenBundleLinks(body, relPaths) {
|
|
2540
|
+
const broken = [];
|
|
2541
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2542
|
+
for (const m of body.matchAll(/!?\[[^\]]*\]\(([^)\s]+)/g)) {
|
|
2543
|
+
let ref = m[1].trim();
|
|
2544
|
+
if (/^(?:https?:|mailto:|tel:|data:|#)/i.test(ref) || ref.startsWith("/")) continue;
|
|
2545
|
+
ref = ref.replace(/^\.\//, "").split("#")[0].split("?")[0].normalize("NFC");
|
|
2546
|
+
if (!ref || seen.has(ref)) continue;
|
|
2547
|
+
seen.add(ref);
|
|
2548
|
+
if (!relPaths.has(ref)) broken.push(ref);
|
|
2549
|
+
}
|
|
2550
|
+
return broken;
|
|
2551
|
+
}
|
|
2552
|
+
function runSkillQuality(dir, opts = {}) {
|
|
2553
|
+
const ranAt = opts.ranAt ?? (/* @__PURE__ */ new Date()).toISOString();
|
|
2554
|
+
const base = { qualityVersion: SKILL_QUALITY_VERSION, ranAt, disclaimer: QUALITY_DISCLAIMER };
|
|
2555
|
+
let loaded;
|
|
2556
|
+
try {
|
|
2557
|
+
loaded = loadSkill(dir);
|
|
2558
|
+
} catch (e) {
|
|
2559
|
+
return {
|
|
2560
|
+
...base,
|
|
2561
|
+
skillRef: opts.skillRef ?? dir,
|
|
2562
|
+
contentHash: "0x",
|
|
2563
|
+
verdict: "malformed",
|
|
2564
|
+
checks: [{ id: "loadable", status: "fail", detail: e instanceof SkillLoadError ? e.message : "could not load skill" }]
|
|
2565
|
+
};
|
|
2566
|
+
}
|
|
2567
|
+
const checks = [];
|
|
2568
|
+
const name = /(^|\n)name\s*:/i.test(loaded.frontmatter);
|
|
2569
|
+
checks.push(
|
|
2570
|
+
name ? { id: "frontmatter-name", status: "pass", detail: "frontmatter has a name" } : { id: "frontmatter-name", status: "fail", detail: "frontmatter is missing `name`" }
|
|
2571
|
+
);
|
|
2572
|
+
checks.push(
|
|
2573
|
+
loaded.description.trim() ? { id: "frontmatter-description", status: "pass", detail: "frontmatter has a non-empty description" } : { id: "frontmatter-description", status: "fail", detail: "frontmatter is missing a non-empty `description` (the skill's activation trigger)" }
|
|
2574
|
+
);
|
|
2575
|
+
checks.push(
|
|
2576
|
+
loaded.body.trim() ? { id: "body-nonempty", status: "pass", detail: "the instruction body is non-empty" } : { id: "body-nonempty", status: "fail", detail: "the instruction body is empty" }
|
|
2577
|
+
);
|
|
2578
|
+
const relPaths = new Set(loaded.files.map((f) => f.relPath));
|
|
2579
|
+
const broken = brokenBundleLinks(loaded.body, relPaths);
|
|
2580
|
+
checks.push(
|
|
2581
|
+
broken.length === 0 ? { id: "bundled-links-resolve", status: "pass", detail: "all relative links in the body resolve to bundled files" } : { id: "bundled-links-resolve", status: "warn", detail: `broken relative link(s) to: ${broken.slice(0, 5).join(", ")}` }
|
|
2582
|
+
);
|
|
2583
|
+
const verdict = checks.some((c) => c.status === "fail") ? "malformed" : checks.some((c) => c.status === "warn") ? "issues" : "well-formed";
|
|
2584
|
+
return { ...base, skillRef: opts.skillRef ?? dir, contentHash: loaded.contentHash, verdict, checks };
|
|
2585
|
+
}
|
|
2586
|
+
async function runSkillQualityJudged(dir, judge, opts = {}) {
|
|
2587
|
+
const bundle = runSkillQuality(dir, opts);
|
|
2588
|
+
if (bundle.contentHash === "0x") return bundle;
|
|
2589
|
+
try {
|
|
2590
|
+
bundle.judged = await judgeSkillQuality(loadSkill(dir), judge, opts);
|
|
2591
|
+
} catch {
|
|
2592
|
+
}
|
|
2593
|
+
return bundle;
|
|
2594
|
+
}
|
|
2595
|
+
|
|
2160
2596
|
export {
|
|
2161
2597
|
connectTarget,
|
|
2162
2598
|
fingerprintToolDefs,
|
|
@@ -2170,5 +2606,23 @@ export {
|
|
|
2170
2606
|
hasHighSeverity,
|
|
2171
2607
|
gradeFromCategories,
|
|
2172
2608
|
assembleBundle,
|
|
2173
|
-
runLitmus
|
|
2609
|
+
runLitmus,
|
|
2610
|
+
SkillLoadError,
|
|
2611
|
+
loadSkill,
|
|
2612
|
+
stripExamples,
|
|
2613
|
+
skillInjection,
|
|
2614
|
+
skillInjectionFails,
|
|
2615
|
+
exfilInstruction,
|
|
2616
|
+
dangerousCommand,
|
|
2617
|
+
overBroadTrigger,
|
|
2618
|
+
gradeSkillCategories,
|
|
2619
|
+
SKILL_METHODOLOGY_VERSION,
|
|
2620
|
+
SKILL_BUNDLE_SCHEMA_VERSION,
|
|
2621
|
+
runSkillLitmus,
|
|
2622
|
+
openAICompatJudge,
|
|
2623
|
+
judgeFromEnv,
|
|
2624
|
+
judgeSkillQuality,
|
|
2625
|
+
SKILL_QUALITY_VERSION,
|
|
2626
|
+
runSkillQuality,
|
|
2627
|
+
runSkillQualityJudged
|
|
2174
2628
|
};
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
canonicalStringify
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-44R4ZYOE.js";
|
|
4
4
|
|
|
5
5
|
// ../cli/src/litmus.ts
|
|
6
6
|
import { existsSync } from "fs";
|
|
@@ -44,7 +44,7 @@ async function runLitmusCli(args) {
|
|
|
44
44
|
);
|
|
45
45
|
return 2;
|
|
46
46
|
}
|
|
47
|
-
const { runLitmus } = await import("./src-
|
|
47
|
+
const { runLitmus } = await import("./src-TG44QXFV.js");
|
|
48
48
|
const input = resolveTarget(target);
|
|
49
49
|
try {
|
|
50
50
|
const bundle = await runLitmus(input, { headers, allowStateChanging });
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import {
|
|
3
|
+
judgeFromEnv,
|
|
4
|
+
runSkillLitmus,
|
|
5
|
+
runSkillQuality,
|
|
6
|
+
runSkillQualityJudged
|
|
7
|
+
} from "./chunk-DN2OX4RT.js";
|
|
8
|
+
import "./chunk-44R4ZYOE.js";
|
|
9
|
+
|
|
10
|
+
// src/cli-skill.ts
|
|
11
|
+
import { statSync } from "fs";
|
|
12
|
+
var HELP = `polygraphso-litmus-skill \u2014 static safety grades for Claude Code skills.
|
|
13
|
+
|
|
14
|
+
usage:
|
|
15
|
+
polygraphso-litmus-skill [--json] <path-to-skill-dir>
|
|
16
|
+
polygraphso-litmus-skill --help
|
|
17
|
+
|
|
18
|
+
The skill dir must contain a SKILL.md. The safety letter is a STATIC scan (no
|
|
19
|
+
execution); an A means the static checks were clean, not that the skill is
|
|
20
|
+
behaviorally safe.
|
|
21
|
+
|
|
22
|
+
It also prints a separate, advisory quality signal. The optional LLM-judged
|
|
23
|
+
axes (honesty, coherence) run only if you provide your own key \u2014 set
|
|
24
|
+
LITMUS_LLM_API_KEY and LITMUS_LLM_MODEL (and LITMUS_LLM_BASE_URL for a non-OpenAI
|
|
25
|
+
endpoint). Without a key only the deterministic well-formedness checks run.
|
|
26
|
+
More at https://polygraph.so
|
|
27
|
+
`;
|
|
28
|
+
function render(b) {
|
|
29
|
+
const lines = [
|
|
30
|
+
`grade: ${b.grade} (${b.methodologyVersion})`,
|
|
31
|
+
`${b.gradeRationale}`,
|
|
32
|
+
`skill: ${b.skillRef}`,
|
|
33
|
+
`hash: ${b.contentHash}`,
|
|
34
|
+
"",
|
|
35
|
+
"categories:"
|
|
36
|
+
];
|
|
37
|
+
for (const c of b.categories) {
|
|
38
|
+
lines.push(` ${c.code} ${c.status}${c.reason ? ` (${c.reason})` : ""}`);
|
|
39
|
+
if (c.status === "fail") {
|
|
40
|
+
for (const f of c.findings.filter((x) => x.severity === "high").slice(0, 5)) {
|
|
41
|
+
lines.push(` ! ${f.kind}${f.file ? ` [${f.file}]` : ""}: ${f.match}`);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
if (b.advisories.length) {
|
|
46
|
+
lines.push("", "advisories (not part of the grade):");
|
|
47
|
+
for (const f of b.advisories.slice(0, 10)) {
|
|
48
|
+
lines.push(` - ${f.kind} (${f.severity})${f.file ? ` [${f.file}]` : ""}: ${f.match}`);
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
lines.push("", b.disclaimer);
|
|
52
|
+
return lines.join("\n") + "\n";
|
|
53
|
+
}
|
|
54
|
+
function renderQuality(q) {
|
|
55
|
+
const lines = ["", `quality (advisory, separate from the grade): ${q.verdict}`];
|
|
56
|
+
for (const c of q.checks) lines.push(` ${c.status === "pass" ? "\xB7" : "!"} ${c.id}: ${c.detail}`);
|
|
57
|
+
if (q.judged) {
|
|
58
|
+
lines.push(` judged by ${q.judged.judge} (${q.judged.samples} sample(s), agreement ${q.judged.agreement}):`);
|
|
59
|
+
for (const a of q.judged.axes) lines.push(` - ${a.axis}: ${a.rating}`);
|
|
60
|
+
} else {
|
|
61
|
+
lines.push(" (LLM-judged axes not run \u2014 no key/sampling; set LITMUS_LLM_API_KEY + LITMUS_LLM_MODEL to enable)");
|
|
62
|
+
}
|
|
63
|
+
return lines.join("\n") + "\n";
|
|
64
|
+
}
|
|
65
|
+
async function main(argv) {
|
|
66
|
+
const args = argv.filter((a) => a !== "--json");
|
|
67
|
+
const json = argv.includes("--json");
|
|
68
|
+
const target = args[0];
|
|
69
|
+
if (!target || target === "--help" || target === "-h" || target === "help") {
|
|
70
|
+
process.stdout.write(HELP);
|
|
71
|
+
return target ? 0 : 2;
|
|
72
|
+
}
|
|
73
|
+
let st;
|
|
74
|
+
try {
|
|
75
|
+
st = statSync(target);
|
|
76
|
+
} catch {
|
|
77
|
+
process.stderr.write(`polygraphso-litmus-skill: no such path: ${target}
|
|
78
|
+
`);
|
|
79
|
+
return 2;
|
|
80
|
+
}
|
|
81
|
+
if (!st.isDirectory()) {
|
|
82
|
+
process.stderr.write(`polygraphso-litmus-skill: not a directory: ${target} (pass the skill folder containing SKILL.md)
|
|
83
|
+
`);
|
|
84
|
+
return 2;
|
|
85
|
+
}
|
|
86
|
+
const safety = runSkillLitmus(target, { skillRef: target });
|
|
87
|
+
const judge = judgeFromEnv();
|
|
88
|
+
const quality = judge ? await runSkillQualityJudged(target, judge, { skillRef: target }) : runSkillQuality(target, { skillRef: target });
|
|
89
|
+
process.stdout.write(
|
|
90
|
+
json ? JSON.stringify({ safety, quality }, null, 2) + "\n" : render(safety) + renderQuality(quality)
|
|
91
|
+
);
|
|
92
|
+
return 0;
|
|
93
|
+
}
|
|
94
|
+
main(process.argv.slice(2)).then((code) => process.exit(code)).catch((err) => {
|
|
95
|
+
process.stderr.write(`polygraphso-litmus-skill: ${err instanceof Error ? err.message : String(err)}
|
|
96
|
+
`);
|
|
97
|
+
process.exit(1);
|
|
98
|
+
});
|
package/dist/cli.js
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import {
|
|
3
3
|
runLitmusCli
|
|
4
|
-
} from "./chunk-
|
|
4
|
+
} from "./chunk-M5HXKZVN.js";
|
|
5
5
|
import {
|
|
6
6
|
parseServerRef,
|
|
7
7
|
serverKey
|
|
8
|
-
} from "./chunk-
|
|
8
|
+
} from "./chunk-44R4ZYOE.js";
|
|
9
9
|
|
|
10
10
|
// src/cli.ts
|
|
11
11
|
import { readFileSync } from "fs";
|